Add per-server telemetry request diagnostics
This commit is contained in:
@@ -244,6 +244,14 @@ See [Game reporter protocol](docs/beta-test-plans.md#game-reporter-protocol) for
|
||||
complete payload examples. Telemetry does not replace RCON's empty-server
|
||||
check before an automated restart.
|
||||
|
||||
For a missing scoreboard, open the server's **Diagnostics → Telemetry API requests**.
|
||||
The log shows received requests for that exact server name, including accepted
|
||||
reports, rejected payloads, HTTP status, errors, and expandable submitted JSON.
|
||||
It refreshes every five seconds while visible and survives app restarts. Logging
|
||||
starts with this version; older requests cannot be reconstructed. See
|
||||
[Telemetry diagnostics](docs/beta-test-plans.md#telemetry-diagnostics) for retention
|
||||
and how to interpret missing or rejected reports.
|
||||
|
||||
The **Leaderboard** records persistent player totals and map records from these
|
||||
same reports, including highest round, cumulative kills, rounds survived,
|
||||
headshots, revives, downs, and games played. See [Leaderboard](docs/leaderboard.md)
|
||||
|
||||
@@ -94,6 +94,44 @@ and the limits of identifying matches from this protocol.
|
||||
The game scripts are uploaded artifacts, not source maintained in this
|
||||
repository. Their reporter should emit the three payloads above.
|
||||
|
||||
### Telemetry diagnostics
|
||||
|
||||
Open a server's **Diagnostics → Telemetry API requests** to inspect incoming
|
||||
`POST /api/telemetry` calls whose top-level `server` exactly matches its registered
|
||||
name. Each entry includes the Electric Cherry receipt time, HTTP result, error
|
||||
reason, event, map, round, player count, content type, and submitted JSON. Rejected
|
||||
reports are included, even when the content type is wrong but the body is readable
|
||||
JSON. Processing failures return 500 and record the internal error when diagnostic
|
||||
storage is available; failed database transactions do not update the scoreboard.
|
||||
|
||||
The panel refreshes every five seconds while visible and has a manual refresh
|
||||
button. History is stored in SQLite starting with this version, with up to 200
|
||||
requests per server from the last seven days and a fleet-wide cap of 10,000
|
||||
requests. Old entries are pruned when new requests arrive, and expired entries
|
||||
are excluded from reads. Payload previews are limited to 16,384 characters and
|
||||
marked when truncated. Request headers other than content type are not stored.
|
||||
`GET /api/servers/{name}/telemetry-requests` returns the same recent history,
|
||||
newest first, with caching disabled.
|
||||
|
||||
- **No entries:** no attributable requests are retained for this exact name.
|
||||
Check the reporter's destination and server field. Calls that never reach
|
||||
Electric Cherry, malformed JSON, and bodies without a usable server name
|
||||
cannot appear in this server's log. Earlier requests are not backfilled.
|
||||
- **400 / 415:** inspect the error and payload. One invalid player's stats or a
|
||||
duplicate GUID rejects the entire report. These requests do not update the
|
||||
scoreboard.
|
||||
- **404:** the reported server was not registered or had been destroyed at the
|
||||
time of the request.
|
||||
- **200:** Electric Cherry processed the report. An empty `players` array
|
||||
produces an empty scoreboard, including when sent with `game_end`. If a
|
||||
populated accepted report is missing from the scoreboard, investigate the
|
||||
live update/browser path.
|
||||
- **500:** Electric Cherry failed to process the request. Inspect the recorded
|
||||
error and the application log.
|
||||
|
||||
RCON's online status and player count are separate from gameplay telemetry, so
|
||||
an occupied server can still have no scoreboard report.
|
||||
|
||||
## Requirements
|
||||
|
||||
Requirements on a qualifying server are ANDed:
|
||||
|
||||
@@ -367,6 +367,24 @@ CREATE TABLE IF NOT EXISTS server_telemetry (
|
||||
received_at TEXT NOT NULL,
|
||||
payload TEXT
|
||||
);
|
||||
-- Request diagnostics survive registry removal and also capture unknown names.
|
||||
-- Bounded independently of the latest scoreboard and permanent leaderboard.
|
||||
CREATE TABLE IF NOT EXISTS telemetry_requests (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
server TEXT NOT NULL,
|
||||
received_at TEXT NOT NULL,
|
||||
status INTEGER NOT NULL,
|
||||
error TEXT,
|
||||
event TEXT,
|
||||
map TEXT,
|
||||
round INTEGER,
|
||||
player_count INTEGER,
|
||||
content_type TEXT,
|
||||
payload TEXT NOT NULL,
|
||||
payload_truncated INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_telemetry_requests_server ON telemetry_requests(server, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_telemetry_requests_received ON telemetry_requests(received_at);
|
||||
-- Lifecycle history deliberately survives removal of the server registry row.
|
||||
CREATE TABLE IF NOT EXISTS server_deployments (
|
||||
id TEXT PRIMARY KEY,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { TELEMETRY_REQUEST_RETENTION, type TelemetryRequestLogEntry } from '../../telemetry-diagnostics.js';
|
||||
|
||||
const MAX_PAYLOAD_LENGTH = 16 * 1024;
|
||||
// Bound storage even when requests claim many different, unregistered names.
|
||||
const MAX_REQUESTS = 10_000;
|
||||
const cutoff = () => new Date(Date.now() - TELEMETRY_REQUEST_RETENTION.maxAgeDays * 86_400_000).toISOString();
|
||||
const textField = (value: unknown) => typeof value === 'string' ? value.slice(0, 255) : null;
|
||||
|
||||
export function recordTelemetryRequest(input: {
|
||||
value: unknown;
|
||||
receivedAt: string;
|
||||
status: number;
|
||||
error: string | null;
|
||||
contentType: string | null;
|
||||
}): void {
|
||||
// Use only the top-level JSON field, never a player's name or a substring.
|
||||
// Malformed JSON cannot be reliably attributed to an individual server.
|
||||
if (!input.value || typeof input.value !== 'object' || Array.isArray(input.value)) return;
|
||||
const body = input.value as Record<string, unknown>;
|
||||
if (typeof body.server !== 'string' || !body.server.trim() || body.server.length > 255) return;
|
||||
const payload = JSON.stringify(body);
|
||||
const handle = db();
|
||||
handle.transaction(() => {
|
||||
handle.prepare(`INSERT INTO telemetry_requests
|
||||
(server, received_at, status, error, event, map, round, player_count, content_type, payload, payload_truncated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
||||
body.server, input.receivedAt, input.status, input.error?.slice(0, 2000) ?? null,
|
||||
textField(body.event), textField(body.map),
|
||||
Number.isSafeInteger(body.round) ? body.round as number : null,
|
||||
Array.isArray(body.players) ? body.players.length : null,
|
||||
textField(input.contentType), payload.slice(0, MAX_PAYLOAD_LENGTH), Number(payload.length > MAX_PAYLOAD_LENGTH)
|
||||
);
|
||||
handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff());
|
||||
handle.prepare(`DELETE FROM telemetry_requests WHERE server = ? AND id <= (
|
||||
SELECT id FROM telemetry_requests WHERE server = ? ORDER BY id DESC LIMIT 1 OFFSET ?
|
||||
)`).run(body.server, body.server, TELEMETRY_REQUEST_RETENTION.maxPerServer);
|
||||
handle.prepare(`DELETE FROM telemetry_requests WHERE id <= (
|
||||
SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 1 OFFSET ?
|
||||
)`).run(MAX_REQUESTS);
|
||||
})();
|
||||
}
|
||||
|
||||
export function listTelemetryRequests(server: string): TelemetryRequestLogEntry[] {
|
||||
const rows = db().prepare(`SELECT id, server, received_at AS receivedAt, status, error, event, map, round,
|
||||
player_count AS playerCount, content_type AS contentType, payload, payload_truncated AS payloadTruncated
|
||||
FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`)
|
||||
.all(server, cutoff(), TELEMETRY_REQUEST_RETENTION.maxPerServer) as Array<Omit<TelemetryRequestLogEntry, 'payloadTruncated'> & { payloadTruncated: number }>;
|
||||
return rows.map(row => ({ ...row, payloadTruncated: !!row.payloadTruncated }));
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/** A received POST /api/telemetry request, including reports rejected by validation. */
|
||||
export interface TelemetryRequestLogEntry {
|
||||
id: number;
|
||||
server: string;
|
||||
receivedAt: string;
|
||||
status: number;
|
||||
error: string | null;
|
||||
event: string | null;
|
||||
map: string | null;
|
||||
round: number | null;
|
||||
playerCount: number | null;
|
||||
contentType: string | null;
|
||||
payload: string;
|
||||
payloadTruncated: boolean;
|
||||
}
|
||||
|
||||
export const TELEMETRY_REQUEST_RETENTION = { maxPerServer: 200, maxAgeDays: 7 } as const;
|
||||
@@ -0,0 +1,198 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { TELEMETRY_REQUEST_RETENTION, type TelemetryRequestLogEntry } from '$lib/telemetry-diagnostics.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import Card from './primitives/Card.svelte';
|
||||
import Badge from './primitives/Badge.svelte';
|
||||
import Button from './primitives/Button.svelte';
|
||||
import EmptyState from './primitives/EmptyState.svelte';
|
||||
import ListFooter from './primitives/ListFooter.svelte';
|
||||
|
||||
let { server }: { server: string } = $props();
|
||||
let requests = $state<TelemetryRequestLogEntry[]>([]);
|
||||
let retention = $state<{ maxPerServer: number; maxAgeDays: number }>({ ...TELEMETRY_REQUEST_RETENTION });
|
||||
let refreshing = $state(false);
|
||||
let loaded = $state(false);
|
||||
let error = $state('');
|
||||
let refreshedAt = $state<string | null>(null);
|
||||
let limit = $state(25);
|
||||
let active = false;
|
||||
let controller: AbortController | null = null;
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
if (!active || refreshing) return;
|
||||
const currentServer = server;
|
||||
const requestController = new AbortController();
|
||||
controller = requestController;
|
||||
refreshing = true;
|
||||
try {
|
||||
const response = await fetch(`/api/servers/${encodeURIComponent(currentServer)}/telemetry-requests`, {
|
||||
cache: 'no-store',
|
||||
signal: requestController.signal
|
||||
});
|
||||
const body = await response.json().catch(() => null);
|
||||
if (!response.ok || !body?.ok || !Array.isArray(body.requests)) {
|
||||
throw new Error(body?.error || `Could not load telemetry requests (HTTP ${response.status}).`);
|
||||
}
|
||||
if (!active || server !== currentServer) return;
|
||||
requests = body.requests;
|
||||
retention = body.retention;
|
||||
loaded = true;
|
||||
error = '';
|
||||
refreshedAt = new Date().toISOString();
|
||||
} catch (cause) {
|
||||
if (active && server === currentServer && !requestController.signal.aborted) {
|
||||
error = cause instanceof Error ? cause.message : 'Could not load telemetry requests.';
|
||||
}
|
||||
} finally {
|
||||
if (active && server === currentServer) refreshing = false;
|
||||
if (controller === requestController) controller = null;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
active = true;
|
||||
const stopPolling = pollVisible(refresh, 5000);
|
||||
return () => {
|
||||
active = false;
|
||||
stopPolling();
|
||||
controller?.abort();
|
||||
};
|
||||
});
|
||||
|
||||
function timestamp(value: string): string {
|
||||
return new Date(value).toLocaleString(undefined, {
|
||||
year: 'numeric', month: 'short', day: 'numeric',
|
||||
hour: '2-digit', minute: '2-digit', second: '2-digit', timeZoneName: 'short'
|
||||
});
|
||||
}
|
||||
|
||||
function outcome(status: number): string {
|
||||
return status >= 500 ? 'Failed' : status >= 400 ? 'Rejected' : 'Accepted';
|
||||
}
|
||||
</script>
|
||||
|
||||
<Card padding="lg" as="section" ariaLabel="Telemetry API requests">
|
||||
<div class="panel-head">
|
||||
<div>
|
||||
<h2>Telemetry API requests</h2>
|
||||
<p class="server-name"><code>{server}</code></p>
|
||||
</div>
|
||||
<Button size="sm" onclick={refresh} disabled={refreshing} loading={refreshing}>Refresh</Button>
|
||||
</div>
|
||||
|
||||
<p class="intro">
|
||||
Incoming <code>POST /api/telemetry</code> requests with this exact server name.
|
||||
The scoreboard uses telemetry; the fleet player count comes from RCON separately.
|
||||
</p>
|
||||
<p class="retention">
|
||||
Newest first · up to {retention.maxPerServer} requests from the last {retention.maxAgeDays} days.
|
||||
Refreshes every 5 seconds while visible.
|
||||
{#if refreshedAt}<span>Last refreshed <time datetime={refreshedAt}>{timestamp(refreshedAt)}</time>.</span>{/if}
|
||||
</p>
|
||||
|
||||
{#if error}
|
||||
<p class="load-error" role="alert">
|
||||
{error}{#if loaded} Previously loaded requests are shown below.{/if}
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
{#if !loaded && !error}
|
||||
<EmptyState title="Loading telemetry requests…" />
|
||||
{:else if loaded && requests.length === 0}
|
||||
<EmptyState
|
||||
title="No telemetry requests recorded"
|
||||
description="No matching requests are retained for this server. An empty log alone does not establish a network problem."
|
||||
/>
|
||||
{:else if requests.length > 0}
|
||||
<ol class="request-list" aria-label="Recent telemetry requests">
|
||||
{#each requests.slice(0, limit) as request (request.id)}
|
||||
<li class="request">
|
||||
<div class="request-head">
|
||||
<div class="request-result">
|
||||
<Badge variant={request.status >= 500 ? 'danger' : request.status >= 400 ? 'warning' : 'online'}>
|
||||
{outcome(request.status)} · HTTP {request.status}
|
||||
</Badge>
|
||||
<code class="event">{request.event ?? 'Event not provided'}</code>
|
||||
</div>
|
||||
<time class="received" datetime={request.receivedAt}>{timestamp(request.receivedAt)}</time>
|
||||
</div>
|
||||
<dl class="request-meta">
|
||||
<div><dt>Map</dt><dd><code>{request.map ?? 'Not provided'}</code></dd></div>
|
||||
<div><dt>Round</dt><dd>{request.round ?? 'Not provided'}</dd></div>
|
||||
<div><dt>Players</dt><dd>{request.playerCount ?? 'Not provided'}</dd></div>
|
||||
</dl>
|
||||
{#if request.error}<p class="request-error">{request.error}</p>{/if}
|
||||
<details class="payload">
|
||||
<summary>Request payload{#if request.payloadTruncated} (truncated){/if}</summary>
|
||||
<p class="content-type">Content type: <code>{request.contentType ?? 'Not provided'}</code></p>
|
||||
<pre>{request.payload}</pre>
|
||||
{#if request.payloadTruncated}<p class="truncation">Only the beginning of this payload was retained.</p>{/if}
|
||||
</details>
|
||||
</li>
|
||||
{/each}
|
||||
</ol>
|
||||
<ListFooter total={requests.length} bind:limit label="requests" />
|
||||
{/if}
|
||||
|
||||
<p class="attribution">
|
||||
Logging starts after this update is installed.
|
||||
Requests can be linked here only when readable JSON contains a matching top-level <code>server</code> field.
|
||||
Calls that never reach Electric Cherry, malformed JSON, and different server names will not appear here.
|
||||
</p>
|
||||
</Card>
|
||||
|
||||
<style>
|
||||
.panel-head, .request-head, .request-result {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-3);
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.panel-head { margin-bottom: var(--space-4); }
|
||||
h2 { margin: 0; font-size: var(--text-md); font-weight: var(--weight-semibold); }
|
||||
p { margin: 0; }
|
||||
code, pre { font-family: var(--font-mono); }
|
||||
.server-name { margin-top: var(--space-2); overflow-wrap: anywhere; color: var(--text-secondary); font-size: var(--text-xs); }
|
||||
.intro { font-size: var(--text-sm); color: var(--text-secondary); line-height: 1.6; }
|
||||
.retention, .attribution, .content-type, .truncation {
|
||||
font-size: var(--text-xs);
|
||||
color: var(--text-tertiary);
|
||||
line-height: 1.6;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.retention { margin-top: var(--space-2); }
|
||||
.retention span { display: block; }
|
||||
.attribution { margin-top: var(--space-4); }
|
||||
.request-list { list-style: none; padding: 0; margin: var(--space-5) 0 0; }
|
||||
.request { padding: var(--space-4) 0; border-top: 1px solid var(--border-subtle); }
|
||||
.request-result { justify-content: flex-start; min-width: 0; }
|
||||
.event { font-size: var(--text-sm); overflow-wrap: anywhere; }
|
||||
.received { color: var(--text-tertiary); font-size: var(--text-xs); font-variant-numeric: tabular-nums; }
|
||||
.request-meta { display: flex; flex-wrap: wrap; gap: var(--space-3) var(--space-6); margin: var(--space-3) 0; font-size: var(--text-xs); }
|
||||
.request-meta div { display: flex; gap: var(--space-2); min-width: 0; }
|
||||
dt { color: var(--text-tertiary); }
|
||||
dd { margin: 0; color: var(--text-secondary); overflow-wrap: anywhere; }
|
||||
.load-error, .request-error { color: var(--state-danger); font-size: var(--text-sm); overflow-wrap: anywhere; }
|
||||
.load-error { margin-top: var(--space-4); }
|
||||
.request-error { margin-bottom: var(--space-3); }
|
||||
.payload { font-size: var(--text-xs); }
|
||||
summary { color: var(--text-secondary); cursor: pointer; width: fit-content; }
|
||||
summary:focus-visible { outline: none; box-shadow: var(--focus-ring); border-radius: var(--radius-sm); }
|
||||
.content-type { margin-top: var(--space-3); }
|
||||
pre {
|
||||
margin: var(--space-2) 0 0;
|
||||
padding: var(--space-3);
|
||||
max-height: 24rem;
|
||||
overflow: auto;
|
||||
white-space: pre-wrap;
|
||||
overflow-wrap: anywhere;
|
||||
background: var(--surface-sunken);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-sm);
|
||||
color: var(--text-secondary);
|
||||
line-height: 1.6;
|
||||
}
|
||||
.truncation { margin-top: var(--space-2); }
|
||||
</style>
|
||||
@@ -0,0 +1,11 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { getServer } from '$lib/server/fleet/servers.js';
|
||||
import { listTelemetryRequests } from '$lib/server/fleet/telemetry-requests.js';
|
||||
import { TELEMETRY_REQUEST_RETENTION } from '$lib/telemetry-diagnostics.js';
|
||||
|
||||
export async function GET({ params }: { params: { name: string } }) {
|
||||
if (!getServer(params.name)) return json({ ok: false, error: 'no such server' }, { status: 404 });
|
||||
return json({ ok: true, requests: listTelemetryRequests(params.name), retention: TELEMETRY_REQUEST_RETENTION }, {
|
||||
headers: { 'Cache-Control': 'no-store' }
|
||||
});
|
||||
}
|
||||
@@ -2,26 +2,51 @@ import { json } from '@sveltejs/kit';
|
||||
import { parseGameEvent, type GameTelemetry } from '$lib/telemetry.js';
|
||||
import { getServer } from '$lib/server/fleet/servers.js';
|
||||
import { saveTelemetry } from '$lib/server/fleet/telemetry.js';
|
||||
import { recordTelemetryRequest } from '$lib/server/fleet/telemetry-requests.js';
|
||||
import { recordGameEvent } from '$lib/server/fleet/beta-tests.js';
|
||||
import { db } from '$lib/server/db/index.js';
|
||||
import { emit } from '$lib/server/pipeline/bus.js';
|
||||
|
||||
export async function POST({ request }: { request: Request }) {
|
||||
if (request.headers.get('content-type')?.split(';')[0].trim().toLowerCase() !== 'application/json') {
|
||||
return json({ ok: false, error: 'Content-Type must be application/json' }, { status: 415 });
|
||||
}
|
||||
const receivedAt = new Date().toISOString();
|
||||
const contentType = request.headers.get('content-type');
|
||||
let value: unknown;
|
||||
let malformed = false;
|
||||
try { value = await request.json(); }
|
||||
catch { return json({ ok: false, error: 'Malformed JSON' }, { status: 400 }); }
|
||||
catch { malformed = true; }
|
||||
|
||||
function log(status: number, error: string | null): void {
|
||||
try { recordTelemetryRequest({ value, receivedAt, status, error, contentType }); }
|
||||
catch (error) {
|
||||
// Diagnostics must not turn a saved report into a failed request.
|
||||
console.error('Could not record telemetry request diagnostics', error);
|
||||
}
|
||||
}
|
||||
function reject(status: number, error: string): Response {
|
||||
log(status, error);
|
||||
return json({ ok: false, error }, { status });
|
||||
}
|
||||
// Even a wrong content type may carry readable JSON with a server name.
|
||||
if (contentType?.split(';')[0].trim().toLowerCase() !== 'application/json') {
|
||||
return reject(415, 'Content-Type must be application/json');
|
||||
}
|
||||
if (malformed) return reject(400, 'Malformed JSON');
|
||||
let telemetry: GameTelemetry;
|
||||
try { telemetry = { ...parseGameEvent(value), receivedAt: new Date().toISOString() }; }
|
||||
catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); }
|
||||
try { telemetry = { ...parseGameEvent(value), receivedAt }; }
|
||||
catch (error) { return reject(400, (error as Error).message); }
|
||||
try {
|
||||
const server = getServer(telemetry.server);
|
||||
if (!server || server.destroyed_at) return json({ ok: false, error: 'Server is not registered or has been destroyed' }, { status: 404 });
|
||||
if (!server || server.destroyed_at) return reject(404, 'Server is not registered or has been destroyed');
|
||||
db().transaction(() => {
|
||||
saveTelemetry(telemetry);
|
||||
recordGameEvent(telemetry);
|
||||
})();
|
||||
emit({ type: 'server-telemetry', entries: [telemetry] });
|
||||
} catch (error) {
|
||||
console.error('Could not process telemetry', error);
|
||||
log(500, `Could not process telemetry: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return json({ ok: false, error: 'Could not process telemetry' }, { status: 500 });
|
||||
}
|
||||
log(200, null);
|
||||
return json({ ok: true, telemetry });
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<script lang="ts">
|
||||
import RoundTelemetry from '$ui/components/RoundTelemetry.svelte';
|
||||
import TelemetryDiagnostics from '$ui/components/TelemetryDiagnostics.svelte';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } from '$ui/stores/fleet.js';
|
||||
import { freshServer, fleetIdentity } from '$lib/fleet-overview.js';
|
||||
@@ -284,7 +285,7 @@
|
||||
<Tabs {tabs} bind:value={activeTab} ariaLabel="Server sections" idPrefix="server-tabs" panelId="server-panel" />
|
||||
</div>
|
||||
|
||||
<div class="tab-panel" role="tabpanel" id="server-panel" aria-labelledby={"server-tabs-" + activeTab}>
|
||||
<div class="tab-panel" class:diagnostics={activeTab === 'diagnostics'} role="tabpanel" id="server-panel" aria-labelledby={"server-tabs-" + activeTab}>
|
||||
{#if tabError}<div class="ui-error" role="alert">{tabError}<Button size="sm" onclick={() => loadTab(name, activeTab)}>Retry</Button></div>{/if}
|
||||
{#if activeTab === 'scoreboard'}
|
||||
<Card padding="lg">
|
||||
@@ -328,6 +329,9 @@
|
||||
</Card>
|
||||
<ListFooter total={inventory.length} bind:limit={inventoryLimit} label="files" />
|
||||
{:else if activeTab === 'diagnostics'}
|
||||
{#key name}
|
||||
<TelemetryDiagnostics server={name} />
|
||||
{/key}
|
||||
<Card padding="lg">
|
||||
<div class="panel-head">
|
||||
<h2>RCON diagnostic</h2>
|
||||
@@ -575,6 +579,10 @@
|
||||
}
|
||||
|
||||
/* Tabs row */
|
||||
.tab-panel.diagnostics {
|
||||
display: grid;
|
||||
gap: var(--space-6);
|
||||
}
|
||||
.tabs-row {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { gameEvent, postTelemetry as post } from '../helpers/telemetry.js';
|
||||
|
||||
const server = 't5_zombie_theater_1';
|
||||
const otherServer = 't5_zombie_theater_2';
|
||||
let fixture: Fixture;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture([server, otherServer]);
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
});
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await fixture.cleanup();
|
||||
});
|
||||
|
||||
async function diagnostics(name = server) {
|
||||
const { GET } = await import('../../src/routes/api/servers/[name]/telemetry-requests/+server.js');
|
||||
return GET({ params: { name } });
|
||||
}
|
||||
|
||||
it('returns uncached, empty diagnostics for a registered server before any requests arrive', async () => {
|
||||
const response = await diagnostics();
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('cache-control')).toContain('no-store');
|
||||
expect(await response.json()).toEqual({
|
||||
ok: true, requests: [], retention: { maxPerServer: 200, maxAgeDays: 7 }
|
||||
});
|
||||
expect((await diagnostics('unknown')).status).toBe(404);
|
||||
});
|
||||
|
||||
it('persists the received payload and response, isolates each server, and lists newest requests first', async () => {
|
||||
const first = gameEvent('round_change', 1);
|
||||
const latest = { ...gameEvent('round_change', 3), scriptVersion: 'diagnostic-version' };
|
||||
expect((await post(first)).status).toBe(200);
|
||||
expect((await post(gameEvent('round_change', 2, 1, otherServer))).status).toBe(200);
|
||||
expect((await post(latest)).status).toBe(200);
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
|
||||
const body = await (await diagnostics()).json();
|
||||
expect(body.requests).toHaveLength(2);
|
||||
expect(body.requests[0]).toEqual({
|
||||
id: expect.any(Number), server, receivedAt: expect.any(String), status: 200, error: null,
|
||||
event: 'round_change', map: 'zombie_theater', round: 3, playerCount: 4,
|
||||
contentType: 'application/json', payload: JSON.stringify(latest), payloadTruncated: false
|
||||
});
|
||||
expect(body.requests[0].id).toBeGreaterThan(body.requests[1].id);
|
||||
expect(body.requests[1].payload).toBe(JSON.stringify(first));
|
||||
expect((await (await diagnostics(otherServer)).json()).requests).toEqual([
|
||||
expect.objectContaining({ server: otherServer, round: 2, playerCount: 1 })
|
||||
]);
|
||||
});
|
||||
|
||||
it('records rejected player data with the same validation reason returned to the telemetry client', async () => {
|
||||
const invalid = { ...gameEvent('round_change', 5), players: [{ name: 'Player', guid: '123' }] };
|
||||
const response = await post(invalid);
|
||||
expect(response.status).toBe(400);
|
||||
const result = await response.json();
|
||||
expect(result.error).toContain('players[0].kills');
|
||||
const { requests } = await (await diagnostics()).json();
|
||||
expect(requests).toEqual([expect.objectContaining({
|
||||
server, status: 400, error: result.error, event: 'round_change', round: 5,
|
||||
playerCount: 1, payload: JSON.stringify(invalid), payloadTruncated: false
|
||||
})]);
|
||||
const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
||||
expect(listTelemetry()).toEqual([]);
|
||||
});
|
||||
|
||||
it('records identifiable JSON sent with an unsupported or missing Content-Type', async () => {
|
||||
const payload = gameEvent('round_change', 2);
|
||||
expect((await post(payload, 'text/plain')).status).toBe(415);
|
||||
const { POST } = await import('../../src/routes/api/telemetry/+server.js');
|
||||
// A byte body avoids Request automatically assigning text/plain to a string body.
|
||||
const request = new Request('http://localhost/api/telemetry', {
|
||||
method: 'POST', body: new TextEncoder().encode(JSON.stringify(payload))
|
||||
});
|
||||
expect(request.headers.get('content-type')).toBeNull();
|
||||
expect((await POST({ request })).status).toBe(415);
|
||||
const { requests } = await (await diagnostics()).json();
|
||||
expect(requests).toEqual([
|
||||
expect.objectContaining({ status: 415, contentType: null, error: 'Content-Type must be application/json' }),
|
||||
expect.objectContaining({ status: 415, contentType: 'text/plain', payload: JSON.stringify(payload) })
|
||||
]);
|
||||
});
|
||||
|
||||
it('distinguishes a zero round and an empty scoreboard from absent or invalid fields', async () => {
|
||||
await post(gameEvent('round_change', 0, 0));
|
||||
await post(gameEvent('game_start', 0, 0));
|
||||
await post({ server, players: 4 });
|
||||
const { requests } = await (await diagnostics()).json();
|
||||
expect(requests).toEqual([
|
||||
expect.objectContaining({ status: 400, event: null, map: null, round: null, playerCount: null }),
|
||||
expect.objectContaining({ status: 200, event: 'game_start', round: null, playerCount: 0 }),
|
||||
expect.objectContaining({ status: 200, event: 'round_change', round: 0, playerCount: 0 })
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps unknown and destroyed server rejections and matches the submitted server name exactly', async () => {
|
||||
const unknown = 't5_zombie_theater_99';
|
||||
const spaced = ` ${server} `;
|
||||
expect((await post(gameEvent('game_start', 0, 0, unknown))).status).toBe(404);
|
||||
expect((await post(gameEvent('game_start', 0, 0, spaced))).status).toBe(404);
|
||||
await post(gameEvent('game_start'));
|
||||
const { setServerField } = await import('../../src/lib/server/fleet/servers.js');
|
||||
setServerField(server, 'destroyed_at', new Date().toISOString());
|
||||
expect((await post(gameEvent('game_end', 1))).status).toBe(404);
|
||||
|
||||
const { listTelemetryRequests } = await import('../../src/lib/server/fleet/telemetry-requests.js');
|
||||
expect(listTelemetryRequests(unknown)).toEqual([expect.objectContaining({ server: unknown, status: 404 })]);
|
||||
expect(listTelemetryRequests(spaced)).toEqual([expect.objectContaining({ server: spaced, status: 404 })]);
|
||||
expect((await diagnostics(unknown)).status).toBe(404);
|
||||
const response = await diagnostics();
|
||||
expect(response.status).toBe(200);
|
||||
expect((await response.json()).requests).toEqual([
|
||||
expect.objectContaining({ server, status: 404, error: 'Server is not registered or has been destroyed' }),
|
||||
expect.objectContaining({ server, status: 200 })
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not attribute malformed JSON or unusable server names to a server', async () => {
|
||||
const { POST } = await import('../../src/routes/api/telemetry/+server.js');
|
||||
const malformed = new Request('http://localhost/api/telemetry', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: `{"server":"${server}",`
|
||||
});
|
||||
expect((await POST({ request: malformed })).status).toBe(400);
|
||||
for (const value of [undefined, null, '', ' ', 123, 's'.repeat(256)]) {
|
||||
expect((await post({ ...gameEvent('game_start'), server: value })).status).toBe(400);
|
||||
}
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ count: 0 });
|
||||
});
|
||||
|
||||
it('bounds stored payloads and marks truncation without rejecting a valid event', async () => {
|
||||
const payload = { ...gameEvent('round_change', 4), diagnostic: 'x'.repeat(20_000) };
|
||||
expect((await post(payload)).status).toBe(200);
|
||||
const { requests } = await (await diagnostics()).json();
|
||||
expect(requests[0]).toMatchObject({ status: 200, round: 4, playerCount: 4, payloadTruncated: true });
|
||||
expect(requests[0].payload).toBe(JSON.stringify(payload).slice(0, 16 * 1024));
|
||||
});
|
||||
|
||||
it('records processing failures after rollback without changing the scoreboard, match state, or broadcasts', async () => {
|
||||
const first = gameEvent('round_change', 3);
|
||||
await post(first);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
||||
const beta = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
const snapshot = listTelemetry();
|
||||
const matches = db().prepare('SELECT * FROM leaderboard_matches ORDER BY id').all();
|
||||
const listener = vi.fn();
|
||||
const unsubscribe = subscribe(listener);
|
||||
vi.spyOn(beta, 'recordGameEvent').mockImplementationOnce(() => { throw new Error('Injected beta processing failure'); });
|
||||
const response = await post(gameEvent('game_end', 99));
|
||||
unsubscribe();
|
||||
|
||||
expect(response.status).toBe(500);
|
||||
expect(await response.json()).toEqual({ ok: false, error: 'Could not process telemetry' });
|
||||
expect(listTelemetry()).toEqual(snapshot);
|
||||
expect(getServer(server)?.game_ended_at).toBeNull();
|
||||
expect(db().prepare('SELECT * FROM leaderboard_matches ORDER BY id').all()).toEqual(matches);
|
||||
expect(listener).not.toHaveBeenCalled();
|
||||
expect((await (await diagnostics()).json()).requests).toEqual([
|
||||
expect.objectContaining({ status: 500, event: 'game_end', round: 99, error: 'Could not process telemetry: Injected beta processing failure' }),
|
||||
expect.objectContaining({ status: 200, round: 3 })
|
||||
]);
|
||||
});
|
||||
|
||||
it('still accepts and broadcasts telemetry if storing diagnostics fails', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().exec(`CREATE TRIGGER reject_diagnostics BEFORE INSERT ON telemetry_requests
|
||||
BEGIN SELECT RAISE(FAIL, 'Injected diagnostic storage failure'); END`);
|
||||
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
||||
const listener = vi.fn();
|
||||
const unsubscribe = subscribe(listener);
|
||||
const payload = gameEvent('round_change', 7);
|
||||
const response = await post(payload);
|
||||
unsubscribe();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
||||
expect(listTelemetry()).toEqual([{ ...payload, receivedAt: expect.any(String) }]);
|
||||
expect(listener).toHaveBeenCalledWith({
|
||||
type: 'server-telemetry', entries: [{ ...payload, receivedAt: expect.any(String) }]
|
||||
});
|
||||
});
|
||||
|
||||
it('retains only the latest 200 requests for a server and removes requests older than seven days', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { recordTelemetryRequest, listTelemetryRequests } = await import('../../src/lib/server/fleet/telemetry-requests.js');
|
||||
const now = new Date().toISOString();
|
||||
const old = new Date(Date.now() - 8 * 86_400_000).toISOString();
|
||||
const insert = db().prepare(`INSERT INTO telemetry_requests
|
||||
(server, received_at, status, payload, payload_truncated) VALUES (?, ?, 200, '{}', 0)`);
|
||||
db().transaction(() => {
|
||||
insert.run(otherServer, old);
|
||||
insert.run(otherServer, now);
|
||||
for (let i = 0; i < 205; i++) insert.run(server, now);
|
||||
})();
|
||||
recordTelemetryRequest({ value: gameEvent('round_change', 9), receivedAt: now, status: 200, error: null, contentType: 'application/json' });
|
||||
|
||||
const requests = listTelemetryRequests(server);
|
||||
expect(requests).toHaveLength(200);
|
||||
expect(requests[0]).toMatchObject({ round: 9 });
|
||||
expect(requests.map(entry => entry.id)).toEqual([...requests.map(entry => entry.id)].sort((a, b) => b - a));
|
||||
expect(listTelemetryRequests(otherServer)).toHaveLength(1);
|
||||
expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests WHERE received_at = ?').get(old)).toEqual({ count: 0 });
|
||||
});
|
||||
|
||||
it('caps global diagnostic history even when rejected requests name many unknown servers', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { recordTelemetryRequest } = await import('../../src/lib/server/fleet/telemetry-requests.js');
|
||||
const now = new Date().toISOString();
|
||||
const insert = db().prepare(`INSERT INTO telemetry_requests
|
||||
(server, received_at, status, payload, payload_truncated) VALUES (?, ?, 404, '{}', 0)`);
|
||||
db().transaction(() => {
|
||||
for (let i = 0; i < 10_005; i++) insert.run(`unknown-${i}`, now);
|
||||
})();
|
||||
recordTelemetryRequest({ value: gameEvent('game_start'), receivedAt: now, status: 200, error: null, contentType: 'application/json' });
|
||||
expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ count: 10_000 });
|
||||
expect(db().prepare('SELECT server FROM telemetry_requests ORDER BY id DESC LIMIT 1').get()).toEqual({ server });
|
||||
expect(db().prepare("SELECT id FROM telemetry_requests WHERE server = 'unknown-0'").get()).toBeUndefined();
|
||||
});
|
||||
Reference in New Issue
Block a user