Add per-server telemetry request diagnostics
This commit is contained in:
@@ -367,6 +367,24 @@ CREATE TABLE IF NOT EXISTS server_telemetry (
|
||||
received_at TEXT NOT NULL,
|
||||
payload TEXT
|
||||
);
|
||||
-- Request diagnostics survive registry removal and also capture unknown names.
|
||||
-- Bounded independently of the latest scoreboard and permanent leaderboard.
|
||||
CREATE TABLE IF NOT EXISTS telemetry_requests (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
server TEXT NOT NULL,
|
||||
received_at TEXT NOT NULL,
|
||||
status INTEGER NOT NULL,
|
||||
error TEXT,
|
||||
event TEXT,
|
||||
map TEXT,
|
||||
round INTEGER,
|
||||
player_count INTEGER,
|
||||
content_type TEXT,
|
||||
payload TEXT NOT NULL,
|
||||
payload_truncated INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_telemetry_requests_server ON telemetry_requests(server, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_telemetry_requests_received ON telemetry_requests(received_at);
|
||||
-- Lifecycle history deliberately survives removal of the server registry row.
|
||||
CREATE TABLE IF NOT EXISTS server_deployments (
|
||||
id TEXT PRIMARY KEY,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { TELEMETRY_REQUEST_RETENTION, type TelemetryRequestLogEntry } from '../../telemetry-diagnostics.js';
|
||||
|
||||
const MAX_PAYLOAD_LENGTH = 16 * 1024;
|
||||
// Bound storage even when requests claim many different, unregistered names.
|
||||
const MAX_REQUESTS = 10_000;
|
||||
const cutoff = () => new Date(Date.now() - TELEMETRY_REQUEST_RETENTION.maxAgeDays * 86_400_000).toISOString();
|
||||
const textField = (value: unknown) => typeof value === 'string' ? value.slice(0, 255) : null;
|
||||
|
||||
export function recordTelemetryRequest(input: {
|
||||
value: unknown;
|
||||
receivedAt: string;
|
||||
status: number;
|
||||
error: string | null;
|
||||
contentType: string | null;
|
||||
}): void {
|
||||
// Use only the top-level JSON field, never a player's name or a substring.
|
||||
// Malformed JSON cannot be reliably attributed to an individual server.
|
||||
if (!input.value || typeof input.value !== 'object' || Array.isArray(input.value)) return;
|
||||
const body = input.value as Record<string, unknown>;
|
||||
if (typeof body.server !== 'string' || !body.server.trim() || body.server.length > 255) return;
|
||||
const payload = JSON.stringify(body);
|
||||
const handle = db();
|
||||
handle.transaction(() => {
|
||||
handle.prepare(`INSERT INTO telemetry_requests
|
||||
(server, received_at, status, error, event, map, round, player_count, content_type, payload, payload_truncated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
||||
body.server, input.receivedAt, input.status, input.error?.slice(0, 2000) ?? null,
|
||||
textField(body.event), textField(body.map),
|
||||
Number.isSafeInteger(body.round) ? body.round as number : null,
|
||||
Array.isArray(body.players) ? body.players.length : null,
|
||||
textField(input.contentType), payload.slice(0, MAX_PAYLOAD_LENGTH), Number(payload.length > MAX_PAYLOAD_LENGTH)
|
||||
);
|
||||
handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff());
|
||||
handle.prepare(`DELETE FROM telemetry_requests WHERE server = ? AND id <= (
|
||||
SELECT id FROM telemetry_requests WHERE server = ? ORDER BY id DESC LIMIT 1 OFFSET ?
|
||||
)`).run(body.server, body.server, TELEMETRY_REQUEST_RETENTION.maxPerServer);
|
||||
handle.prepare(`DELETE FROM telemetry_requests WHERE id <= (
|
||||
SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 1 OFFSET ?
|
||||
)`).run(MAX_REQUESTS);
|
||||
})();
|
||||
}
|
||||
|
||||
export function listTelemetryRequests(server: string): TelemetryRequestLogEntry[] {
|
||||
const rows = db().prepare(`SELECT id, server, received_at AS receivedAt, status, error, event, map, round,
|
||||
player_count AS playerCount, content_type AS contentType, payload, payload_truncated AS payloadTruncated
|
||||
FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`)
|
||||
.all(server, cutoff(), TELEMETRY_REQUEST_RETENTION.maxPerServer) as Array<Omit<TelemetryRequestLogEntry, 'payloadTruncated'> & { payloadTruncated: number }>;
|
||||
return rows.map(row => ({ ...row, payloadTruncated: !!row.payloadTruncated }));
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/** A received POST /api/telemetry request, including reports rejected by validation. */
|
||||
export interface TelemetryRequestLogEntry {
|
||||
id: number;
|
||||
server: string;
|
||||
receivedAt: string;
|
||||
status: number;
|
||||
error: string | null;
|
||||
event: string | null;
|
||||
map: string | null;
|
||||
round: number | null;
|
||||
playerCount: number | null;
|
||||
contentType: string | null;
|
||||
payload: string;
|
||||
payloadTruncated: boolean;
|
||||
}
|
||||
|
||||
export const TELEMETRY_REQUEST_RETENTION = { maxPerServer: 200, maxAgeDays: 7 } as const;
|
||||
@@ -0,0 +1,198 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { TELEMETRY_REQUEST_RETENTION, type TelemetryRequestLogEntry } from '$lib/telemetry-diagnostics.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import Card from './primitives/Card.svelte';
|
||||
import Badge from './primitives/Badge.svelte';
|
||||
import Button from './primitives/Button.svelte';
|
||||
import EmptyState from './primitives/EmptyState.svelte';
|
||||
import ListFooter from './primitives/ListFooter.svelte';
|
||||
|
||||
let { server }: { server: string } = $props();
|
||||
let requests = $state<TelemetryRequestLogEntry[]>([]);
|
||||
let retention = $state<{ maxPerServer: number; maxAgeDays: number }>({ ...TELEMETRY_REQUEST_RETENTION });
|
||||
let refreshing = $state(false);
|
||||
let loaded = $state(false);
|
||||
let error = $state('');
|
||||
let refreshedAt = $state<string | null>(null);
|
||||
let limit = $state(25);
|
||||
let active = false;
|
||||
let controller: AbortController | null = null;
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
if (!active || refreshing) return;
|
||||
const currentServer = server;
|
||||
const requestController = new AbortController();
|
||||
controller = requestController;
|
||||
refreshing = true;
|
||||
try {
|
||||
const response = await fetch(`/api/servers/${encodeURIComponent(currentServer)}/telemetry-requests`, {
|
||||
cache: 'no-store',
|
||||
signal: requestController.signal
|
||||
});
|
||||
const body = await response.json().catch(() => null);
|
||||
if (!response.ok || !body?.ok || !Array.isArray(body.requests)) {
|
||||
throw new Error(body?.error || `Could not load telemetry requests (HTTP ${response.status}).`);
|
||||
}
|
||||
if (!active || server !== currentServer) return;
|
||||
requests = body.requests;
|
||||
retention = body.retention;
|
||||
loaded = true;
|
||||
error = '';
|
||||
refreshedAt = new Date().toISOString();
|
||||
} catch (cause) {
|
||||
if (active && server === currentServer && !requestController.signal.aborted) {
|
||||
error = cause instanceof Error ? cause.message : 'Could not load telemetry requests.';
|
||||
}
|
||||
} finally {
|
||||
if (active && server === currentServer) refreshing = false;
|
||||
if (controller === requestController) controller = null;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
active = true;
|
||||
const stopPolling = pollVisible(refresh, 5000);
|
||||
return () => {
|
||||
active = false;
|
||||
stopPolling();
|
||||
controller?.abort();
|
||||
};
|
||||
});
|
||||
|
||||
function timestamp(value: string): string {
|
||||
return new Date(value).toLocaleString(undefined, {
|
||||
year: 'numeric', month: 'short', day: 'numeric',
|
||||
hour: '2-digit', minute: '2-digit', second: '2-digit', timeZoneName: 'short'
|
||||
});
|
||||
}
|
||||
|
||||
function outcome(status: number): string {
|
||||
return status >= 500 ? 'Failed' : status >= 400 ? 'Rejected' : 'Accepted';
|
||||
}
|
||||
</script>
|
||||
|
||||
<Card padding="lg" as="section" ariaLabel="Telemetry API requests">
|
||||
<div class="panel-head">
|
||||
<div>
|
||||
<h2>Telemetry API requests</h2>
|
||||
<p class="server-name"><code>{server}</code></p>
|
||||
</div>
|
||||
<Button size="sm" onclick={refresh} disabled={refreshing} loading={refreshing}>Refresh</Button>
|
||||
</div>
|
||||
|
||||
<p class="intro">
|
||||
Incoming <code>POST /api/telemetry</code> requests with this exact server name.
|
||||
The scoreboard uses telemetry; the fleet player count comes from RCON separately.
|
||||
</p>
|
||||
<p class="retention">
|
||||
Newest first · up to {retention.maxPerServer} requests from the last {retention.maxAgeDays} days.
|
||||
Refreshes every 5 seconds while visible.
|
||||
{#if refreshedAt}<span>Last refreshed <time datetime={refreshedAt}>{timestamp(refreshedAt)}</time>.</span>{/if}
|
||||
</p>
|
||||
|
||||
{#if error}
|
||||
<p class="load-error" role="alert">
|
||||
{error}{#if loaded} Previously loaded requests are shown below.{/if}
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
{#if !loaded && !error}
|
||||
<EmptyState title="Loading telemetry requests…" />
|
||||
{:else if loaded && requests.length === 0}
|
||||
<EmptyState
|
||||
title="No telemetry requests recorded"
|
||||
description="No matching requests are retained for this server. An empty log alone does not establish a network problem."
|
||||
/>
|
||||
{:else if requests.length > 0}
|
||||
<ol class="request-list" aria-label="Recent telemetry requests">
|
||||
{#each requests.slice(0, limit) as request (request.id)}
|
||||
<li class="request">
|
||||
<div class="request-head">
|
||||
<div class="request-result">
|
||||
<Badge variant={request.status >= 500 ? 'danger' : request.status >= 400 ? 'warning' : 'online'}>
|
||||
{outcome(request.status)} · HTTP {request.status}
|
||||
</Badge>
|
||||
<code class="event">{request.event ?? 'Event not provided'}</code>
|
||||
</div>
|
||||
<time class="received" datetime={request.receivedAt}>{timestamp(request.receivedAt)}</time>
|
||||
</div>
|
||||
<dl class="request-meta">
|
||||
<div><dt>Map</dt><dd><code>{request.map ?? 'Not provided'}</code></dd></div>
|
||||
<div><dt>Round</dt><dd>{request.round ?? 'Not provided'}</dd></div>
|
||||
<div><dt>Players</dt><dd>{request.playerCount ?? 'Not provided'}</dd></div>
|
||||
</dl>
|
||||
{#if request.error}<p class="request-error">{request.error}</p>{/if}
|
||||
<details class="payload">
|
||||
<summary>Request payload{#if request.payloadTruncated} (truncated){/if}</summary>
|
||||
<p class="content-type">Content type: <code>{request.contentType ?? 'Not provided'}</code></p>
|
||||
<pre>{request.payload}</pre>
|
||||
{#if request.payloadTruncated}<p class="truncation">Only the beginning of this payload was retained.</p>{/if}
|
||||
</details>
|
||||
</li>
|
||||
{/each}
|
||||
</ol>
|
||||
<ListFooter total={requests.length} bind:limit label="requests" />
|
||||
{/if}
|
||||
|
||||
<p class="attribution">
|
||||
Logging starts after this update is installed.
|
||||
Requests can be linked here only when readable JSON contains a matching top-level <code>server</code> field.
|
||||
Calls that never reach Electric Cherry, malformed JSON, and different server names will not appear here.
|
||||
</p>
|
||||
</Card>
|
||||
|
||||
<style>
|
||||
.panel-head, .request-head, .request-result {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-3);
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.panel-head { margin-bottom: var(--space-4); }
|
||||
h2 { margin: 0; font-size: var(--text-md); font-weight: var(--weight-semibold); }
|
||||
p { margin: 0; }
|
||||
code, pre { font-family: var(--font-mono); }
|
||||
.server-name { margin-top: var(--space-2); overflow-wrap: anywhere; color: var(--text-secondary); font-size: var(--text-xs); }
|
||||
.intro { font-size: var(--text-sm); color: var(--text-secondary); line-height: 1.6; }
|
||||
.retention, .attribution, .content-type, .truncation {
|
||||
font-size: var(--text-xs);
|
||||
color: var(--text-tertiary);
|
||||
line-height: 1.6;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.retention { margin-top: var(--space-2); }
|
||||
.retention span { display: block; }
|
||||
.attribution { margin-top: var(--space-4); }
|
||||
.request-list { list-style: none; padding: 0; margin: var(--space-5) 0 0; }
|
||||
.request { padding: var(--space-4) 0; border-top: 1px solid var(--border-subtle); }
|
||||
.request-result { justify-content: flex-start; min-width: 0; }
|
||||
.event { font-size: var(--text-sm); overflow-wrap: anywhere; }
|
||||
.received { color: var(--text-tertiary); font-size: var(--text-xs); font-variant-numeric: tabular-nums; }
|
||||
.request-meta { display: flex; flex-wrap: wrap; gap: var(--space-3) var(--space-6); margin: var(--space-3) 0; font-size: var(--text-xs); }
|
||||
.request-meta div { display: flex; gap: var(--space-2); min-width: 0; }
|
||||
dt { color: var(--text-tertiary); }
|
||||
dd { margin: 0; color: var(--text-secondary); overflow-wrap: anywhere; }
|
||||
.load-error, .request-error { color: var(--state-danger); font-size: var(--text-sm); overflow-wrap: anywhere; }
|
||||
.load-error { margin-top: var(--space-4); }
|
||||
.request-error { margin-bottom: var(--space-3); }
|
||||
.payload { font-size: var(--text-xs); }
|
||||
summary { color: var(--text-secondary); cursor: pointer; width: fit-content; }
|
||||
summary:focus-visible { outline: none; box-shadow: var(--focus-ring); border-radius: var(--radius-sm); }
|
||||
.content-type { margin-top: var(--space-3); }
|
||||
pre {
|
||||
margin: var(--space-2) 0 0;
|
||||
padding: var(--space-3);
|
||||
max-height: 24rem;
|
||||
overflow: auto;
|
||||
white-space: pre-wrap;
|
||||
overflow-wrap: anywhere;
|
||||
background: var(--surface-sunken);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-sm);
|
||||
color: var(--text-secondary);
|
||||
line-height: 1.6;
|
||||
}
|
||||
.truncation { margin-top: var(--space-2); }
|
||||
</style>
|
||||
@@ -0,0 +1,11 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { getServer } from '$lib/server/fleet/servers.js';
|
||||
import { listTelemetryRequests } from '$lib/server/fleet/telemetry-requests.js';
|
||||
import { TELEMETRY_REQUEST_RETENTION } from '$lib/telemetry-diagnostics.js';
|
||||
|
||||
export async function GET({ params }: { params: { name: string } }) {
|
||||
if (!getServer(params.name)) return json({ ok: false, error: 'no such server' }, { status: 404 });
|
||||
return json({ ok: true, requests: listTelemetryRequests(params.name), retention: TELEMETRY_REQUEST_RETENTION }, {
|
||||
headers: { 'Cache-Control': 'no-store' }
|
||||
});
|
||||
}
|
||||
@@ -2,26 +2,51 @@ import { json } from '@sveltejs/kit';
|
||||
import { parseGameEvent, type GameTelemetry } from '$lib/telemetry.js';
|
||||
import { getServer } from '$lib/server/fleet/servers.js';
|
||||
import { saveTelemetry } from '$lib/server/fleet/telemetry.js';
|
||||
import { recordTelemetryRequest } from '$lib/server/fleet/telemetry-requests.js';
|
||||
import { recordGameEvent } from '$lib/server/fleet/beta-tests.js';
|
||||
import { db } from '$lib/server/db/index.js';
|
||||
import { emit } from '$lib/server/pipeline/bus.js';
|
||||
|
||||
export async function POST({ request }: { request: Request }) {
|
||||
if (request.headers.get('content-type')?.split(';')[0].trim().toLowerCase() !== 'application/json') {
|
||||
return json({ ok: false, error: 'Content-Type must be application/json' }, { status: 415 });
|
||||
}
|
||||
const receivedAt = new Date().toISOString();
|
||||
const contentType = request.headers.get('content-type');
|
||||
let value: unknown;
|
||||
let malformed = false;
|
||||
try { value = await request.json(); }
|
||||
catch { return json({ ok: false, error: 'Malformed JSON' }, { status: 400 }); }
|
||||
catch { malformed = true; }
|
||||
|
||||
function log(status: number, error: string | null): void {
|
||||
try { recordTelemetryRequest({ value, receivedAt, status, error, contentType }); }
|
||||
catch (error) {
|
||||
// Diagnostics must not turn a saved report into a failed request.
|
||||
console.error('Could not record telemetry request diagnostics', error);
|
||||
}
|
||||
}
|
||||
function reject(status: number, error: string): Response {
|
||||
log(status, error);
|
||||
return json({ ok: false, error }, { status });
|
||||
}
|
||||
// Even a wrong content type may carry readable JSON with a server name.
|
||||
if (contentType?.split(';')[0].trim().toLowerCase() !== 'application/json') {
|
||||
return reject(415, 'Content-Type must be application/json');
|
||||
}
|
||||
if (malformed) return reject(400, 'Malformed JSON');
|
||||
let telemetry: GameTelemetry;
|
||||
try { telemetry = { ...parseGameEvent(value), receivedAt: new Date().toISOString() }; }
|
||||
catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); }
|
||||
const server = getServer(telemetry.server);
|
||||
if (!server || server.destroyed_at) return json({ ok: false, error: 'Server is not registered or has been destroyed' }, { status: 404 });
|
||||
db().transaction(() => {
|
||||
saveTelemetry(telemetry);
|
||||
recordGameEvent(telemetry);
|
||||
})();
|
||||
emit({ type: 'server-telemetry', entries: [telemetry] });
|
||||
try { telemetry = { ...parseGameEvent(value), receivedAt }; }
|
||||
catch (error) { return reject(400, (error as Error).message); }
|
||||
try {
|
||||
const server = getServer(telemetry.server);
|
||||
if (!server || server.destroyed_at) return reject(404, 'Server is not registered or has been destroyed');
|
||||
db().transaction(() => {
|
||||
saveTelemetry(telemetry);
|
||||
recordGameEvent(telemetry);
|
||||
})();
|
||||
emit({ type: 'server-telemetry', entries: [telemetry] });
|
||||
} catch (error) {
|
||||
console.error('Could not process telemetry', error);
|
||||
log(500, `Could not process telemetry: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return json({ ok: false, error: 'Could not process telemetry' }, { status: 500 });
|
||||
}
|
||||
log(200, null);
|
||||
return json({ ok: true, telemetry });
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<script lang="ts">
|
||||
import RoundTelemetry from '$ui/components/RoundTelemetry.svelte';
|
||||
import TelemetryDiagnostics from '$ui/components/TelemetryDiagnostics.svelte';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } from '$ui/stores/fleet.js';
|
||||
import { freshServer, fleetIdentity } from '$lib/fleet-overview.js';
|
||||
@@ -284,7 +285,7 @@
|
||||
<Tabs {tabs} bind:value={activeTab} ariaLabel="Server sections" idPrefix="server-tabs" panelId="server-panel" />
|
||||
</div>
|
||||
|
||||
<div class="tab-panel" role="tabpanel" id="server-panel" aria-labelledby={"server-tabs-" + activeTab}>
|
||||
<div class="tab-panel" class:diagnostics={activeTab === 'diagnostics'} role="tabpanel" id="server-panel" aria-labelledby={"server-tabs-" + activeTab}>
|
||||
{#if tabError}<div class="ui-error" role="alert">{tabError}<Button size="sm" onclick={() => loadTab(name, activeTab)}>Retry</Button></div>{/if}
|
||||
{#if activeTab === 'scoreboard'}
|
||||
<Card padding="lg">
|
||||
@@ -328,6 +329,9 @@
|
||||
</Card>
|
||||
<ListFooter total={inventory.length} bind:limit={inventoryLimit} label="files" />
|
||||
{:else if activeTab === 'diagnostics'}
|
||||
{#key name}
|
||||
<TelemetryDiagnostics server={name} />
|
||||
{/key}
|
||||
<Card padding="lg">
|
||||
<div class="panel-head">
|
||||
<h2>RCON diagnostic</h2>
|
||||
@@ -575,6 +579,10 @@
|
||||
}
|
||||
|
||||
/* Tabs row */
|
||||
.tab-panel.diagnostics {
|
||||
display: grid;
|
||||
gap: var(--space-6);
|
||||
}
|
||||
.tabs-row {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
|
||||
Reference in New Issue
Block a user