diff --git a/README.md b/README.md index 156b739..2730117 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,14 @@ See [Game reporter protocol](docs/beta-test-plans.md#game-reporter-protocol) for complete payload examples. Telemetry does not replace RCON's empty-server check before an automated restart. +For a missing scoreboard, open the server's **Diagnostics → Telemetry API requests**. +The log shows received requests for that exact server name, including accepted +reports, rejected payloads, HTTP status, errors, and expandable submitted JSON. +It refreshes every five seconds while visible and survives app restarts. Logging +starts with this version; older requests cannot be reconstructed. See +[Telemetry diagnostics](docs/beta-test-plans.md#telemetry-diagnostics) for retention +and how to interpret missing or rejected reports. + The **Leaderboard** records persistent player totals and map records from these same reports, including highest round, cumulative kills, rounds survived, headshots, revives, downs, and games played. See [Leaderboard](docs/leaderboard.md) diff --git a/docs/beta-test-plans.md b/docs/beta-test-plans.md index 64e112c..1e07377 100644 --- a/docs/beta-test-plans.md +++ b/docs/beta-test-plans.md @@ -94,6 +94,44 @@ and the limits of identifying matches from this protocol. The game scripts are uploaded artifacts, not source maintained in this repository. Their reporter should emit the three payloads above. +### Telemetry diagnostics + +Open a server's **Diagnostics → Telemetry API requests** to inspect incoming +`POST /api/telemetry` calls whose top-level `server` exactly matches its registered +name. Each entry includes the Electric Cherry receipt time, HTTP result, error +reason, event, map, round, player count, content type, and submitted JSON. Rejected +reports are included, even when the content type is wrong but the body is readable +JSON. Processing failures return 500 and record the internal error when diagnostic +storage is available; failed database transactions do not update the scoreboard. + +The panel refreshes every five seconds while visible and has a manual refresh +button. History is stored in SQLite starting with this version, with up to 200 +requests per server from the last seven days and a fleet-wide cap of 10,000 +requests. Old entries are pruned when new requests arrive, and expired entries +are excluded from reads. Payload previews are limited to 16,384 characters and +marked when truncated. Request headers other than content type are not stored. +`GET /api/servers/{name}/telemetry-requests` returns the same recent history, +newest first, with caching disabled. + +- **No entries:** no attributable requests are retained for this exact name. + Check the reporter's destination and server field. Calls that never reach + Electric Cherry, malformed JSON, and bodies without a usable server name + cannot appear in this server's log. Earlier requests are not backfilled. +- **400 / 415:** inspect the error and payload. One invalid player's stats or a + duplicate GUID rejects the entire report. These requests do not update the + scoreboard. +- **404:** the reported server was not registered or had been destroyed at the + time of the request. +- **200:** Electric Cherry processed the report. An empty `players` array + produces an empty scoreboard, including when sent with `game_end`. If a + populated accepted report is missing from the scoreboard, investigate the + live update/browser path. +- **500:** Electric Cherry failed to process the request. Inspect the recorded + error and the application log. + +RCON's online status and player count are separate from gameplay telemetry, so +an occupied server can still have no scoreboard report. + ## Requirements Requirements on a qualifying server are ANDed: diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index c0bbe1a..283ce82 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -367,6 +367,24 @@ CREATE TABLE IF NOT EXISTS server_telemetry ( received_at TEXT NOT NULL, payload TEXT ); +-- Request diagnostics survive registry removal and also capture unknown names. +-- Bounded independently of the latest scoreboard and permanent leaderboard. +CREATE TABLE IF NOT EXISTS telemetry_requests ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + server TEXT NOT NULL, + received_at TEXT NOT NULL, + status INTEGER NOT NULL, + error TEXT, + event TEXT, + map TEXT, + round INTEGER, + player_count INTEGER, + content_type TEXT, + payload TEXT NOT NULL, + payload_truncated INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS idx_telemetry_requests_server ON telemetry_requests(server, id DESC); +CREATE INDEX IF NOT EXISTS idx_telemetry_requests_received ON telemetry_requests(received_at); -- Lifecycle history deliberately survives removal of the server registry row. CREATE TABLE IF NOT EXISTS server_deployments ( id TEXT PRIMARY KEY, diff --git a/src/lib/server/fleet/telemetry-requests.ts b/src/lib/server/fleet/telemetry-requests.ts new file mode 100644 index 0000000..ebf5166 --- /dev/null +++ b/src/lib/server/fleet/telemetry-requests.ts @@ -0,0 +1,50 @@ +import { db } from '../db/index.js'; +import { TELEMETRY_REQUEST_RETENTION, type TelemetryRequestLogEntry } from '../../telemetry-diagnostics.js'; + +const MAX_PAYLOAD_LENGTH = 16 * 1024; +// Bound storage even when requests claim many different, unregistered names. +const MAX_REQUESTS = 10_000; +const cutoff = () => new Date(Date.now() - TELEMETRY_REQUEST_RETENTION.maxAgeDays * 86_400_000).toISOString(); +const textField = (value: unknown) => typeof value === 'string' ? value.slice(0, 255) : null; + +export function recordTelemetryRequest(input: { + value: unknown; + receivedAt: string; + status: number; + error: string | null; + contentType: string | null; +}): void { + // Use only the top-level JSON field, never a player's name or a substring. + // Malformed JSON cannot be reliably attributed to an individual server. + if (!input.value || typeof input.value !== 'object' || Array.isArray(input.value)) return; + const body = input.value as Record; + if (typeof body.server !== 'string' || !body.server.trim() || body.server.length > 255) return; + const payload = JSON.stringify(body); + const handle = db(); + handle.transaction(() => { + handle.prepare(`INSERT INTO telemetry_requests + (server, received_at, status, error, event, map, round, player_count, content_type, payload, payload_truncated) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run( + body.server, input.receivedAt, input.status, input.error?.slice(0, 2000) ?? null, + textField(body.event), textField(body.map), + Number.isSafeInteger(body.round) ? body.round as number : null, + Array.isArray(body.players) ? body.players.length : null, + textField(input.contentType), payload.slice(0, MAX_PAYLOAD_LENGTH), Number(payload.length > MAX_PAYLOAD_LENGTH) + ); + handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff()); + handle.prepare(`DELETE FROM telemetry_requests WHERE server = ? AND id <= ( + SELECT id FROM telemetry_requests WHERE server = ? ORDER BY id DESC LIMIT 1 OFFSET ? + )`).run(body.server, body.server, TELEMETRY_REQUEST_RETENTION.maxPerServer); + handle.prepare(`DELETE FROM telemetry_requests WHERE id <= ( + SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 1 OFFSET ? + )`).run(MAX_REQUESTS); + })(); +} + +export function listTelemetryRequests(server: string): TelemetryRequestLogEntry[] { + const rows = db().prepare(`SELECT id, server, received_at AS receivedAt, status, error, event, map, round, + player_count AS playerCount, content_type AS contentType, payload, payload_truncated AS payloadTruncated + FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`) + .all(server, cutoff(), TELEMETRY_REQUEST_RETENTION.maxPerServer) as Array & { payloadTruncated: number }>; + return rows.map(row => ({ ...row, payloadTruncated: !!row.payloadTruncated })); +} diff --git a/src/lib/telemetry-diagnostics.ts b/src/lib/telemetry-diagnostics.ts new file mode 100644 index 0000000..9f9800a --- /dev/null +++ b/src/lib/telemetry-diagnostics.ts @@ -0,0 +1,17 @@ +/** A received POST /api/telemetry request, including reports rejected by validation. */ +export interface TelemetryRequestLogEntry { + id: number; + server: string; + receivedAt: string; + status: number; + error: string | null; + event: string | null; + map: string | null; + round: number | null; + playerCount: number | null; + contentType: string | null; + payload: string; + payloadTruncated: boolean; +} + +export const TELEMETRY_REQUEST_RETENTION = { maxPerServer: 200, maxAgeDays: 7 } as const; diff --git a/src/lib/ui/components/TelemetryDiagnostics.svelte b/src/lib/ui/components/TelemetryDiagnostics.svelte new file mode 100644 index 0000000..1adab71 --- /dev/null +++ b/src/lib/ui/components/TelemetryDiagnostics.svelte @@ -0,0 +1,198 @@ + + + +
+
+

Telemetry API requests

+

{server}

+
+ +
+ +

+ Incoming POST /api/telemetry requests with this exact server name. + The scoreboard uses telemetry; the fleet player count comes from RCON separately. +

+

+ Newest first · up to {retention.maxPerServer} requests from the last {retention.maxAgeDays} days. + Refreshes every 5 seconds while visible. + {#if refreshedAt}Last refreshed .{/if} +

+ + {#if error} + + {/if} + + {#if !loaded && !error} + + {:else if loaded && requests.length === 0} + + {:else if requests.length > 0} +
    + {#each requests.slice(0, limit) as request (request.id)} +
  1. +
    +
    + = 500 ? 'danger' : request.status >= 400 ? 'warning' : 'online'}> + {outcome(request.status)} · HTTP {request.status} + + {request.event ?? 'Event not provided'} +
    + +
    +
    +
    Map
    {request.map ?? 'Not provided'}
    +
    Round
    {request.round ?? 'Not provided'}
    +
    Players
    {request.playerCount ?? 'Not provided'}
    +
    + {#if request.error}

    {request.error}

    {/if} +
    + Request payload{#if request.payloadTruncated} (truncated){/if} +

    Content type: {request.contentType ?? 'Not provided'}

    + 
    {request.payload}
    + {#if request.payloadTruncated}

    Only the beginning of this payload was retained.

    {/if} +
    +
    2. + {/each} +
+ + {/if} + +

+ Logging starts after this update is installed. + Requests can be linked here only when readable JSON contains a matching top-level server field. + Calls that never reach Electric Cherry, malformed JSON, and different server names will not appear here. +

+ + + diff --git a/src/routes/api/servers/[name]/telemetry-requests/+server.ts b/src/routes/api/servers/[name]/telemetry-requests/+server.ts new file mode 100644 index 0000000..d50d708 --- /dev/null +++ b/src/routes/api/servers/[name]/telemetry-requests/+server.ts @@ -0,0 +1,11 @@ +import { json } from '@sveltejs/kit'; +import { getServer } from '$lib/server/fleet/servers.js'; +import { listTelemetryRequests } from '$lib/server/fleet/telemetry-requests.js'; +import { TELEMETRY_REQUEST_RETENTION } from '$lib/telemetry-diagnostics.js'; + +export async function GET({ params }: { params: { name: string } }) { + if (!getServer(params.name)) return json({ ok: false, error: 'no such server' }, { status: 404 }); + return json({ ok: true, requests: listTelemetryRequests(params.name), retention: TELEMETRY_REQUEST_RETENTION }, { + headers: { 'Cache-Control': 'no-store' } + }); +} diff --git a/src/routes/api/telemetry/+server.ts b/src/routes/api/telemetry/+server.ts index ca7a48b..d91dd43 100644 --- a/src/routes/api/telemetry/+server.ts +++ b/src/routes/api/telemetry/+server.ts @@ -2,26 +2,51 @@ import { json } from '@sveltejs/kit'; import { parseGameEvent, type GameTelemetry } from '$lib/telemetry.js'; import { getServer } from '$lib/server/fleet/servers.js'; import { saveTelemetry } from '$lib/server/fleet/telemetry.js'; +import { recordTelemetryRequest } from '$lib/server/fleet/telemetry-requests.js'; import { recordGameEvent } from '$lib/server/fleet/beta-tests.js'; import { db } from '$lib/server/db/index.js'; import { emit } from '$lib/server/pipeline/bus.js'; export async function POST({ request }: { request: Request }) { - if (request.headers.get('content-type')?.split(';')[0].trim().toLowerCase() !== 'application/json') { - return json({ ok: false, error: 'Content-Type must be application/json' }, { status: 415 }); - } + const receivedAt = new Date().toISOString(); + const contentType = request.headers.get('content-type'); let value: unknown; + let malformed = false; try { value = await request.json(); } - catch { return json({ ok: false, error: 'Malformed JSON' }, { status: 400 }); } + catch { malformed = true; } + + function log(status: number, error: string | null): void { + try { recordTelemetryRequest({ value, receivedAt, status, error, contentType }); } + catch (error) { + // Diagnostics must not turn a saved report into a failed request. + console.error('Could not record telemetry request diagnostics', error); + } + } + function reject(status: number, error: string): Response { + log(status, error); + return json({ ok: false, error }, { status }); + } + // Even a wrong content type may carry readable JSON with a server name. + if (contentType?.split(';')[0].trim().toLowerCase() !== 'application/json') { + return reject(415, 'Content-Type must be application/json'); + } + if (malformed) return reject(400, 'Malformed JSON'); let telemetry: GameTelemetry; - try { telemetry = { ...parseGameEvent(value), receivedAt: new Date().toISOString() }; } - catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); } - const server = getServer(telemetry.server); - if (!server || server.destroyed_at) return json({ ok: false, error: 'Server is not registered or has been destroyed' }, { status: 404 }); - db().transaction(() => { - saveTelemetry(telemetry); - recordGameEvent(telemetry); - })(); - emit({ type: 'server-telemetry', entries: [telemetry] }); + try { telemetry = { ...parseGameEvent(value), receivedAt }; } + catch (error) { return reject(400, (error as Error).message); } + try { + const server = getServer(telemetry.server); + if (!server || server.destroyed_at) return reject(404, 'Server is not registered or has been destroyed'); + db().transaction(() => { + saveTelemetry(telemetry); + recordGameEvent(telemetry); + })(); + emit({ type: 'server-telemetry', entries: [telemetry] }); + } catch (error) { + console.error('Could not process telemetry', error); + log(500, `Could not process telemetry: ${error instanceof Error ? error.message : String(error)}`); + return json({ ok: false, error: 'Could not process telemetry' }, { status: 500 }); + } + log(200, null); return json({ ok: true, telemetry }); } diff --git a/src/routes/servers/[name]/+page.svelte b/src/routes/servers/[name]/+page.svelte index aa2f4d4..9e7e5ca 100644 --- a/src/routes/servers/[name]/+page.svelte +++ b/src/routes/servers/[name]/+page.svelte @@ -1,5 +1,6 @@