Add persistent player leaderboards from game telemetry

2026-09-18 16:43:44 -04:00
parent 0f55fa5f77
commit a182aeb950
16 changed files with 1135 additions and 17 deletions
@@ -5,6 +5,7 @@ import { config } from '../config.js';
 import schemaSql from './schema.sql?raw';
 import { requirement, type BetaPlan, type BetaRequirement } from '../../beta-plan.js';
 import { emptyMetrics } from '../fleet/beta-evaluation.js';
+import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js';

 let _db: Database.Database | null = null;

@@ -259,6 +260,7 @@ export function db(): Database.Database {
        backfillMultiScriptJoins(handle);
        assertSnapshotSchemeSafe(handle);
        migrateGameTelemetry(handle);
+        backfillLeaderboard(handle);
    } catch (err) {
        handle.close();
        throw err;
@@ -6,6 +6,56 @@
 PRAGMA journal_mode = WAL;
 PRAGMA foreign_keys = ON;

+-- Historical gameplay deliberately has no FK to servers: destroying a server
+-- must not erase records. Each player's counters are high-water marks per match.
+CREATE TABLE IF NOT EXISTS leaderboard_matches (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    server TEXT NOT NULL,
+    game TEXT NOT NULL,
+    map TEXT NOT NULL,
+    map_key TEXT NOT NULL,
+    map_name TEXT NOT NULL,
+    started_at TEXT NOT NULL,
+    ended_at TEXT,
+    last_received_at TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT 'active', -- active | completed | interrupted
+    start_observed INTEGER NOT NULL DEFAULT 0,
+    highest_round INTEGER NOT NULL DEFAULT 0,
+    last_event TEXT NOT NULL,
+    last_fingerprint TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_leaderboard_server ON leaderboard_matches(server, id DESC);
+CREATE INDEX IF NOT EXISTS idx_leaderboard_map ON leaderboard_matches(game, map_key);
+
+CREATE TABLE IF NOT EXISTS leaderboard_players (
+    game TEXT NOT NULL,
+    guid TEXT NOT NULL,
+    name TEXT NOT NULL,
+    first_seen_at TEXT NOT NULL,
+    last_seen_at TEXT NOT NULL,
+    PRIMARY KEY (game, guid)
+);
+
+CREATE TABLE IF NOT EXISTS leaderboard_match_players (
+    match_id INTEGER NOT NULL REFERENCES leaderboard_matches(id) ON DELETE CASCADE,
+    guid TEXT NOT NULL,
+    highest_round INTEGER NOT NULL DEFAULT 0,
+    rounds_survived INTEGER NOT NULL DEFAULT 0,
+    kills INTEGER NOT NULL DEFAULT 0,
+    headshots INTEGER NOT NULL DEFAULT 0,
+    downs INTEGER NOT NULL DEFAULT 0,
+    revives INTEGER NOT NULL DEFAULT 0,
+    best_score INTEGER NOT NULL DEFAULT 0,
+    last_round INTEGER NOT NULL DEFAULT 0,
+    present INTEGER NOT NULL DEFAULT 1,
+    has_stats INTEGER NOT NULL DEFAULT 0,
+    completed INTEGER NOT NULL DEFAULT 0,
+    first_seen_at TEXT NOT NULL,
+    last_seen_at TEXT NOT NULL,
+    PRIMARY KEY (match_id, guid)
+);
+CREATE INDEX IF NOT EXISTS idx_leaderboard_player_matches ON leaderboard_match_players(guid, match_id);
+
 CREATE TABLE IF NOT EXISTS servers (
    name             TEXT PRIMARY KEY,
    komodo_type      TEXT NOT NULL DEFAULT 'deployment',  -- deployment|stack|container
@@ -0,0 +1,137 @@
+import { createHash } from 'node:crypto';
+import type Database from 'better-sqlite3';
+import { parseServerName, selectedZombieMap, zombieMaps } from '../../map-aliases.js';
+import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
+
+interface Match {
+    id: number;
+    map_key: string;
+    game: string;
+    status: 'active' | 'completed' | 'interrupted';
+    highest_round: number;
+    last_event: GameTelemetry['event'];
+    last_fingerprint: string;
+}
+
+interface Participation {
+    guid: string;
+    last_round: number;
+    present: number;
+}
+
+function mapIdentity(handle: Database.Database, report: GameTelemetry) {
+    const server = handle.prepare('SELECT game, map, map_slug FROM servers WHERE name = ?').get(report.server) as
+        { game: string | null; map: string | null; map_slug: string | null } | undefined;
+    const parsed = parseServerName(report.server);
+    const game = (server?.game ?? parsed?.game ?? 'unknown').toLowerCase();
+    const map = report.map.toLowerCase();
+    // A shared engine map (notably zm_transit) is not enough to identify T6's
+    // location/ruleset. Use the registered selection only when it matches.
+    const parsedSlug = parsed?.mapCanonical === map ? parsed.mapSlug : undefined;
+    const slug = server?.map === map ? server.map_slug ?? parsedSlug : parsedSlug;
+    const candidates = zombieMaps(game).filter(entry => entry.canonical === map);
+    const selected = game === 't6' && candidates.length > 1 && !slug
+        ? undefined : selectedZombieMap(game, map, slug);
+    const variant = game === 't6' ? selected?.slug ?? slug ?? 'unspecified' : '';
+    return {
+        game, map,
+        key: [game, map, ...(variant ? [variant] : [])].join(':'),
+        name: selected?.friendly ?? (game === 't6' && candidates.length > 1 ? `${map} (${slug ?? 'unspecified variant'})` : map)
+    };
+}
+
+function fingerprint(report: GameTelemetry): string {
+    // Receipt timestamps and roster ordering do not identify new gameplay.
+    const { receivedAt: _receivedAt, ...event } = report;
+    const players = [...event.players].sort((a, b) => a.guid.localeCompare(b.guid));
+    return createHash('sha256').update(JSON.stringify({ ...event, players })).digest('hex');
+}
+
+/** Called inside the same transaction as the latest scoreboard snapshot. */
+export function recordLeaderboardTelemetry(handle: Database.Database, report: GameTelemetry): void {
+    const identity = mapIdentity(handle, report);
+    const signature = fingerprint(report);
+    let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1')
+        .get(report.server) as Match | undefined;
+    if (match?.last_fingerprint === signature && match.map_key === identity.key) return;
+
+    const isStart = report.event === 'game_start';
+    const round = isStart ? 0 : report.round;
+    const newMatch = !match || match.map_key !== identity.key
+        || (isStart && (match.status !== 'active' || match.last_event !== 'game_start'))
+        || (!isStart && round < match.highest_round)
+        || (report.event === 'round_change' && match.status !== 'active');
+
+    if (newMatch) {
+        if (match?.status === 'active') {
+            handle.prepare("UPDATE leaderboard_matches SET status = 'interrupted', ended_at = ? WHERE id = ?")
+                .run(report.receivedAt, match.id);
+        }
+        const inserted = handle.prepare(`INSERT INTO leaderboard_matches
+            (server, game, map, map_key, map_name, started_at, last_received_at, start_observed, last_event, last_fingerprint)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+            .run(report.server, identity.game, identity.map, identity.key, identity.name,
+                report.receivedAt, report.receivedAt, Number(isStart), report.event, signature);
+        match = { id: Number(inserted.lastInsertRowid), game: identity.game, map_key: identity.key,
+            status: 'active', highest_round: 0, last_event: report.event, last_fingerprint: signature };
+    }
+    if (!match) return;
+
+    const previous = new Map((handle.prepare('SELECT guid, last_round, present FROM leaderboard_match_players WHERE match_id = ?')
+        .all(match.id) as Participation[]).map(player => [player.guid, player]));
+    handle.prepare('UPDATE leaderboard_match_players SET present = 0 WHERE match_id = ?').run(match.id);
+    const saveIdentity = handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
+        VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET
+        name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END,
+        last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`);
+    const savePlayer = handle.prepare(`INSERT INTO leaderboard_match_players
+        (match_id, guid, highest_round, rounds_survived, kills, headshots, downs, revives, best_score,
+            last_round, present, has_stats, completed, first_seen_at, last_seen_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?)
+        ON CONFLICT(match_id, guid) DO UPDATE SET
+            highest_round = MAX(highest_round, excluded.highest_round),
+            rounds_survived = rounds_survived + excluded.rounds_survived,
+            kills = MAX(kills, excluded.kills), headshots = MAX(headshots, excluded.headshots),
+            downs = MAX(downs, excluded.downs), revives = MAX(revives, excluded.revives),
+            best_score = MAX(best_score, excluded.best_score), last_round = excluded.last_round,
+            present = 1, has_stats = MAX(has_stats, excluded.has_stats),
+            completed = MAX(completed, excluded.completed), last_seen_at = excluded.last_seen_at`);
+
+    // Highest round includes late joins; total rounds only credits progression
+    // while present at both ends of an observed interval. Entering round 8
+    // completes round 7, but a player's first sighting at round 8 starts at zero.
+    const completedRounds = Math.max(0, round - 1);
+    for (const player of report.players) {
+        const prior = previous.get(player.guid);
+        const survived = !isStart && prior?.present ? Math.max(0, completedRounds - prior.last_round) : 0;
+        saveIdentity.run(identity.game, player.guid, player.name, report.receivedAt, report.receivedAt);
+        const stats = 'kills' in player ? player : null;
+        savePlayer.run(match.id, player.guid, round, survived,
+            stats?.kills ?? 0, stats?.headshots ?? 0, stats?.downs ?? 0, stats?.revives ?? 0, stats?.score ?? 0,
+            completedRounds, Number(!isStart), Number(report.event === 'game_end'), report.receivedAt, report.receivedAt);
+    }
+    handle.prepare(`UPDATE leaderboard_matches SET highest_round = MAX(highest_round, ?),
+        last_received_at = ?, last_event = ?, last_fingerprint = ?,
+        status = CASE WHEN ? = 'game_end' THEN 'completed' ELSE status END,
+        ended_at = CASE WHEN ? = 'game_end' THEN ? ELSE ended_at END WHERE id = ?`)
+        .run(round, report.receivedAt, report.event, signature, report.event, report.event, report.receivedAt, match.id);
+}
+
+/** Seed the single surviving scoreboard per server once; older games are gone. */
+export function backfillLeaderboard(handle: Database.Database): void {
+    if (handle.prepare("SELECT 1 FROM settings WHERE key = 'leaderboard_v1'").get()) return;
+    handle.transaction(() => {
+        const reports = handle.prepare(`SELECT payload FROM server_telemetry WHERE payload IS NOT NULL ORDER BY received_at, server`)
+            .all() as Array<{ payload: string }>;
+        for (const row of reports) {
+            let report: GameTelemetry;
+            try {
+                const stored = JSON.parse(row.payload);
+                if (typeof stored.receivedAt !== 'string' || !Number.isFinite(Date.parse(stored.receivedAt))) continue;
+                report = { ...parseGameEvent(stored), receivedAt: stored.receivedAt };
+            } catch { continue; } // Legacy malformed snapshots cannot establish player records.
+            recordLeaderboardTelemetry(handle, report);
+        }
+        handle.prepare("INSERT INTO settings (key, value) VALUES ('leaderboard_v1', '1')").run();
+    })();
+}
@@ -0,0 +1,58 @@
+import { db } from '../db/index.js';
+import { leaderboardMetrics, type LeaderboardEntry, type LeaderboardMap, type LeaderboardMetric,
+    type LeaderboardOptions, type LeaderboardResponse } from '../../leaderboard.js';
+
+const metrics = new Set<string>(leaderboardMetrics.map(metric => metric.key));
+export const MAX_LEADERBOARD_OFFSET = 1_000_000;
+
+export function isLeaderboardMetric(value: string): value is LeaderboardMetric {
+    return metrics.has(value);
+}
+
+/** Aggregate persisted per-match records; filtering a player keeps their actual rank. */
+export function getLeaderboard(options: LeaderboardOptions = {}): LeaderboardResponse {
+    const handle = db();
+    const metric = options.metric && isLeaderboardMetric(options.metric) ? options.metric : 'highestRound';
+    const limit = Number.isSafeInteger(options.limit) ? Math.min(100, Math.max(1, options.limit!)) : 50;
+    const offset = Number.isSafeInteger(options.offset) ? Math.min(MAX_LEADERBOARD_OFFSET, Math.max(0, options.offset!)) : 0;
+    const conditions = ['mp.has_stats = 1'];
+    const parameters: string[] = [];
+    if (options.game) { conditions.push('m.game = ?'); parameters.push(options.game); }
+    if (options.map) { conditions.push('m.map_key = ?'); parameters.push(options.map); }
+    const scoped = `FROM leaderboard_match_players mp
+        JOIN leaderboard_matches m ON m.id = mp.match_id
+        WHERE ${conditions.join(' AND ')}`;
+    const aggregated = `WITH totals AS (
+        SELECT m.game, mp.guid, MAX(mp.highest_round) AS highestRound,
+            SUM(mp.kills) AS kills, SUM(mp.rounds_survived) AS roundsSurvived,
+            SUM(mp.headshots) AS headshots, SUM(mp.revives) AS revives, SUM(mp.downs) AS downs,
+            COUNT(*) AS gamesPlayed, SUM(mp.completed) AS gamesCompleted,
+            MAX(mp.kills) AS bestKills, MAX(mp.best_score) AS bestScore, MAX(mp.last_seen_at) AS lastSeen
+        ${scoped} GROUP BY m.game, mp.guid
+    ), ranked AS (
+        SELECT totals.*, p.name, RANK() OVER (ORDER BY totals.${metric} DESC) AS rank
+        FROM totals JOIN leaderboard_players p ON p.game = totals.game AND p.guid = totals.guid
+    )`;
+    const query = options.q?.trim() ?? '';
+    const search = query ? 'WHERE instr(lower(name), lower(?)) > 0 OR instr(lower(guid), lower(?)) > 0' : '';
+    const searchedParameters = query ? [...parameters, query, query] : parameters;
+    const entries = handle.prepare(`${aggregated} SELECT * FROM ranked ${search}
+        ORDER BY ${metric} DESC, name COLLATE NOCASE, name, game, guid LIMIT ? OFFSET ?`)
+        .all(...searchedParameters, limit, offset) as LeaderboardEntry[];
+    const { total } = handle.prepare(`${aggregated} SELECT COUNT(*) AS total FROM ranked ${search}`)
+        .get(...searchedParameters) as { total: number };
+    const summary = handle.prepare(`SELECT
+        (SELECT COUNT(*) FROM (SELECT m.game, mp.guid ${scoped} GROUP BY m.game, mp.guid)) AS players,
+        COUNT(DISTINCT m.id) AS matches, COALESCE(SUM(mp.kills), 0) AS kills,
+        COALESCE(SUM(mp.rounds_survived), 0) AS roundsSurvived,
+        COALESCE(MAX(mp.highest_round), 0) AS highestRound,
+        MIN(m.started_at) AS firstTrackedAt, MAX(m.last_received_at) AS updatedAt
+        ${scoped}`).get(...parameters, ...parameters) as LeaderboardResponse['summary'];
+    // Keep all recorded map variants available, even after their server is removed.
+    const maps = handle.prepare(`SELECT map_key AS key, game, map, map_name AS name
+        FROM leaderboard_matches ${options.game ? 'WHERE game = ?' : ''}
+        GROUP BY game, map_key HAVING id = MAX(id)
+        ORDER BY game, name COLLATE NOCASE, map_key`)
+        .all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
+    return { entries, maps, total, limit, offset, metric, summary };
+}
@@ -1,5 +1,6 @@
 import { db } from '../db/index.js';
 import type { GameTelemetry } from '../../telemetry.js';
+import { recordLeaderboardTelemetry } from './leaderboard-ingest.js';

 export function listTelemetry(): GameTelemetry[] {
    const rows = db().prepare(`SELECT t.payload FROM server_telemetry t JOIN servers s ON s.name = t.server
@@ -8,17 +9,22 @@ export function listTelemetry(): GameTelemetry[] {
 }

 export function saveTelemetry(report: GameTelemetry): void {
-    db().prepare(`INSERT INTO server_telemetry (server, round, players, received_at, payload)
-        VALUES (?, ?, ?, ?, ?) ON CONFLICT(server) DO UPDATE SET
-        round = excluded.round, players = excluded.players, received_at = excluded.received_at, payload = excluded.payload`)
-        .run(report.server, report.event === 'game_start' ? 0 : report.round, report.players.length, report.receivedAt, JSON.stringify(report));
-    if (report.event === 'game_start') {
-        db().prepare('UPDATE servers SET game_started_at = ?, game_ended_at = NULL WHERE name = ?').run(report.receivedAt, report.server);
-    } else if (report.event === 'game_end') {
-        db().prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?').run(report.receivedAt, report.server);
-        // Legacy betas also take match completion directly from the script.
-        db().prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND stage = 'awaiting-match'
-            AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`)
-            .run(report.receivedAt, report.server, report.receivedAt);
-    }
+    // Keep callers outside the HTTP route atomic too.
+    const handle = db();
+    handle.transaction(() => {
+        recordLeaderboardTelemetry(handle, report);
+        handle.prepare(`INSERT INTO server_telemetry (server, round, players, received_at, payload)
+            VALUES (?, ?, ?, ?, ?) ON CONFLICT(server) DO UPDATE SET
+            round = excluded.round, players = excluded.players, received_at = excluded.received_at, payload = excluded.payload`)
+            .run(report.server, report.event === 'game_start' ? 0 : report.round, report.players.length, report.receivedAt, JSON.stringify(report));
+        if (report.event === 'game_start') {
+            handle.prepare('UPDATE servers SET game_started_at = ?, game_ended_at = NULL WHERE name = ?').run(report.receivedAt, report.server);
+        } else if (report.event === 'game_end') {
+            handle.prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?').run(report.receivedAt, report.server);
+            // Legacy betas also take match completion directly from the script.
+            handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND stage = 'awaiting-match'
+                AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`)
+                .run(report.receivedAt, report.server, report.receivedAt);
+        }
+    })();
 }