Improve application responsiveness and background processing

src/lib/server/fleet/leaderboard.ts

@@ -10,6 +10,11 @@ const metrics = new Set<string>(leaderboardMetrics.map(metric => metric.key));
 export const MAX_LEADERBOARD_OFFSET = 1_000_000;
 type RankedEntry = Omit<LeaderboardEntry, 'records'>;
 
+interface RankedScope { revision: number; entries: RankedEntry[]; maps: LeaderboardMap[]; summary: LeaderboardResponse['summary'] }
+const scopeCaches = new WeakMap<object, Map<string, RankedScope>>();
+// Match SQLite's built-in lower(), which folds ASCII rather than Unicode.
+const searchText = (value: string) => value.replace(/[A-Z]/g, ch => ch.toLowerCase());
+
 interface LeaderboardRow extends Omit<RankedEntry, 'guid'> {
     // A left join keeps metadata available when the requested page has no players.
     guid: string | null;
@@ -91,21 +96,15 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
         SELECT measures.*, p.name, RANK() OVER (ORDER BY measures.${metric} DESC) AS rank
         FROM measures JOIN identities p ON p.guid = measures.guid AND p.recency = 1 ${qualification}
     )`;
-    if (exactGuid !== undefined) {
-        const entries = handle.prepare(`${aggregated} SELECT * FROM ranked WHERE guid = ?`)
-            .all(...parameters, exactGuid) as RankedEntry[];
-        return addRecordSources(handle, entries, options)[0] ?? null;
-    }
-
-    const query = options.q?.trim() ?? '';
-    const search = query ? 'WHERE instr(lower(name), lower(?)) > 0 OR instr(lower(guid), lower(?)) > 0' : '';
-    // Reuse one ranked set for the page and matching count. Summaries deliberately
-    // cover the full game/map scope, including players below an efficiency threshold.
-    const rows = handle.prepare(`${aggregated}, searched AS MATERIALIZED (
-        SELECT * FROM ranked ${search}
-    ), page AS (
-        SELECT * FROM searched ORDER BY ${metric} DESC, name COLLATE NOCASE, name, guid LIMIT ? OFFSET ?
-    ), playerSummary AS (
+    let revision = -1;
+    try { revision = (handle.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'").get() as { revision: number }).revision; }
+    catch { /* Standalone history tools can operate without cache metadata. */ }
+    let cache = scopeCaches.get(handle);
+    if (!cache) { cache = new Map(); scopeCaches.set(handle, cache); }
+    const cacheKey = JSON.stringify([options.game ?? '', options.map ?? '', metric]);
+    let saved = cache.get(cacheKey);
+    if (!saved || revision < 0 || saved.revision !== revision) {
+    const rows = handle.prepare(`${aggregated}, playerSummary AS (
         SELECT COUNT(*) AS players, COALESCE(SUM(kills), 0) AS kills,
             COALESCE(SUM(roundsSurvived), 0) AS roundsSurvived,
             COALESCE(MAX(highestRound), 0) AS highestRound FROM totals
@@ -113,21 +112,18 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
         SELECT COUNT(DISTINCT match_id) AS matches, MIN(matchStartedAt) AS firstTrackedAt,
             MAX(matchUpdatedAt) AS updatedAt FROM observations
     ), metadata AS (
-        SELECT (SELECT COUNT(*) FROM searched) AS total, playerSummary.*, matchSummary.*
-        FROM playerSummary CROSS JOIN matchSummary
-    ) SELECT page.*, metadata.total AS matchingTotal, metadata.players AS summaryPlayers,
+        SELECT playerSummary.*, matchSummary.* FROM playerSummary CROSS JOIN matchSummary
+    ) SELECT ranked.*, metadata.players AS summaryPlayers,
         metadata.matches AS summaryMatches, metadata.kills AS summaryKills,
         metadata.roundsSurvived AS summaryRounds, metadata.highestRound AS summaryHighestRound,
         metadata.firstTrackedAt AS summaryFirstTrackedAt, metadata.updatedAt AS summaryUpdatedAt
-        FROM metadata LEFT JOIN page ON 1
-        ORDER BY page.${metric} DESC, page.name COLLATE NOCASE, page.name, page.guid`)
-        .all(...parameters, ...(query ? [query, query] : []), limit, offset) as LeaderboardRow[];
+        FROM metadata LEFT JOIN ranked ON 1
+        ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`)
+        .all(...parameters) as LeaderboardRow[];
     const rankedEntries = rows.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
         .map(({ matchingTotal, summaryPlayers, summaryMatches, summaryKills, summaryRounds,
             summaryHighestRound, summaryFirstTrackedAt, summaryUpdatedAt, ...entry }) => entry);
-    const entries = addRecordSources(handle, rankedEntries, options);
     const metadata = rows[0];
-    const total = metadata.matchingTotal;
     const summary: LeaderboardResponse['summary'] = {
         players: metadata.summaryPlayers, matches: metadata.summaryMatches, kills: metadata.summaryKills,
         roundsSurvived: metadata.summaryRounds, highestRound: metadata.summaryHighestRound,
@@ -140,5 +136,18 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
         GROUP BY game, map_key HAVING id = MAX(id)
         ORDER BY game, name COLLATE NOCASE, map_key`)
         .all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
-    return { entries, maps, total, limit, offset, metric, summary };
+    saved = { revision, entries: rankedEntries, maps, summary };
+    cache.delete(cacheKey);
+    if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
+    // Bound total retained players as well as the number of scopes.
+    while (cache.size > 8 || [...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000) cache.delete(cache.keys().next().value!);
+    }
+    if (exactGuid !== undefined) {
+        const entry = saved.entries.find(entry => entry.guid === exactGuid);
+        return entry ? addRecordSources(handle, [entry], options)[0] : null;
+    }
+    const query = searchText(options.q?.trim() ?? '');
+    const matching = query ? saved.entries.filter(entry => searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)) : saved.entries;
+    const entries = addRecordSources(handle, matching.slice(offset, offset + limit), options);
+    return { entries, maps: saved.maps, total: matching.length, limit, offset, metric, summary: saved.summary };
 }

src/lib/server/fleet/servers.ts

@@ -179,6 +179,10 @@ export function setServerField(
     field: keyof ServerRow,
     value: string | number | null
 ): void {
+    setServerFields(name, { [field]: value });
+}
+
+export function setServerFields(name: string, fields: Partial<Record<keyof ServerRow, string | number | null>>): void {
     const allowed: Array<keyof ServerRow> = [
         'komodo_type',
         'komodo_name',
@@ -211,8 +215,12 @@ export function setServerField(
         'game_ended_at',
         'game_cycle_key'
     ];
-    if (!allowed.includes(field)) throw new Error(`refusing to set ${field}`);
-    db().prepare(`UPDATE servers SET ${field} = ? WHERE name = ?`).run(value, name);
+    const entries = Object.entries(fields);
+    if (!entries.length) return;
+    for (const [field] of entries) if (!allowed.includes(field as keyof ServerRow)) throw new Error(`refusing to set ${field}`);
+    db().prepare(`UPDATE servers SET ${entries.map(([field]) => `${field} = ?`).join(', ')} WHERE name = ?
+        AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})`)
+        .run(...entries.map(([,value]) => value), name, ...entries.map(([,value]) => value));
 }
 
 /**

src/lib/server/fleet/snapshot.ts

@@ -0,0 +1,51 @@
+import { db } from '../db/index.js';
+import { listServers } from './servers.js';
+import { observationReader } from './status.js';
+import { config, komodoConfigured, rconConfigured } from '../config.js';
+import { runningVersions } from '../integrations/plutonium.js';
+import { listHosts } from '../hosts/index.js';
+import type { NativeInstance } from '../backends/native.js';
+
+export function fleetSnapshot() {
+    const rconHost = config.rcon.host;
+    const hosts = listHosts();
+    const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host]));
+    const versions = new Map(runningVersions().map(version => [version.server, version]));
+    const instances = new Map((db().prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[]).map(instance => [instance.server, instance]));
+    const holds = new Map<string, { target: string; betaId: number }[]>();
+    for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
+        FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id JOIN beta_scripts bs ON bs.beta_id=b.id
+        JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).all() as
+        { server: string; target: string; betaId: number }[]) {
+        const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries);
+    }
+    const observe = observationReader();
+    const servers = listServers().map((s) => {
+        const instance = s.native_instance_id ? instances.get(s.name) : null;
+        return {
+            ...observe(s),
+            host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
+            network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
+            plutonium: versions.get(s.name),
+            // Derived endpoint — same logic as endpointFor() but without asking
+            // Komodo. If we don't have a port yet, endpoint is null.
+            endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
+            held: holds.get(s.name) ?? []
+        };
+    });
+    return {
+        servers,
+        serverNow: Date.now(),
+        reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
+        integrations: {
+            native: hosts.length > 0 || !komodoConfigured(),
+            hostsReady: hosts.filter(host => host.readiness === 'ready').length,
+            hostsTotal: hosts.length,
+            rcon: rconConfigured(),
+            komodo: komodoConfigured(),
+            rconTimeoutMs: config.rcon.timeoutMs,
+            defaultRconHost: rconHost
+        }
+    };
+}
+

src/lib/server/fleet/status.ts

@@ -4,21 +4,57 @@ import { isProvisioning } from './provisioning.js';
 import { restartSettling } from '../provisioning/restarts.js';
 import { type ServerRow } from './servers.js';
 import { telemetryReport, latestGameSubmission } from './status-evidence.js';
+import { db } from '../db/index.js';
+import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
+import { restartRequestInFlight } from '../provisioning/restarts.js';
+
+interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null }
+
+/** One set of evidence reads per snapshot, rather than four queries per server. */
+export function observationReader() {
+    const handle = db();
+    const operations = new Map((handle.prepare(`SELECT server, action, state FROM server_deployments
+        WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as
+        { server: string; action: string; state: string }[]).map(row => [row.server, row]));
+    const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as
+        { server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row]));
+    const submissions = new Map((handle.prepare(`SELECT server, MAX(at) AS at FROM (
+        SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
+        UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as
+        { server: string; at: string | null }[]).map(row => [row.server, row.at]));
+    const reports = new Map<string, GameTelemetry>();
+    for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
+        { server: string; received_at: string; payload: string }[]) {
+        try {
+            const saved = JSON.parse(row.payload) as GameTelemetry;
+            const event = parseGameEvent(saved);
+            if (event.server === row.server) reports.set(row.server, { ...event, receivedAt: row.received_at, timing: saved.timing });
+        } catch { /* Invalid retained telemetry is unavailable. */ }
+    }
+    return (server: ServerRow) => {
+        const operation = operations.get(server.name), restart = restarts.get(server.name);
+        return serverObservation(server, {
+            provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
+            settling: restartRequestInFlight(server.name) || !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
+            telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
+        });
+    };
+}
 
 /** Shared snapshot for REST, status events, metrics, and other server-side readers. */
-export function serverObservation(server: ServerRow) {
+export function serverObservation(server: ServerRow, evidence?: Evidence) {
     let roster = null;
     try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
     const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
         last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
-    const provisioning = isProvisioning(server.name);
-    const state = restartSettling(server.name) ? 'restarting' : server.container_state;
+    const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
+    const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
     const ready = !provisioning && (!state || state === 'running');
     const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
     return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
         status_source: null as 'telemetry' | 'rcon' | null,
-        telemetry: telemetryReport(server.name) ?? undefined,
-        telemetry_after: latestGameSubmission(server.name),
+        telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,
+        telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name),
         last_error: ready ? server.last_error : null,
         replySnippet: ready ? server.last_reply_snippet : null,
         active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null,