From 3a63e6d3ce053fa31995c2a667c75d6a637d0b61 Mon Sep 17 00:00:00 2001 From: josh Date: Sun, 20 Sep 2026 21:58:08 -0400 Subject: [PATCH] Add Metrics tab with historical server utilization and map capacity --- docs/metrics.md | 16 +++++ src/lib/server/db/schema.sql | 10 +++ src/lib/server/fleet/metrics.ts | 78 ++++++++++++++++++++ src/lib/server/integrations/rcon.ts | 7 ++ src/lib/server/pipeline/worker.ts | 14 ++++ src/routes/+layout.svelte | 9 +-- src/routes/metrics/+page.server.ts | 7 ++ src/routes/metrics/+page.svelte | 108 ++++++++++++++++++++++++++++ tests/unit/metrics.test.ts | 59 +++++++++++++++ 9 files changed, 304 insertions(+), 4 deletions(-) create mode 100644 docs/metrics.md create mode 100644 src/lib/server/fleet/metrics.ts create mode 100644 src/routes/metrics/+page.server.ts create mode 100644 src/routes/metrics/+page.svelte create mode 100644 tests/unit/metrics.test.ts diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 0000000..a2297c0 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,16 @@ +# Metrics + +Metrics is a primary navigation tab at `/metrics` (keyboard shortcut: `g m`). +Choose the past 24 hours, 7 days, or 30 days. The page refreshes every minute. + +- Utilized means a successful status response reported at least one player. +- Idle means a successful response reported zero players. +- Unknown covers failed checks, provisioning, and unavailable containers; it is excluded from utilization percentages. +- Player-hours integrate the reported player count over observed time. Bots are included if the server counts them as players. +- All lobbies full measures map wall time when every tracked lobby in that game/map group has a known capacity and is full. Missing capacity or status makes that interval unknown. This is a capacity-pressure indicator, not a count of rejected joins or unmet demand. + +Observations are recorded by the existing status worker. Each remains valid until the next collection, up to the 45-second freshness limit. Longer manager outages are excluded. Charts split time at UTC hour/day boundaries; partial periods remain partial. Map variants use the configured variant when it matches the reported map. + +Capacity comes from `sv_maxclients` / `sv_maxClients` in the status response. When unavailable, the UI displays unknown rather than assuming a four-player limit. Capacity coverage is the map time for which every tracked lobby reported both occupancy and capacity. + +Collection begins after this feature is installed; previous occupancy cannot be reconstructed. Samples are retained for 31 days and pruned during collection. They survive removal of a server and contain no player identities. The additive database table is created automatically on startup. diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 82ee6ee..81889dd 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -560,3 +560,13 @@ BEGIN WHEN NEW.restart_state = 'manual' THEN 'Manual restart required' WHEN NEW.restart_state = 'held' THEN 'Changes held' ELSE NEW.restart_state END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END); END; +-- Historical occupancy survives server removal; no player identities are stored. +CREATE TABLE IF NOT EXISTS metric_samples ( + at INTEGER NOT NULL, + server TEXT NOT NULL, + game TEXT NOT NULL, + map TEXT NOT NULL, + players INTEGER, + capacity INTEGER, + PRIMARY KEY (at, server) +); diff --git a/src/lib/server/fleet/metrics.ts b/src/lib/server/fleet/metrics.ts new file mode 100644 index 0000000..f703852 --- /dev/null +++ b/src/lib/server/fleet/metrics.ts @@ -0,0 +1,78 @@ +import { db } from '../db/index.js'; +import { ONLINE_MAX_MS } from '../../server-freshness.js'; + +export interface MetricSample { + server: string; + game: string; + map: string; + players: number | null; + capacity: number | null; +} + +export function recordMetrics(samples: MetricSample[], at = Date.now()): void { + const handle = db(); + handle.transaction(() => { + const insert = handle.prepare('INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)'); + for (const s of samples) insert.run(at, s.server, s.game, s.map, s.players, s.capacity); + handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - 31 * 86400_000); + })(); +} + +export interface Utilization { + observed: number; + active: number; + idle: number; + unknown: number; + playerHours: number; +} +export interface MapMetrics extends Utilization { + game: string; + map: string; + full: number; + capacityObserved: number; +} + +/** Treat each observation as valid until the next poll, capped at the freshness limit. + * Longer collection gaps are excluded, never interpreted as empty servers. */ +export function readMetrics(days: number, now = Date.now()) { + const since = now - days * 86400_000; + const bucketMs = days === 1 ? 3600_000 : 86400_000; + const handle = db(); + const cte = `WITH times AS ( + SELECT at, LEAD(at, 1, @now) OVER (ORDER BY at) AS next_at + FROM (SELECT DISTINCT at FROM metric_samples WHERE at >= @since - @fresh AND at <= @now) + ), samples AS ( + SELECT s.*, MAX(0, MIN(t.next_at, s.at + @fresh, @now) - MAX(s.at, @since)) / 3600000.0 AS hours + FROM metric_samples s JOIN times t ON t.at = s.at + )`; + const params = { now, since, fresh: ONLINE_MAX_MS }; + const totals = `COALESCE(SUM(CASE WHEN players IS NOT NULL THEN hours ELSE 0 END), 0) AS observed, + COALESCE(SUM(CASE WHEN players > 0 THEN hours ELSE 0 END), 0) AS active, + COALESCE(SUM(CASE WHEN players = 0 THEN hours ELSE 0 END), 0) AS idle, + COALESCE(SUM(CASE WHEN players IS NULL THEN hours ELSE 0 END), 0) AS unknown, + COALESCE(SUM(players * hours), 0) AS playerHours`; + const summary = handle.prepare(`${cte} SELECT ${totals} FROM samples`).get(params) as Utilization; + const servers = handle.prepare(`${cte} SELECT server, ${totals} FROM samples GROUP BY server ORDER BY active DESC, server`).all(params) as (Utilization & { server: string })[]; + const maps = handle.prepare(`${cte}, pressure AS ( + SELECT game, map, at, MAX(hours) AS hours, + MIN(CASE WHEN players IS NOT NULL AND capacity > 0 THEN 1 ELSE 0 END) AS known, + MIN(CASE WHEN players >= capacity AND capacity > 0 THEN 1 ELSE 0 END) AS full + FROM samples GROUP BY game, map, at + ), map_pressure AS ( + SELECT game, map, SUM(CASE WHEN known = 1 THEN hours ELSE 0 END) AS capacityObserved, + SUM(CASE WHEN known = 1 AND full = 1 THEN hours ELSE 0 END) AS full + FROM pressure GROUP BY game, map + ) SELECT s.game, s.map, ${totals}, p.full, p.capacityObserved + FROM samples s JOIN map_pressure p ON p.game = s.game AND p.map = s.map + GROUP BY s.game, s.map ORDER BY playerHours DESC, s.map`).all(params) as MapMetrics[]; + // Split intervals at chart boundaries so a midnight poll cannot shift time into the wrong day. + const points: (Utilization & { at: number })[] = []; + for (let at = Math.floor(since / bucketMs) * bucketMs; at < now; at += bucketMs) { + const end = Math.min(at + bucketMs, now); + const start = Math.max(at, since); + const point = handle.prepare(`${cte} SELECT ${totals} FROM samples`) + .get({ ...params, since: start, now: end }) as Utilization; + points.push({ at, ...point }); + } + return { days, since, now, summary, servers, maps, points }; +} diff --git a/src/lib/server/integrations/rcon.ts b/src/lib/server/integrations/rcon.ts index 839b4f0..947f0d0 100644 --- a/src/lib/server/integrations/rcon.ts +++ b/src/lib/server/integrations/rcon.ts @@ -310,6 +310,7 @@ export async function rconGetPlayerCount(server: string): Promise export interface RconStatus { players: number; + capacity?: number | null; map: string | null; gametype: string | null; roster: RconPlayer[]; @@ -437,6 +438,7 @@ export async function rconStatusResult(server: string): Promise 0 ? count : null; +} diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index 2a7642b..f9ece10 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -11,6 +11,8 @@ import { restartSettling } from './restart-rollout.js'; import { classifyHealth } from '../fleet/health.js'; import { isProvisioning } from '../fleet/provisioning.js'; import { STATUS_REFRESH_MS } from '../../server-freshness.js'; +import { recordMetrics } from '../fleet/metrics.js'; +import { selectedZombieMap } from '../../map-aliases.js'; const TICK_MS = 15_000; // Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the @@ -68,6 +70,7 @@ async function refreshStatus(): Promise { if (servers.length === 0) return; const entries: ServerStatusEntry[] = new Array(servers.length); + const capacities = new Map(); let cursor = 0; async function pump(): Promise { @@ -119,6 +122,7 @@ async function refreshStatus(): Promise { const now = new Date().toISOString(); if (result.ok) { + capacities.set(srv.name, result.status.capacity ?? null); // Compare against the previously persisted count BEFORE we // overwrite last_players — occupancy transitions are anchored // to that boundary. @@ -203,6 +207,16 @@ async function refreshStatus(): Promise { } await Promise.all(Array.from({ length: STATUS_CONCURRENCY }, pump)); + try { + recordMetrics(servers.map((srv, i) => ({ + server: srv.name, + game: srv.game ?? 'unknown', + map: selectedZombieMap(srv.game, entries[i].map ?? srv.map, srv.map_slug)?.friendly ?? entries[i].map ?? srv.map ?? 'Unknown map', + players: capacities.has(srv.name) ? entries[i].players : null, + capacity: capacities.get(srv.name) ?? null + }))); + } catch (err) { console.error('[worker] metrics error:', err); } + emit({ type: 'server-status', entries }); } diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index d3a957d..0dbadc6 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -51,7 +51,7 @@ // `g b` betas, `g s` scripts. Ignored while typing in inputs. const shortcuts: Record = { f: '/', d: '/deployments', a: '/campaigns', b: '/betas', - k: '/server-keys', s: '/scripts', c: '/configs', p: '/plugins', l: '/leaderboard', h: '/settings' + k: '/server-keys', s: '/scripts', c: '/configs', p: '/plugins', l: '/leaderboard', h: '/settings', m: '/metrics' }; let armed = false; let armTimer: ReturnType | null = null; @@ -100,7 +100,8 @@ { href: '/betas', label: 'Betas', match: (p: string) => p.startsWith('/betas') }, { href: '/settings', label: 'Settings', match: (p: string) => p.startsWith('/server-keys') || p.startsWith('/settings') }, { href: '/leaderboard', label: 'Leaderboard', match: (p: string) => p.startsWith('/leaderboard') }, - { href: '/matches', label: 'Match History', match: (p: string) => p.startsWith('/matches') } + { href: '/matches', label: 'Match History', match: (p: string) => p.startsWith('/matches') }, + { href: '/metrics', label: 'Metrics', match: (p: string) => p === '/metrics' } ]; const secondary = $derived(tabs[1].match($page.url.pathname) ? [{ href: '/scripts', label: 'Scripts' }, { href: '/configs', label: 'Configs' }, { href: '/plugins', label: 'Plugins' }] @@ -145,7 +146,7 @@ -
{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Server deployments'],['g a','Campaigns'],['g b','Betas'],['g h','System status'],['g k','Server keys'],['g l','Leaderboard'],['?','This help']] as [keys,label]}
{keys}
{label}
{/each}
+
{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Server deployments'],['g a','Campaigns'],['g b','Betas'],['g h','System status'],['g k','Server keys'],['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}
{keys}
{label}
{/each}
 @@ -268,7 +269,7 @@ } /* The full navigation needs more room than a phone or portrait tablet. */ - @media (max-width: 1080px) { + @media (max-width: 1200px) { nav { flex-wrap: wrap; justify-content: space-between; diff --git a/src/routes/metrics/+page.server.ts b/src/routes/metrics/+page.server.ts new file mode 100644 index 0000000..a26f2f3 --- /dev/null +++ b/src/routes/metrics/+page.server.ts @@ -0,0 +1,7 @@ +import { readMetrics } from '$lib/server/fleet/metrics.js'; +import type { PageServerLoad } from './$types'; + +export const load: PageServerLoad = ({ url }) => { + const requested = Number(url.searchParams.get('days') ?? 7); + return readMetrics([1, 7, 30].includes(requested) ? requested : 7); +}; diff --git a/src/routes/metrics/+page.svelte b/src/routes/metrics/+page.svelte new file mode 100644 index 0000000..0d09f2a --- /dev/null +++ b/src/routes/metrics/+page.svelte @@ -0,0 +1,108 @@ + + +Metrics · Electric Cherry + +
+ +
+ + Updated {new Date(data.now).toISOString().slice(11, 19)} UTC · refreshes every minute +
+ + {#if data.servers.length === 0} +

Collecting your first observations

Metrics start with the next server status checks. Leave the manager running to build a history of busy hours and map demand.

+ {:else} +
+
Servers utilized{percent(data.summary.active, data.summary.observed)}{hours(data.summary.active)} with players
+
Servers idle{percent(data.summary.idle, data.summary.observed)}{hours(data.summary.idle)} empty
+
Player time{hours(data.summary.playerHours)}Players × observed time
+
Unknown status{hours(data.summary.unknown)}Excluded from utilization percentages
+
+ +
+

Utilization over time

+

Server-hours per {data.days === 1 ? 'hour' : 'day'} (UTC). Taller bars mean more time under observation.

+
UtilizedIdleUnknown
+ +
{label(data.points[0].at)}0–{hours(peak)} per bar{label(data.points[data.points.length - 1].at)} UTC
+
View time data
{#each data.points as point}{/each}
Period (UTC)UtilizedIdleUnknownPlayer time
{label(point.at)}{hours(point.active)}{hours(point.idle)}{hours(point.unknown)}{hours(point.playerHours)}
+
+ +
+

Map play time & capacity pressure

+

Ranked by player-hours. “All lobbies full” measures time when every tracked lobby for a map has known capacity and is full. It indicates capacity pressure; failed joins and unmet demand are not tracked.

+
+ + {#each data.maps as map}{/each} +
MapPlayer timeUtilized server timeAll lobbies fullCapacity coverage
{map.map}{map.game.toUpperCase()}{hours(map.playerHours)}{hours(map.active)}{map.capacityObserved > 0 ? `${hours(map.full)} (${percent(map.full, map.capacityObserved)})` : 'Unknown'}{hours(map.capacityObserved)}
+

Capacity coverage is map time with a known player limit and status for every tracked lobby. Servers that do not report a player limit cannot establish whether all lobbies are full.

+
+ +
+

Server utilization

+
+ {#each data.servers as server}{/each} +
ServerUtilizedIdleObserved timeUnknown
{server.server}{percent(server.active, server.observed)}{percent(server.idle, server.observed)}{hours(server.observed)}{hours(server.unknown)}
+
+ {/if} +

Estimates use periodic server status observations. Each observation covers at most 45 seconds; longer collection gaps are excluded. History begins when collection is enabled and is retained for 31 days. Player time includes all players reported by the server.

+
+ + diff --git a/tests/unit/metrics.test.ts b/tests/unit/metrics.test.ts new file mode 100644 index 0000000..163af94 --- /dev/null +++ b/tests/unit/metrics.test.ts @@ -0,0 +1,59 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +let fixture: Fixture; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { await fixture.cleanup(); }); + +describe('historical metrics', () => { + it('weights occupancy by time and excludes failed checks and collection gaps', async () => { + const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const sample = { server: 'one', game: 't5', map: 'Moon', players: 2, capacity: 4 }; + const at = Date.UTC(2026, 0, 2, 12); + recordMetrics([sample], at); + recordMetrics([{ ...sample, players: 0 }], at + 30_000); + recordMetrics([{ ...sample, players: null }], at + 60_000); + const result = readMetrics(1, at + 3600_000); + expect(result.summary.active * 3600).toBeCloseTo(30); + expect(result.summary.idle * 3600).toBeCloseTo(30); + expect(result.summary.unknown * 3600).toBeCloseTo(45); + expect(result.summary.playerHours * 3600).toBeCloseTo(60); + expect(result.points.reduce((sum, p) => sum + p.observed, 0)).toBeCloseTo(result.summary.observed); + }); + + it('only counts map saturation when every lobby has known capacity and is full', async () => { + const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const at = Date.UTC(2026, 0, 2, 12); + const sample = { server: 'one', game: 't5', map: 'Moon', players: 4, capacity: 4 }; + recordMetrics([sample, { ...sample, server: 'two' }], at); + recordMetrics([sample, { ...sample, server: 'two', players: 2 }], at + 30_000); + recordMetrics([sample, { ...sample, server: 'two', capacity: null }], at + 60_000); + recordMetrics([sample, { ...sample, server: 'two', players: null }], at + 90_000); + const result = readMetrics(1, at + 120_000); + expect(result.maps[0].full * 3600).toBeCloseTo(30); + expect(result.maps[0].capacityObserved * 3600).toBeCloseTo(60); + }); + + it('clips the reporting window and splits observations across UTC chart boundaries', async () => { + const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const midnight = Date.UTC(2026, 0, 2); + const sample = { server: 'one', game: 't5', map: 'Moon', players: 1, capacity: 4 }; + recordMetrics([sample], midnight - 15_000); + let result = readMetrics(1, midnight + 15_000); + expect(result.points.filter(p => p.active > 0).map(p => p.active * 3600)).toEqual([15, 15]); + result = readMetrics(1, midnight + 86400_000); + expect(result.summary.active * 3600).toBeCloseTo(30); + expect(result.points.reduce((sum, p) => sum + p.active, 0)).toBeCloseTo(result.summary.active); + }); + + it('retains history without a live server and prunes observations older than 31 days', async () => { + const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const at = Date.UTC(2026, 0, 2); + const sample = { server: 'removed', game: 't6', map: 'Town', players: 1, capacity: null }; + recordMetrics([sample], at); + expect(readMetrics(1, at + 30_000).servers[0].server).toBe('removed'); + recordMetrics([sample], at + 32 * 86400_000); + expect(db().prepare('SELECT COUNT(*) AS n FROM metric_samples').get()).toEqual({ n: 1 }); + }); +});