Add Metrics tab with historical server utilization and map capacity

This commit is contained in:
2026-09-20 21:58:08 -04:00
parent 3c8b5b2e40
commit 3a63e6d3ce
9 changed files with 304 additions and 4 deletions
+16
View File
@@ -0,0 +1,16 @@
# Metrics
Metrics is a primary navigation tab at `/metrics` (keyboard shortcut: `g m`).
Choose the past 24 hours, 7 days, or 30 days. The page refreshes every minute.
- Utilized means a successful status response reported at least one player.
- Idle means a successful response reported zero players.
- Unknown covers failed checks, provisioning, and unavailable containers; it is excluded from utilization percentages.
- Player-hours integrate the reported player count over observed time. Bots are included if the server counts them as players.
- All lobbies full measures map wall time when every tracked lobby in that game/map group has a known capacity and is full. Missing capacity or status makes that interval unknown. This is a capacity-pressure indicator, not a count of rejected joins or unmet demand.
Observations are recorded by the existing status worker. Each remains valid until the next collection, up to the 45-second freshness limit. Longer manager outages are excluded. Charts split time at UTC hour/day boundaries; partial periods remain partial. Map variants use the configured variant when it matches the reported map.
Capacity comes from `sv_maxclients` / `sv_maxClients` in the status response. When unavailable, the UI displays unknown rather than assuming a four-player limit. Capacity coverage is the map time for which every tracked lobby reported both occupancy and capacity.
Collection begins after this feature is installed; previous occupancy cannot be reconstructed. Samples are retained for 31 days and pruned during collection. They survive removal of a server and contain no player identities. The additive database table is created automatically on startup.
+10
View File
@@ -560,3 +560,13 @@ BEGIN
WHEN NEW.restart_state = 'manual' THEN 'Manual restart required'
WHEN NEW.restart_state = 'held' THEN 'Changes held' ELSE NEW.restart_state END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END);
END;
-- Historical occupancy survives server removal; no player identities are stored.
CREATE TABLE IF NOT EXISTS metric_samples (
at INTEGER NOT NULL,
server TEXT NOT NULL,
game TEXT NOT NULL,
map TEXT NOT NULL,
players INTEGER,
capacity INTEGER,
PRIMARY KEY (at, server)
);
+78
View File
@@ -0,0 +1,78 @@
import { db } from '../db/index.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
export interface MetricSample {
server: string;
game: string;
map: string;
players: number | null;
capacity: number | null;
}
export function recordMetrics(samples: MetricSample[], at = Date.now()): void {
const handle = db();
handle.transaction(() => {
const insert = handle.prepare('INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)');
for (const s of samples) insert.run(at, s.server, s.game, s.map, s.players, s.capacity);
handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - 31 * 86400_000);
})();
}
export interface Utilization {
observed: number;
active: number;
idle: number;
unknown: number;
playerHours: number;
}
export interface MapMetrics extends Utilization {
game: string;
map: string;
full: number;
capacityObserved: number;
}
/** Treat each observation as valid until the next poll, capped at the freshness limit.
* Longer collection gaps are excluded, never interpreted as empty servers. */
export function readMetrics(days: number, now = Date.now()) {
const since = now - days * 86400_000;
const bucketMs = days === 1 ? 3600_000 : 86400_000;
const handle = db();
const cte = `WITH times AS (
SELECT at, LEAD(at, 1, @now) OVER (ORDER BY at) AS next_at
FROM (SELECT DISTINCT at FROM metric_samples WHERE at >= @since - @fresh AND at <= @now)
), samples AS (
SELECT s.*, MAX(0, MIN(t.next_at, s.at + @fresh, @now) - MAX(s.at, @since)) / 3600000.0 AS hours
FROM metric_samples s JOIN times t ON t.at = s.at
)`;
const params = { now, since, fresh: ONLINE_MAX_MS };
const totals = `COALESCE(SUM(CASE WHEN players IS NOT NULL THEN hours ELSE 0 END), 0) AS observed,
COALESCE(SUM(CASE WHEN players > 0 THEN hours ELSE 0 END), 0) AS active,
COALESCE(SUM(CASE WHEN players = 0 THEN hours ELSE 0 END), 0) AS idle,
COALESCE(SUM(CASE WHEN players IS NULL THEN hours ELSE 0 END), 0) AS unknown,
COALESCE(SUM(players * hours), 0) AS playerHours`;
const summary = handle.prepare(`${cte} SELECT ${totals} FROM samples`).get(params) as Utilization;
const servers = handle.prepare(`${cte} SELECT server, ${totals} FROM samples GROUP BY server ORDER BY active DESC, server`).all(params) as (Utilization & { server: string })[];
const maps = handle.prepare(`${cte}, pressure AS (
SELECT game, map, at, MAX(hours) AS hours,
MIN(CASE WHEN players IS NOT NULL AND capacity > 0 THEN 1 ELSE 0 END) AS known,
MIN(CASE WHEN players >= capacity AND capacity > 0 THEN 1 ELSE 0 END) AS full
FROM samples GROUP BY game, map, at
), map_pressure AS (
SELECT game, map, SUM(CASE WHEN known = 1 THEN hours ELSE 0 END) AS capacityObserved,
SUM(CASE WHEN known = 1 AND full = 1 THEN hours ELSE 0 END) AS full
FROM pressure GROUP BY game, map
) SELECT s.game, s.map, ${totals}, p.full, p.capacityObserved
FROM samples s JOIN map_pressure p ON p.game = s.game AND p.map = s.map
GROUP BY s.game, s.map ORDER BY playerHours DESC, s.map`).all(params) as MapMetrics[];
// Split intervals at chart boundaries so a midnight poll cannot shift time into the wrong day.
const points: (Utilization & { at: number })[] = [];
for (let at = Math.floor(since / bucketMs) * bucketMs; at < now; at += bucketMs) {
const end = Math.min(at + bucketMs, now);
const start = Math.max(at, since);
const point = handle.prepare(`${cte} SELECT ${totals} FROM samples`)
.get({ ...params, since: start, now: end }) as Utilization;
points.push({ at, ...point });
}
return { days, since, now, summary, servers, maps, points };
}
+7
View File
@@ -310,6 +310,7 @@ export async function rconGetPlayerCount(server: string): Promise<number | null>
export interface RconStatus {
players: number;
capacity?: number | null;
map: string | null;
gametype: string | null;
roster: RconPlayer[];
@@ -437,6 +438,7 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
// Prefer statusResponse `\info\` line values when present, fall
// back to print-body `key: value` header lines otherwise.
players,
capacity: positiveCapacity(info.sv_maxclients ?? info.sv_maxClients ?? header.sv_maxclients ?? header.sv_maxClients),
map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null,
gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null,
roster: parsePlayerRoster(text)
@@ -444,3 +446,8 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
rawReply: formatRawReply(send.buf)
};
}
function positiveCapacity(value: string | undefined): number | null {
const count = Number(value);
return Number.isInteger(count) && count > 0 ? count : null;
}
+14
View File
@@ -11,6 +11,8 @@ import { restartSettling } from './restart-rollout.js';
import { classifyHealth } from '../fleet/health.js';
import { isProvisioning } from '../fleet/provisioning.js';
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
import { recordMetrics } from '../fleet/metrics.js';
import { selectedZombieMap } from '../../map-aliases.js';
const TICK_MS = 15_000;
// Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
@@ -68,6 +70,7 @@ async function refreshStatus(): Promise<void> {
if (servers.length === 0) return;
const entries: ServerStatusEntry[] = new Array(servers.length);
const capacities = new Map<string, number | null>();
let cursor = 0;
async function pump(): Promise<void> {
@@ -119,6 +122,7 @@ async function refreshStatus(): Promise<void> {
const now = new Date().toISOString();
if (result.ok) {
capacities.set(srv.name, result.status.capacity ?? null);
// Compare against the previously persisted count BEFORE we
// overwrite last_players — occupancy transitions are anchored
// to that boundary.
@@ -203,6 +207,16 @@ async function refreshStatus(): Promise<void> {
}
await Promise.all(Array.from({ length: STATUS_CONCURRENCY }, pump));
try {
recordMetrics(servers.map((srv, i) => ({
server: srv.name,
game: srv.game ?? 'unknown',
map: selectedZombieMap(srv.game, entries[i].map ?? srv.map, srv.map_slug)?.friendly ?? entries[i].map ?? srv.map ?? 'Unknown map',
players: capacities.has(srv.name) ? entries[i].players : null,
capacity: capacities.get(srv.name) ?? null
})));
} catch (err) { console.error('[worker] metrics error:', err); }
emit({ type: 'server-status', entries });
}
+5 -4
View File
@@ -51,7 +51,7 @@
// `g b` betas, `g s` scripts. Ignored while typing in inputs.
const shortcuts: Record<string, string> = {
f: '/', d: '/deployments', a: '/campaigns', b: '/betas',
k: '/server-keys', s: '/scripts', c: '/configs', p: '/plugins', l: '/leaderboard', h: '/settings'
k: '/server-keys', s: '/scripts', c: '/configs', p: '/plugins', l: '/leaderboard', h: '/settings', m: '/metrics'
};
let armed = false;
let armTimer: ReturnType<typeof setTimeout> | null = null;
@@ -100,7 +100,8 @@
{ href: '/betas', label: 'Betas', match: (p: string) => p.startsWith('/betas') },
{ href: '/settings', label: 'Settings', match: (p: string) => p.startsWith('/server-keys') || p.startsWith('/settings') },
{ href: '/leaderboard', label: 'Leaderboard', match: (p: string) => p.startsWith('/leaderboard') },
{ href: '/matches', label: 'Match History', match: (p: string) => p.startsWith('/matches') }
{ href: '/matches', label: 'Match History', match: (p: string) => p.startsWith('/matches') },
{ href: '/metrics', label: 'Metrics', match: (p: string) => p === '/metrics' }
];
const secondary = $derived(tabs[1].match($page.url.pathname)
? [{ href: '/scripts', label: 'Scripts' }, { href: '/configs', label: 'Configs' }, { href: '/plugins', label: 'Plugins' }]
@@ -145,7 +146,7 @@
<Confirmation />
<NotificationHistory bind:open={notificationsOpen} />
<Dialog bind:open={helpOpen} title="Keyboard shortcuts" description="Press g, then the destination key. Shortcuts are inactive while typing or using a dialog.">
<dl class="shortcuts">{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Server deployments'],['g a','Campaigns'],['g b','Betas'],['g h','System status'],['g k','Server keys'],['g l','Leaderboard'],['?','This help']] as [keys,label]}<div><dt><kbd>{keys}</kbd></dt><dd>{label}</dd></div>{/each}</dl>
<dl class="shortcuts">{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Server deployments'],['g a','Campaigns'],['g b','Betas'],['g h','System status'],['g k','Server keys'],['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}<div><dt><kbd>{keys}</kbd></dt><dd>{label}</dd></div>{/each}</dl>
</Dialog>
</div>
@@ -268,7 +269,7 @@
}
/* The full navigation needs more room than a phone or portrait tablet. */
@media (max-width: 1080px) {
@media (max-width: 1200px) {
nav {
flex-wrap: wrap;
justify-content: space-between;
+7
View File
@@ -0,0 +1,7 @@
import { readMetrics } from '$lib/server/fleet/metrics.js';
import type { PageServerLoad } from './$types';
export const load: PageServerLoad = ({ url }) => {
const requested = Number(url.searchParams.get('days') ?? 7);
return readMetrics([1, 7, 30].includes(requested) ? requested : 7);
};
+108
View File
@@ -0,0 +1,108 @@
<script lang="ts">
import { onMount } from 'svelte';
import { invalidateAll } from '$app/navigation';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import type { PageData } from './$types';
let { data }: { data: PageData } = $props();
const percent = (value: number, total: number) => total > 0 ? `${(100 * value / total).toFixed(1)}%` : '—';
const hours = (value: number) => `${value.toFixed(1)} h`;
const peak = $derived(Math.max(1, ...data.points.map(p => p.active + p.idle + p.unknown)));
const label = (at: number) => new Date(at).toISOString().slice(0, data.days === 1 ? 16 : 10).replace('T', ' ');
onMount(() => {
const timer = setInterval(() => { if (!document.hidden) void invalidateAll(); }, 60_000);
return () => clearInterval(timer);
});
</script>
<svelte:head><title>Metrics · Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title="Metrics" subtitle="Plan hosting around server utilization, map play time, and capacity pressure." />
<div class="toolbar">
<nav aria-label="Metrics time range">
{#each [1, 7, 30] as days}
<a href={`/metrics?days=${days}`} aria-current={data.days === days ? 'page' : undefined}>{days === 1 ? '24 hours' : `${days} days`}</a>
{/each}
</nav>
<span>Updated {new Date(data.now).toISOString().slice(11, 19)} UTC · refreshes every minute</span>
</div>
{#if data.servers.length === 0}
<section><h2>Collecting your first observations</h2><p>Metrics start with the next server status checks. Leave the manager running to build a history of busy hours and map demand.</p></section>
{:else}
<div class="stats">
<section><span>Servers utilized</span><strong>{percent(data.summary.active, data.summary.observed)}</strong><small>{hours(data.summary.active)} with players</small></section>
<section><span>Servers idle</span><strong>{percent(data.summary.idle, data.summary.observed)}</strong><small>{hours(data.summary.idle)} empty</small></section>
<section><span>Player time</span><strong>{hours(data.summary.playerHours)}</strong><small>Players × observed time</small></section>
<section><span>Unknown status</span><strong>{hours(data.summary.unknown)}</strong><small>Excluded from utilization percentages</small></section>
</div>
<section>
<h2>Utilization over time</h2>
<p>Server-hours per {data.days === 1 ? 'hour' : 'day'} (UTC). Taller bars mean more time under observation.</p>
<div class="legend"><span><i class="active"></i>Utilized</span><span><i class="idle"></i>Idle</span><span><i class="unknown"></i>Unknown</span></div>
<div class="chart" role="img" aria-label="Stacked server-hours chart. Exact values are in the time data table below.">
{#each data.points as point}
<div class="bar" title={`${label(point.at)} UTC: ${hours(point.active)} utilized, ${hours(point.idle)} idle, ${hours(point.unknown)} unknown`}>
<div class="unknown" style:height={`${point.unknown / peak * 100}%`}></div>
<div class="idle" style:height={`${point.idle / peak * 100}%`}></div>
<div class="active" style:height={`${point.active / peak * 100}%`}></div>
</div>
{/each}
</div>
<div class="axis"><span>{label(data.points[0].at)}</span><span>0–{hours(peak)} per bar</span><span>{label(data.points[data.points.length - 1].at)} UTC</span></div>
<details><summary>View time data</summary><div class="table-wrap"><table><thead><tr><th>Period (UTC)</th><th>Utilized</th><th>Idle</th><th>Unknown</th><th>Player time</th></tr></thead><tbody>{#each data.points as point}<tr><td>{label(point.at)}</td><td>{hours(point.active)}</td><td>{hours(point.idle)}</td><td>{hours(point.unknown)}</td><td>{hours(point.playerHours)}</td></tr>{/each}</tbody></table></div></details>
</section>
<section>
<h2>Map play time & capacity pressure</h2>
<p>Ranked by player-hours. “All lobbies full” measures time when every tracked lobby for a map has known capacity and is full. It indicates capacity pressure; failed joins and unmet demand are not tracked.</p>
<div class="table-wrap"><table>
<thead><tr><th>Map</th><th>Player time</th><th>Utilized server time</th><th>All lobbies full</th><th>Capacity coverage</th></tr></thead>
<tbody>{#each data.maps as map}<tr><td>{map.map}<small>{map.game.toUpperCase()}</small></td><td>{hours(map.playerHours)}</td><td>{hours(map.active)}</td><td>{map.capacityObserved > 0 ? `${hours(map.full)} (${percent(map.full, map.capacityObserved)})` : 'Unknown'}</td><td>{hours(map.capacityObserved)}</td></tr>{/each}</tbody>
</table></div>
<p class="note">Capacity coverage is map time with a known player limit and status for every tracked lobby. Servers that do not report a player limit cannot establish whether all lobbies are full.</p>
</section>
<section>
<h2>Server utilization</h2>
<div class="table-wrap"><table><thead><tr><th>Server</th><th>Utilized</th><th>Idle</th><th>Observed time</th><th>Unknown</th></tr></thead>
<tbody>{#each data.servers as server}<tr><td>{server.server}</td><td>{percent(server.active, server.observed)}</td><td>{percent(server.idle, server.observed)}</td><td>{hours(server.observed)}</td><td>{hours(server.unknown)}</td></tr>{/each}</tbody>
</table></div>
</section>
{/if}
<p class="note">Estimates use periodic server status observations. Each observation covers at most 45 seconds; longer collection gaps are excluded. History begins when collection is enabled and is retained for 31 days. Player time includes all players reported by the server.</p>
</div>
<style>
.page { display: grid; gap: var(--space-6); min-width: 0; }
section { background: var(--surface); border: 1px solid var(--border-subtle); border-radius: var(--radius-lg); padding: var(--space-5); min-width: 0; }
h2 { font-size: var(--text-lg); margin: 0 0 12px; }
p { color: var(--text-secondary); margin: 8px 0 16px; }
.toolbar, .axis, .legend { display: flex; justify-content: space-between; gap: 16px; flex-wrap: wrap; }
.toolbar { align-items: center; }
.toolbar span, .axis, .note, small { color: var(--text-tertiary); font-size: var(--text-sm); }
nav { display: flex; gap: 8px; }
nav a { padding: 8px 12px; border-radius: var(--radius-md); text-decoration: none; }
nav a[aria-current] { background: var(--surface-raised); color: var(--text-primary); }
.stats { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 16px; }
.stats section { display: grid; gap: 8px; }
strong { font-size: 30px; font-variant-numeric: tabular-nums; }
small { display: block; }
.legend { justify-content: flex-start; font-size: var(--text-sm); }
.legend span { display: flex; align-items: center; gap: 6px; }
i { display: inline-block; width: 10px; height: 10px; border-radius: 2px; }
.active { background: var(--accent-primary); }
.idle { background: #7b9baf; }
.unknown { background: repeating-linear-gradient(135deg, #605b64 0 3px, #a7a0ad 3px 5px); }
.chart { display: flex; height: 200px; gap: 4px; border-bottom: 1px solid var(--border-strong); margin: 20px 0 8px; }
.bar { flex: 1; min-width: 0; display: flex; flex-direction: column; justify-content: flex-end; }
details { margin-top: 20px; } summary { cursor: pointer; }
.table-wrap { overflow-x: auto; }
table { width: 100%; border-collapse: collapse; text-align: left; font-size: var(--text-sm); }
th, td { padding: 12px; border-bottom: 1px solid var(--border-subtle); font-variant-numeric: tabular-nums; }
th { color: var(--text-secondary); white-space: nowrap; }
@media (max-width: 800px) { .stats { grid-template-columns: repeat(2, minmax(0, 1fr)); } }
@media (max-width: 450px) { .stats { grid-template-columns: 1fr; } }
</style>
+59
View File
@@ -0,0 +1,59 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); });
describe('historical metrics', () => {
it('weights occupancy by time and excludes failed checks and collection gaps', async () => {
const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js');
const sample = { server: 'one', game: 't5', map: 'Moon', players: 2, capacity: 4 };
const at = Date.UTC(2026, 0, 2, 12);
recordMetrics([sample], at);
recordMetrics([{ ...sample, players: 0 }], at + 30_000);
recordMetrics([{ ...sample, players: null }], at + 60_000);
const result = readMetrics(1, at + 3600_000);
expect(result.summary.active * 3600).toBeCloseTo(30);
expect(result.summary.idle * 3600).toBeCloseTo(30);
expect(result.summary.unknown * 3600).toBeCloseTo(45);
expect(result.summary.playerHours * 3600).toBeCloseTo(60);
expect(result.points.reduce((sum, p) => sum + p.observed, 0)).toBeCloseTo(result.summary.observed);
});
it('only counts map saturation when every lobby has known capacity and is full', async () => {
const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js');
const at = Date.UTC(2026, 0, 2, 12);
const sample = { server: 'one', game: 't5', map: 'Moon', players: 4, capacity: 4 };
recordMetrics([sample, { ...sample, server: 'two' }], at);
recordMetrics([sample, { ...sample, server: 'two', players: 2 }], at + 30_000);
recordMetrics([sample, { ...sample, server: 'two', capacity: null }], at + 60_000);
recordMetrics([sample, { ...sample, server: 'two', players: null }], at + 90_000);
const result = readMetrics(1, at + 120_000);
expect(result.maps[0].full * 3600).toBeCloseTo(30);
expect(result.maps[0].capacityObserved * 3600).toBeCloseTo(60);
});
it('clips the reporting window and splits observations across UTC chart boundaries', async () => {
const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js');
const midnight = Date.UTC(2026, 0, 2);
const sample = { server: 'one', game: 't5', map: 'Moon', players: 1, capacity: 4 };
recordMetrics([sample], midnight - 15_000);
let result = readMetrics(1, midnight + 15_000);
expect(result.points.filter(p => p.active > 0).map(p => p.active * 3600)).toEqual([15, 15]);
result = readMetrics(1, midnight + 86400_000);
expect(result.summary.active * 3600).toBeCloseTo(30);
expect(result.points.reduce((sum, p) => sum + p.active, 0)).toBeCloseTo(result.summary.active);
});
it('retains history without a live server and prunes observations older than 31 days', async () => {
const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js');
const { db } = await import('../../src/lib/server/db/index.js');
const at = Date.UTC(2026, 0, 2);
const sample = { server: 'removed', game: 't6', map: 'Town', players: 1, capacity: null };
recordMetrics([sample], at);
expect(readMetrics(1, at + 30_000).servers[0].server).toBe('removed');
recordMetrics([sample], at + 32 * 86400_000);
expect(db().prepare('SELECT COUNT(*) AS n FROM metric_samples').get()).toEqual({ n: 1 });
});
});