josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: honor Komodo health and prefer fresh telemetry across fleet status

Browse Source
This commit is contained in:
josh
2026-09-27 15:34:30 -04:00
parent 46a89ea4b0
commit 8ea696417f
27 changed files with 488 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/fleet-overview.ts
+10 -2
View File
@@ -1,4 +1,5 @@
import { parseServerName, zombieMapBySlug } from './map-aliases.js';
import { freshTelemetry, type GameObservation } from './server-observation.js';
import { freshStatusTimestamp } from './server-freshness.js';
import type { ZombieGame } from './map-aliases.js';
import type { GameTelemetry } from './telemetry.js';
@@ -6,6 +7,13 @@ import type { Health, RosterPlayer } from './ui/stores/servers.js';
import { mapTitle, serverTitle } from './ui/server-labels.js';
export interface FleetServer {
container_state?: string | null;
container_checked_at?: string | null;
last_checked_at?: string | null;
telemetry_after?: string | null;
rcon_status?: GameObservation;
telemetry?: GameTelemetry;
status_source?: 'telemetry' | 'rcon' | null;
name: string;
game: string | null;
map: string | null;
@@ -64,11 +72,11 @@ export function fleetIdentity(server: FleetServer) {
}
export function currentRound(server: FleetServer, report: GameTelemetry | undefined, now: number): number | null {
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
if (!report || (report.event !== 'round_change' && report.event !== 'heartbeat') || !freshServer(server, now) || !server.active_since || !server.game_started_at
|| server.game_ended_at || (server.last_players ?? 0) === 0) return null;
const received = Date.parse(report.receivedAt);
const age = now - received;
return age >= 0 && age <= 120_000 && received >= Date.parse(server.game_started_at)
return freshTelemetry(report, now, server.telemetry_after) && received >= Date.parse(server.game_started_at)
&& received >= Date.parse(server.active_since) ? report.round : null;
}
src/lib/server-observation.ts
+68
View File
@@ -0,0 +1,68 @@
import { freshStatusTimestamp } from './server-freshness.js';
import { parseGameEvent, type GameTelemetry } from './telemetry.js';
export interface StatusPlayer {
num: number | null;
score: number | null;
ping: number | null;
guid: string | null;
name: string;
}
export interface GameObservation {
last_players: number | null;
last_map: string | null;
last_gametype: string | null;
last_seen_at: string | null;
last_error: string | null;
roster: StatusPlayer[] | null;
}
export interface ServerObservation extends GameObservation {
name: string;
health: 'online' | 'stale' | 'offline' | 'starting' | 'provisioning';
container_state?: string | null;
container_checked_at?: string | null;
last_checked_at?: string | null;
telemetry_after?: string | null;
status_source?: 'telemetry' | 'rcon' | null;
rcon_status?: GameObservation;
telemetry?: GameTelemetry;
active_since?: string | null;
inactive_since?: string | null;
game_started_at?: string | null;
game_ended_at?: string | null;
}
export function freshTelemetry(report: GameTelemetry | undefined, now: number, after?: string | null): report is GameTelemetry {
if (!report || !freshStatusTimestamp(report.receivedAt, now)) return false;
if (after && (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))) return false;
try { parseGameEvent(report); return true; } catch { return false; }
}
/** Display policy: use one current roster/count source, keeping raw RCON for diagnostics/fallback. */
export function selectServerObservation<T extends ServerObservation>(server: T, report: GameTelemetry | undefined, now: number): T {
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
const raw = server.rcon_status ?? server;
const result: T = { ...server, ...raw, telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running') || (!server.container_state && server.health === 'offline')) {
return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null };
}
const checked = server.container_checked_at ?? server.last_checked_at ?? raw.last_seen_at;
const containerFresh = server.container_state === 'running' && freshStatusTimestamp(checked ?? null, now);
if (containerFresh && report?.server === server.name && freshTelemetry(report, now, server.telemetry_after)) {
const players = report.players.length;
return { ...result, health: 'online', status_source: 'telemetry', last_players: players,
last_map: report.map, last_seen_at: report.receivedAt, last_error: null,
roster: report.players.map(player => ({ num: null, score: 'score' in player ? player.score : null,
ping: null, guid: player.guid, name: player.name })),
active_since: players > 0 ? server.active_since ?? report.receivedAt : null,
inactive_since: players === 0 ? server.inactive_since ?? report.receivedAt : null,
game_started_at: players === 0 ? null : report.event === 'game_start' ? report.receivedAt : server.game_started_at,
game_ended_at: players === 0 || report.event === 'game_start' ? null : report.event === 'game_end' ? report.receivedAt : server.game_ended_at };
}
const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !raw.last_error;
const containerExpired = !!server.container_checked_at && !containerFresh;
const age = now - Date.parse(raw.last_seen_at ?? '');
return { ...result, health: fresh && !containerExpired ? 'online'
: after && age >= 0 && age <= 120_000 ? 'stale' : 'offline' };
}
src/lib/server/db/index.ts
+1
View File
@@ -15,6 +15,7 @@ let _db: Database.Database | null = null;
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
// backfill columns; this can.
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'container_checked_at', ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT' },
{ name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
{ name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
// Base columns that shipped in the very first schema stay here for parity
src/lib/server/db/schema.sql
+1
View File
@@ -63,6 +63,7 @@ CREATE TABLE IF NOT EXISTS servers (
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
komodo_name TEXT NOT NULL,
komodo_server TEXT,
container_checked_at TEXT,
container_state TEXT, -- last observed Komodo runtime state
desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped')),
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
src/lib/server/fleet/beta-tests.ts
+12 -8
View File
@@ -1,3 +1,5 @@
import { getServer } from './servers.js';
import { serverObservation } from './status.js';
import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
@@ -52,17 +54,19 @@ export function recordGameEvent(event: GameTelemetry): void {
export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
const m = readMetrics(run);
const server = db().prepare('SELECT last_checked_at, last_seen_at, last_error, container_state FROM servers WHERE name = ?').get(run.server) as {
last_checked_at: string | null; last_seen_at: string | null; last_error: string | null; container_state: string | null };
const sample = server.last_checked_at ? Date.parse(server.last_checked_at) : 0;
const freshSample = sample >= Date.parse(run.observing_at ?? run.created_at) && Date.now() - sample <= 60_000;
const healthy = freshSample && !server.last_error && server.container_state === 'running' && server.last_seen_at === server.last_checked_at;
const at = freshSample && server.container_state === 'running' ? sample : Date.now();
const row = getServer(run.server);
const server = row ? serverObservation(row) : null;
const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
const sample = Date.parse(timestamp ?? '');
const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
&& Date.now() - sample >= 0 && Date.now() - sample <= 45_000;
const healthy = freshSample && server?.health === 'online';
const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
const before = m.wasHealthy;
const lastSample = m.lastSample;
applySample(plan, m, at, healthy);
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: server.last_checked_at });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server.last_error });
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, source: server?.status_source });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
saveMetrics(run, m);
}
src/lib/server/fleet/collections.ts
+3 -3
View File
@@ -7,7 +7,7 @@ import { creationQueue, destroyServer, nextAvailablePort, nextLobbyNumber, queue
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
import { housePartyTargets } from './house-party.js';
import { getServerKey } from './server-keys.js';
import { classifyHealth } from './health.js';
import { serverObservation } from './status.js';
import { getServer } from './servers.js';
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
@@ -37,8 +37,8 @@ function coverage(entries: BlueprintEntry[]) {
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
: server?.container_state && !['running', 'starting', 'restarting'].includes(server.container_state) ? 'stopped'
: classifyHealth(server?.last_seen_at ?? null, server?.last_error ?? null, server?.container_state),
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
: server ? serverObservation(server).health : 'offline',
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
});
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
src/lib/server/fleet/servers.ts
+2
View File
@@ -68,6 +68,7 @@ export interface ServerRow {
komodo_name: string;
komodo_server: string | null;
container_state: string | null;
container_checked_at: string | null;
desired_state: 'running' | 'stopped';
game_port: number | null;
rcon_host: string | null;
@@ -162,6 +163,7 @@ export function setServerField(
'komodo_name',
'komodo_server',
'container_state',
'container_checked_at',
'game_port',
'rcon_host',
'game',
src/lib/server/fleet/status-evidence.ts
+29 -10
View File
@@ -1,28 +1,43 @@
import { db } from '../db/index.js';
import { parseGameEvent } from '../../telemetry.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
export interface TelemetryEvidence { players: number; map: string; receivedAt: string }
/** Only validated, recently received reports can establish current game state. */
export function telemetryEvidence(server: string, after?: string | null): TelemetryEvidence | null {
/** Reports from before the latest lifecycle submission describe the old process. */
export function latestGameSubmission(server: string): string | null {
return (db().prepare(`SELECT MAX(at) AS at FROM (
SELECT submitted_at AS at FROM server_deployments WHERE server=? AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
}
export function telemetryReport(server: string): GameTelemetry | null {
const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as
{ received_at: string; payload: string | null } | undefined;
if (!row?.payload) return null;
const received = Date.parse(row.received_at);
try {
const saved = JSON.parse(row.payload) as GameTelemetry;
const event = parseGameEvent(saved);
return event.server === server ? { ...event, receivedAt: row.received_at, timing: saved.timing } : null;
} catch { return null; }
}
/** Only validated, recently received reports can establish current game state. */
export function telemetryEvidence(server: string, after: string | null = latestGameSubmission(server)): TelemetryEvidence | null {
const report = telemetryReport(server);
if (!report) return null;
const received = Date.parse(report.receivedAt);
const age = Date.now() - received;
if (!Number.isFinite(age) || age < 0 || age > ONLINE_MAX_MS) return null;
if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after))) return null;
try {
const event = parseGameEvent(JSON.parse(row.payload));
if (event.server !== server) return null;
return { players: event.players.length, map: event.map, receivedAt: row.received_at };
} catch { return null; }
return { players: report.players.length, map: report.map, receivedAt: report.receivedAt };
}
/** Positive occupancy from either source always vetoes an empty-server claim. */
export async function probePlayerCount(server: string): Promise<number | null> {
const occupied = telemetryEvidence(server);
if (occupied && occupied.players > 0) return occupied.players;
const players = await rconGetPlayerCount(server).catch(() => null);
// Read after the network request: a player may join while RCON is pending.
const telemetry = telemetryEvidence(server);
@@ -35,10 +50,14 @@ export async function probePlayerCount(server: string): Promise<number | null> {
export async function probeServerStatus(server: string, after?: string | null): Promise<
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry' } | { ok: false; reason: string }
> {
const current = telemetryEvidence(server, after);
if (current) return { ok: true, ...current, source: 'telemetry' };
let reason = 'RCON unavailable';
try {
const rcon = await rconStatusResult(server);
if (rcon.ok) return { ok: true, players: Math.max(rcon.status.players, telemetryEvidence(server, after)?.players ?? 0), map: rcon.status.map, source: 'rcon' };
const latest = telemetryEvidence(server, after);
if (latest) return { ok: true, ...latest, source: 'telemetry' };
if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon' };
reason = rcon.reason;
} catch (err) { reason = (err as Error).message; }
const telemetry = telemetryEvidence(server, after);
src/lib/server/fleet/status.ts
+27
View File
@@ -0,0 +1,27 @@
import { selectServerObservation, type GameObservation } from '../../server-observation.js';
import { classifyHealth } from './health.js';
import { isProvisioning } from './provisioning.js';
import { restartSettling } from '../provisioning/restarts.js';
import { type ServerRow } from './servers.js';
import { telemetryReport, latestGameSubmission } from './status-evidence.js';
/** Shared snapshot for REST, status events, metrics, and other server-side readers. */
export function serverObservation(server: ServerRow) {
let roster = null;
try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
const provisioning = isProvisioning(server.name);
const state = restartSettling(server.name) ? 'restarting' : server.container_state;
const ready = !provisioning && (!state || state === 'running');
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
status_source: null as 'telemetry' | 'rcon' | null,
telemetry: telemetryReport(server.name) ?? undefined,
telemetry_after: latestGameSubmission(server.name),
last_error: ready ? server.last_error : null,
replySnippet: ready ? server.last_reply_snippet : null,
active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null,
game_started_at: ready ? server.game_started_at : null, game_ended_at: ready ? server.game_ended_at : null
}, undefined, Date.now());
}
src/lib/server/integrations/komodo.ts
+52 -75
View File
@@ -257,85 +257,62 @@ export type KomodoHealthResult =
| { ok: true; state: string }
| { ok: false; reason: string; state?: string; snippet?: string };
// Shape-agnostic scan for the container's running-state. Walks the entire
// payload looking for anything that resembles a state field. Covered shapes:
// - Docker inspect: { State: { Status: "running", Running: true, Health: { Status: "healthy" } } }
// - Komodo GetDeployment: top-level or nested `state`, `Status`, `container_state`
// - Komodo GetDeploymentStatus: { state: "Running" }
// - Anything with `Running: true` at any depth
// When it can't find any of these, callers get the raw JSON in the error so
// we can spot which field to add here.
const STATE_KEYS = new Set([
'state',
'State',
'status',
'Status',
'container_state',
'containerState',
'deployment_state',
'deploymentState',
'docker_status'
]);
// Komodo's container summary has { state: "running", container: { status:
// "Up 35 seconds (health: starting)" } }. Inspect responses instead have
// State.Health.Status (or snake/lowercase equivalents). Runtime health wins
// over a coarse running flag. Unknown/configuration strings are not readiness.
export function extractContainerState(payload: unknown): string | null {
const seen = new Set<unknown>();
let found: string | null = null;
const walk = (v: unknown): void => {
if (found || v === null || typeof v !== 'object' || seen.has(v)) return;
seen.add(v);
if (Array.isArray(v)) {
for (const item of v) walk(item);
return;
const known = new Set(['running','healthy','starting','restarting','unhealthy','stopped','exited','created','dead','paused','removing','notdeployed','unknown']);
const stateOf = (value: unknown): string | null => {
if (typeof value !== 'string') return null;
const state = value.toLowerCase().trim();
if (known.has(state)) return state;
if (/^up\b/.test(state)) {
const health = state.match(/\((?:health:\s*)?(starting|unhealthy|healthy)\)/)?.[1];
return health && health !== 'healthy' ? health : 'running';
}
const obj = v as Record<string, unknown>;
// Komodo summaries expose health alongside a flat running state.
const flatState = obj['state'] ?? obj['State'] ?? obj['status'] ?? obj['Status'];
const flatHealth = obj['health'] ?? obj['Health'];
const healthStatus = typeof flatHealth === 'string' ? flatHealth
: flatHealth && typeof flatHealth === 'object'
? ((flatHealth as Record<string, unknown>)['status'] ?? (flatHealth as Record<string, unknown>)['Status']) : null;
if (typeof flatState === 'string' && flatState.toLowerCase() === 'running'
&& typeof healthStatus === 'string' && healthStatus.toLowerCase() !== 'healthy') {
found = healthStatus;
return;
}
// Docker inspect nested: { State: { Status, Running, Health: { Status } } }
const stateObj = obj['State'];
if (stateObj && typeof stateObj === 'object') {
const so = stateObj as Record<string, unknown>;
const status = so['Status'];
const running = so['Running'];
const health = so['Health'];
// A running process can still be starting or unhealthy. Honor an
// explicit Docker health check before accepting its running state.
if ((running === true || status === 'running') && health && typeof health === 'object') {
const hs = (health as Record<string, unknown>)['Status'];
if (typeof hs === 'string' && hs.toLowerCase() !== 'healthy') { found = hs; return; }
}
if (typeof status === 'string') { found = status; return; }
if (running === true) { found = 'running'; return; }
if (health && typeof health === 'object') {
const hs = (health as Record<string, unknown>)['Status'];
if (typeof hs === 'string') { found = hs; return; }
}
}
// Flat string-valued state under any of the recognized keys.
for (const key of STATE_KEYS) {
const val = obj[key];
if (typeof val === 'string' && val.length > 0) {
found = val;
return;
}
}
for (const val of Object.values(obj)) walk(val);
if (/^exited\b/.test(state)) return 'exited';
if (/^restarting\b/.test(state)) return 'restarting';
return null;
};
walk(payload);
return found;
const walk = (value: unknown): string | null => {
if (!value || typeof value !== 'object' || seen.has(value)) return null;
seen.add(value);
if (Array.isArray(value)) {
const states = value.map(walk).filter((state): state is string => !!state);
return states.find(state => !['running','healthy'].includes(state)) ?? states[0] ?? null;
}
const object = value as Record<string, unknown>;
const rawState = object.State ?? object.state;
const state = stateOf(rawState) ?? stateOf(object.container_state ?? object.containerState ?? object.deployment_state ?? object.deploymentState);
// A stopped process is never ready just because its previous health check passed.
if (state && !['running','healthy','unknown'].includes(state)) return state;
if (rawState && typeof rawState === 'object') {
const nested = walk(rawState);
if (nested) return nested;
}
for (const key of ['container','Container','containers','Containers']) {
if (object[key] === null && state === 'running') return 'unknown';
const nested = walk(object[key]);
if (nested) return nested;
}
const status = stateOf(object.Status ?? object.status ?? object.docker_status);
if (status && !['running','healthy'].includes(status)) return status;
const health = object.Health ?? object.health;
const healthState = stateOf(typeof health === 'object' && health ?
(health as Record<string, unknown>).Status ?? (health as Record<string, unknown>).status : health);
if (healthState && healthState !== 'healthy') return healthState;
if (state || status) return state ?? status;
if (object.Running === true || object.running === true) return healthState ?? 'running';
for (const [key, item] of Object.entries(object)) {
if (['config','Config','labels','Labels'].includes(key)) continue;
const nested = walk(item);
if (nested) return nested;
}
return null;
};
return walk(payload);
}
const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
src/lib/server/pipeline/bus.ts
+8 -2
View File
@@ -1,6 +1,7 @@
import type { StatusPlayer, GameObservation } from '../../server-observation.js';
import { EventEmitter } from 'node:events';
import type { GameTelemetry } from '../../telemetry.js';
import type { RconEndpoint, RconFailure, RconPlayer } from '../integrations/rcon.js';
import type { RconEndpoint, RconFailure } from '../integrations/rcon.js';
import type { Health } from '../fleet/health.js';
// One in-process event bus, multiplexed by the SSE endpoints. Kept separate
@@ -9,10 +10,15 @@ import type { Health } from '../fleet/health.js';
export interface ServerStatusEntry {
server: string;
containerState?: string | null;
containerCheckedAt?: string | null;
telemetryAfter?: string | null;
rconStatus?: GameObservation;
telemetry?: GameTelemetry;
players: number | null;
map: string | null;
gametype: string | null;
roster: RconPlayer[] | null;
roster: StatusPlayer[] | null;
health: Health;
lastSeenAt: string | null;
lastError: string | null;
src/lib/server/pipeline/worker.ts
+61 -142
View File
@@ -1,3 +1,4 @@
import { config } from '../config.js';
import { processProvisioning } from '../provisioning/worker.js';
import { sweep } from './sweep.js';
import { betaTick } from './beta-runner.js';
@@ -8,7 +9,8 @@ import type { RconFailure } from '../integrations/rcon.js';
import { getServer, listServers, setServerField } from '../fleet/servers.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { restartSettling } from '../provisioning/index.js';
import { classifyHealth } from '../fleet/health.js';
import { serverObservation } from '../fleet/status.js';
import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
import { isProvisioning } from '../fleet/provisioning.js';
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
import { recordMetrics } from '../fleet/metrics.js';
@@ -68,167 +70,84 @@ export function fanoutStatus(): Promise<void> {
async function refreshStatus(): Promise<void> {
const servers = listServers();
if (servers.length === 0) return;
if (!servers.length) return;
const entries: ServerStatusEntry[] = new Array(servers.length);
const capacities = new Map<string, number | null>();
let cursor = 0;
async function pump(): Promise<void> {
while (true) {
async function pump() {
while (cursor < servers.length) {
const i = cursor++;
if (i >= servers.length) return;
const srv = servers[i];
// Creation performs its own container, RCON, and map verification.
if (isProvisioning(srv.name)) {
if (!isProvisioning(srv.name)) {
const epoch = latestGameSubmission(srv.name);
const container = restartSettling(srv.name)
? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
: await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
// A lifecycle request may be submitted while an inspection is in flight.
if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) {
setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown');
setServerField(srv.name, 'container_checked_at', new Date().toISOString());
if (!container.ok) {
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
} else if (!telemetryEvidence(srv.name)) {
const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) {
const now = new Date().toISOString();
setServerField(srv.name, 'last_checked_at', now);
setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null);
if (result.ok) {
capacities.set(srv.name, result.status.capacity ?? null);
setServerField(srv.name, 'last_players', result.status.players);
setServerField(srv.name, 'last_map', result.status.map);
setServerField(srv.name, 'last_gametype', result.status.gametype);
setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
setServerField(srv.name, 'last_seen_at', now);
setServerField(srv.name, 'last_error', null);
} else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail));
}
}
}
} else {
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
entries[i] = {
server: srv.name, players: null, map: srv.map, gametype: null,
roster: null, health: 'provisioning', lastSeenAt: srv.last_seen_at,
lastError: null, endpoint: null, reason: null, replySnippet: null,
activeSince: null, inactiveSince: null, gameStartedAt: null, gameEndedAt: null
};
continue;
}
const container = restartSettling(srv.name)
? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
: await komodoInspectHealthy(srv.name).catch((err) => ({
ok: false as const, reason: (err as Error).message
}));
const containerState = container.ok ? 'running'
: ('state' in container && typeof container.state === 'string' ? container.state.toLowerCase() : 'unknown');
setServerField(srv.name, 'container_state', containerState);
if (!container.ok) {
// Container readiness precedes RCON readiness. Old replies and
// failures describe the previous process, not this startup.
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
entries[i] = {
server: srv.name, players: null, map: srv.map, gametype: null,
roster: null, health: classifyHealth(null, null, containerState),
lastSeenAt: srv.last_seen_at, lastError: null, endpoint: null,
reason: null, replySnippet: null, activeSince: null,
inactiveSince: null, gameStartedAt: null, gameEndedAt: null
};
continue;
}
const result = await rconStatusResult(srv.name).catch((err) => ({
ok: false as const,
endpoint: null,
reason: 'send-error' as RconFailure,
detail: (err as Error).message,
rawReply: undefined as string | undefined
}));
const now = new Date().toISOString();
if (result.ok) {
capacities.set(srv.name, result.status.capacity ?? null);
// Compare against the previously persisted count BEFORE we
// overwrite last_players — occupancy transitions are anchored
// to that boundary.
const prevPlayers = srv.last_players;
const nextPlayers = result.status.players;
let activeSince = srv.active_since;
let inactiveSince = srv.inactive_since;
// Telemetry can arrive while the RCON request is in flight.
const current = getServer(srv.name) ?? srv;
const gameStartedAt = nextPlayers === 0 ? null : current.game_started_at;
const gameEndedAt = nextPlayers === 0 ? null : current.game_ended_at;
if (nextPlayers === 0) {
const current = getServer(srv.name) ?? srv;
const status = serverObservation(current);
// Occupancy timers and match clearing follow the same source as the UI.
if (status.health === 'online' && status.last_players !== null) {
if (status.last_players > 0) {
setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at);
setServerField(srv.name, 'inactive_since', null);
} else {
setServerField(srv.name, 'active_since', null);
setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at);
setServerField(srv.name, 'game_started_at', null);
setServerField(srv.name, 'game_ended_at', null);
}
if (nextPlayers > 0 && (activeSince === null || prevPlayers === 0 || prevPlayers === null)) {
activeSince = now;
inactiveSince = null;
setServerField(srv.name, 'active_since', activeSince);
setServerField(srv.name, 'inactive_since', null);
} else if (nextPlayers === 0 && (inactiveSince === null || (prevPlayers !== null && prevPlayers > 0))) {
inactiveSince = now;
activeSince = null;
setServerField(srv.name, 'inactive_since', inactiveSince);
setServerField(srv.name, 'active_since', null);
}
// Persist last-observed state. last_seen_at is the health anchor.
setServerField(srv.name, 'last_players', nextPlayers);
setServerField(srv.name, 'last_map', result.status.map);
setServerField(srv.name, 'last_gametype', result.status.gametype);
setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
setServerField(srv.name, 'last_seen_at', now);
setServerField(srv.name, 'last_checked_at', now);
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', result.rawReply);
entries[i] = {
server: srv.name,
players: nextPlayers,
map: result.status.map,
gametype: result.status.gametype,
roster: result.status.roster,
health: classifyHealth(now, null),
lastSeenAt: now,
lastError: null,
endpoint: result.endpoint,
reason: null,
replySnippet: result.rawReply,
activeSince,
inactiveSince,
gameStartedAt,
gameEndedAt
};
} else {
// RCON unreachable — leave last_seen_at, active_since, and
// inactive_since alone (health decays to offline naturally;
// occupancy timers survive transient outages). Record the
// structured failure.
const encoded = encodeError(result.reason, result.detail);
const snippet = result.rawReply ?? null;
setServerField(srv.name, 'last_checked_at', now);
setServerField(srv.name, 'last_error', encoded);
setServerField(srv.name, 'last_reply_snippet', snippet);
entries[i] = {
server: srv.name,
players: srv.last_players,
map: srv.last_map,
gametype: srv.last_gametype,
roster: srv.last_roster ? safeJson(srv.last_roster) : null,
health: classifyHealth(srv.last_seen_at, encoded),
lastSeenAt: srv.last_seen_at,
lastError: encoded,
endpoint: result.endpoint,
reason: result.reason,
replySnippet: snippet,
activeSince: srv.active_since,
inactiveSince: srv.inactive_since,
gameStartedAt: srv.game_started_at,
gameEndedAt: srv.game_ended_at
};
}
const final = serverObservation(getServer(srv.name) ?? current);
entries[i] = { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map,
gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at,
lastError: final.last_error, replySnippet: final.replySnippet,
endpoint: current.game_port ? { host: current.rcon_host ?? config.rcon.host, port: current.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null,
activeSince: final.active_since, inactiveSince: final.inactive_since,
gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at,
containerState: final.container_state, containerCheckedAt: final.container_checked_at,
telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry };
}
}
await Promise.all(Array.from({ length: STATUS_CONCURRENCY }, pump));
await Promise.all(Array.from({ length: Math.min(STATUS_CONCURRENCY, servers.length) }, pump));
try {
recordMetrics(servers.map((srv, i) => ({
server: srv.name,
game: srv.game ?? 'unknown',
recordMetrics(servers.map((srv, i) => ({ server: srv.name, game: srv.game ?? 'unknown',
map: selectedZombieMap(srv.game, entries[i].map ?? srv.map, srv.map_slug)?.friendly ?? entries[i].map ?? srv.map ?? 'Unknown map',
players: capacities.has(srv.name) ? entries[i].players : null,
capacity: capacities.get(srv.name) ?? null
})));
players: entries[i].health === 'online' ? entries[i].players : null,
capacity: capacities.get(srv.name) ?? null })));
} catch (err) { console.error('[worker] metrics error:', err); }
emit({ type: 'server-status', entries });
}
function safeJson<T>(raw: string): T | null {
try {
return JSON.parse(raw) as T;
} catch {
return null;
}
}
export function startWorker(): void {
if (started) return;
started = true;
src/lib/ui/components/PlayerScoreboard.svelte
+8 -5
View File
@@ -1,6 +1,7 @@
<script lang="ts">
import { timeFormat } from '$ui/stores/time-zone.js';
import type { GameTelemetry } from '$lib/telemetry.js';
import { freshTelemetry } from '$lib/server-observation.js';
import { uiNow } from '$ui/stores/servers.js';
import { elapsedSince } from '$ui/duration.js';
import { mapTitle } from '$ui/server-labels.js';
@@ -9,8 +10,9 @@
import Badge from './primitives/Badge.svelte';
import EmptyState from './primitives/EmptyState.svelte';
let { report, mapLabel, game }: { report: GameTelemetry | undefined; mapLabel?: string; game: string } = $props();
const timingNow = $derived(report && (report.event === 'game_end' || $uiNow - Date.parse(report.receivedAt) > 120_000)
let { report, mapLabel, game, current = true, after = null }: { report: GameTelemetry | undefined; mapLabel?: string; game: string; current?: boolean; after?: string | null } = $props();
const fresh = $derived(current && freshTelemetry(report, $uiNow, after));
const timingNow = $derived(report && (report.event === 'game_end' || !fresh)
? Date.parse(report.receivedAt) : $uiNow);
const columns = [
{ key: 'score', label: 'Score' },
@@ -38,11 +40,11 @@
</script>
{#if !report}
<EmptyState title="Waiting for game data" description="The scoreboard will appear when this server reports a game start, round change, or game end." />
<EmptyState title="Waiting for game data" description="The scoreboard will appear when this server reports a game start, round change, heartbeat, or game end." />
{:else}
<header class="scoreboard-header">
<div class="match">
<span class="eyebrow">{report.event === 'game_end' ? 'Final scoreboard' : 'Latest scoreboard'}</span>
<span class="eyebrow">{report.event === 'game_end' ? 'Final scoreboard' : fresh ? 'Live scoreboard' : 'Last reported scoreboard'}</span>
<div class="match-title">
<h2 title={report.map}>{mapLabel ?? mapTitle(report.map)}</h2>
{#if report.event !== 'game_start'}<Badge variant="accent" size="md">Round {report.round}</Badge>{/if}
@@ -51,7 +53,8 @@
</div>
</div>
<div class="report-meta">
<span>{players.length} {players.length === 1 ? 'player' : 'players'}</span>
<span>{players.length} {players.length === 1 ? 'player' : 'players'} in this report</span>
{#if !fresh}<Badge variant="warning">Historical report</Badge>{/if}
<time datetime={report.receivedAt} title={$timeFormat.dateTime(report.receivedAt)}>
Updated {elapsedSince(report.receivedAt, $uiNow)} ago
</time>
src/lib/ui/components/RoundTelemetry.svelte
+6 -5
View File
@@ -1,18 +1,19 @@
<script lang="ts">
import { timeFormat } from '$ui/stores/time-zone.js';
import { telemetryByServer, uiNow } from '$ui/stores/servers.js';
import { freshStatusTimestamp } from '$lib/server-freshness.js';
import { elapsedSince } from '$ui/duration.js';
let { name, since = null }: { name: string; since?: string | null } = $props();
let report = $derived($telemetryByServer.get(name));
let fresh = $derived(report && $uiNow - Date.parse(report.receivedAt) >= 0
&& $uiNow - Date.parse(report.receivedAt) <= 120_000
import type { GameTelemetry } from '$lib/telemetry.js';
let { name, since = null, report: observation }: { name: string; since?: string | null; report?: GameTelemetry } = $props();
let report = $derived(observation ?? $telemetryByServer.get(name));
let fresh = $derived(report && freshStatusTimestamp(report.receivedAt, $uiNow)
&& (!since || Date.parse(report.receivedAt) >= Date.parse(since)));
</script>
{#if report && (report.event === 'round_change' || report.event === 'heartbeat') && fresh}
<div class="round" aria-live="polite">
<strong>Round {report.round}</strong>
<span title={$timeFormat.dateTime(report.receivedAt)}>Last reported {elapsedSince(report.receivedAt, $uiNow)} ago · {report.players.length} players</span>
<span title={$timeFormat.dateTime(report.receivedAt)}>Last reported {elapsedSince(report.receivedAt, $uiNow)} ago</span>
</div>
{/if}
src/lib/ui/components/ServerCard.svelte
+6 -4
View File
@@ -16,6 +16,7 @@
betaId: number;
}
interface Props {
telemetry?: import('$lib/telemetry.js').GameTelemetry;
name: string;
players?: number | null;
map?: string | null;
@@ -35,6 +36,7 @@
mapLabelOverride?: string;
}
let {
telemetry,
name,
players = null,
map = null,
@@ -66,7 +68,7 @@
let topRoster = $derived(
(roster ?? [])
.slice()
.sort((a, b) => b.score - a.score)
.sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
.slice(0, 4)
);
let showRoster = $derived(health === 'online' || health === 'stale');
@@ -134,7 +136,7 @@
{#if topRoster.length}<div class="roster-heading">Last reported players</div>{/if}
{/if}
{#if health === 'online' && players !== null && players > 0 && gameInProgress}<RoundTelemetry {name} since={gameStartedAt} />{/if}
{#if health === 'online' && players !== null && players > 0 && gameInProgress}<RoundTelemetry {name} report={telemetry} since={gameStartedAt} />{/if}
{#if showTiming && !(compactEmpty && players === 0)}
<div class="timing">
@@ -169,8 +171,8 @@
{#each topRoster as p (p.guid ?? p.name)}
<li>
<span class="pname">{p.name || '—'}</span>
<span class="pscore">{p.score}</span>
<span class="pping">{p.ping}ms</span>
<span class="pscore">{p.score ?? '—'}</span>
<span class="pping">{p.ping === null ? '—' : `${p.ping}ms`}</span>
</li>
{/each}
</ul>
src/lib/ui/stores/fleet.ts
+12 -5
View File
@@ -1,6 +1,7 @@
import { selectServerObservation } from '$lib/server-observation.js';
import { derived, writable } from 'svelte/store';
import { browser } from '$app/environment';
import { statusByServer, statusReceivedAt, syncServerTime, matchEventsByServer } from './servers.js';
import { statusByServer, statusReceivedAt, syncServerTime, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js';
export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
@@ -10,18 +11,24 @@ export const fleetError = writable('');
export const fleetRefreshing = writable(false);
export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer], ([snapshot, live, received, matches]) =>
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, uiNow], ([snapshot, live, received, matches, reports, now]) =>
snapshot.servers.map(server => {
const status = received > snapshot.receivedAt ? live.get(server.name) : undefined;
const current = status ? { ...server, last_players: status.players, last_map: status.map,
const base = status ? { ...server, last_players: status.players, last_map: status.map,
last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
inactive_since: status.inactiveSince, game_started_at: status.gameStartedAt,
game_ended_at: status.gameEndedAt } : server;
game_ended_at: status.gameEndedAt, container_state: status.containerState ?? server.container_state,
container_checked_at: status.containerCheckedAt ?? server.container_checked_at,
telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus,
telemetry: status.telemetry ?? server.telemetry } : server;
const current = selectServerObservation(base, reports.get(server.name), now);
const match = matches.get(server.name);
// Apply lifecycle events immediately, until a newer status observes them
// or confirms that the lobby has emptied.
if (!match || Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
if (!match || current.health !== 'online'
|| (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after))
|| Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
|| match.receivedAt < (current.game_started_at ?? '')
|| match.receivedAt < (current.game_ended_at ?? '')) return current;
return { ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
src/lib/ui/stores/servers.ts
+7 -7
View File
@@ -5,13 +5,8 @@ import type { GameTelemetry } from '$lib/telemetry.js';
export const telemetryByServer = writable<Map<string, GameTelemetry>>(new Map());
export const matchEventsByServer = writable<Map<string, GameTelemetry>>(new Map());
export interface RosterPlayer {
num: number | null;
score: number;
ping: number;
guid: string | null;
name: string;
}
export type { StatusPlayer as RosterPlayer } from '$lib/server-observation.js';
import type { StatusPlayer as RosterPlayer, GameObservation } from '$lib/server-observation.js';
export type Health = 'online' | 'stale' | 'offline' | 'starting' | 'provisioning';
export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
@@ -23,6 +18,11 @@ export interface RconEndpoint {
export interface ServerStatus {
server: string;
containerState?: string | null;
containerCheckedAt?: string | null;
telemetryAfter?: string | null;
rconStatus?: GameObservation;
telemetry?: GameTelemetry;
players: number | null;
map: string | null;
gametype: string | null;