fix: honor Komodo health and prefer fresh telemetry across fleet status
This commit is contained in:
@@ -8,10 +8,20 @@ Existing database registrations are preserved, and missing mounts do not remove
|
|||||||
servers from the fleet. The legacy `seedServerRegistry` helper is an explicit
|
servers from the fleet. The legacy `seedServerRegistry` helper is an explicit
|
||||||
import utility only.
|
import utility only.
|
||||||
|
|
||||||
Status refreshes check Komodo container readiness before polling RCON. Starting
|
Status refreshes check Komodo container readiness, including nested Docker health
|
||||||
or restarting containers show **Starting**, without old RCON errors or player
|
status. A running container with `health: starting` is still **Starting**. Starting
|
||||||
data. Stopped, unhealthy, or unverifiable containers are not polled. Containers
|
or restarting containers hide old game data and RCON errors; stopped, unhealthy,
|
||||||
without a Docker health check use their running state as the readiness signal.
|
or unverifiable containers do not trigger routine RCON polling. Containers without
|
||||||
|
a Docker health check use their running state as the readiness signal.
|
||||||
|
|
||||||
|
With fresh container health, validated telemetry received in the last 45 seconds
|
||||||
|
supplies the current player count, roster, map, and game status. RCON is the fallback
|
||||||
|
when telemetry is unavailable or stale. Fleet, server details, metrics, collection
|
||||||
|
coverage, setup readiness, and beta health checks share this policy. Reports from
|
||||||
|
before the latest start, restart, or rebuild cannot verify the new process.
|
||||||
|
Historical scoreboards remain visible and are labeled separately from live counts;
|
||||||
|
RCON diagnostics retain their own results. Telemetry does not report ping or player
|
||||||
|
capacity, so those fields remain unknown in telemetry-only observations.
|
||||||
|
|
||||||
## Fleet overview
|
## Fleet overview
|
||||||
|
|
||||||
@@ -33,7 +43,7 @@ status between refreshes. Failed refreshes keep the last snapshot visible.
|
|||||||
- SQLite via better-sqlite3 (replaces `script_metadata.json` as the source of
|
- SQLite via better-sqlite3 (replaces `script_metadata.json` as the source of
|
||||||
truth)
|
truth)
|
||||||
- Server-sent events for live pipeline / server-status streams
|
- Server-sent events for live pipeline / server-status streams
|
||||||
- RCON (UDP) direct-to-game-server for player counts
|
- Game telemetry for current player counts and rosters, with direct UDP RCON as fallback
|
||||||
- Komodo API for restarts and per-server game-port discovery
|
- Komodo API for restarts and per-server game-port discovery
|
||||||
- Runs as a single Komodo-managed container on the same host as the game
|
- Runs as a single Komodo-managed container on the same host as the game
|
||||||
servers, with `/root/zombies` bind-mounted in
|
servers, with `/root/zombies` bind-mounted in
|
||||||
@@ -83,7 +93,7 @@ does not cancel work already queued; use **Cancel** for those jobs.
|
|||||||
|
|
||||||
Maintenance waits for a fresh zero-player RCON or telemetry report and for existing
|
Maintenance waits for a fresh zero-player RCON or telemetry report and for existing
|
||||||
deployment/beta work, then uses the shared rolling restart limit. Unreachable or
|
deployment/beta work, then uses the shared rolling restart limit. Unreachable or
|
||||||
occupied servers remain queued. After container and RCON recovery, Electric
|
occupied servers remain queued. After container and game-status recovery, Electric
|
||||||
Cherry queries the running revision again. If a fresh published revision was
|
Cherry queries the running revision again. If a fresh published revision was
|
||||||
known when queued, recovery on an older or unknown revision is reported as a
|
known when queued, recovery on an older or unknown revision is reported as a
|
||||||
failed verification, without an automatic restart loop. Ambiguous restart
|
failed verification, without an automatic restart loop. Ambiguous restart
|
||||||
@@ -123,7 +133,7 @@ normally run in submission order; on a shared server, beta activation precedes
|
|||||||
campaign activation, and maintenance waits for both. Busy
|
campaign activation, and maintenance waits for both. Busy
|
||||||
or unreachable servers stay queued without taking a slot, so other empty
|
or unreachable servers stay queued without taking a slot, so other empty
|
||||||
servers can proceed. A successful Komodo restart request retains its slot
|
servers can proceed. A successful Komodo restart request retains its slot
|
||||||
until the container is healthy and RCON answers. Health checks start no sooner
|
until the container is healthy and RCON or fresh telemetry verifies game status. Health checks start no sooner
|
||||||
than 15 seconds after the request returns, avoiding an immediate check of the
|
than 15 seconds after the request returns, avoiding an immediate check of the
|
||||||
old process. An unhealthy server keeps its slot rather than letting failures
|
old process. An unhealthy server keeps its slot rather than letting failures
|
||||||
spread across the fleet. A timed-out restart also keeps its slot because Komodo
|
spread across the fleet. A timed-out restart also keeps its slot because Komodo
|
||||||
@@ -133,7 +143,7 @@ Slots persist in SQLite across app restarts, and overlapping timer/API triggers
|
|||||||
share the active run. Beta options to skip health or match gates still work;
|
share the active run. Beta options to skip health or match gates still work;
|
||||||
skipping a beta gate does not release the fleet's recovery slot early. New-server
|
skipping a beta gate does not release the fleet's recovery slot early. New-server
|
||||||
creation uses a separate shared startup queue, holding its slot through container,
|
creation uses a separate shared startup queue, holding its slot through container,
|
||||||
RCON, and selected-map verification.
|
game-status, and selected-map verification.
|
||||||
|
|
||||||
No token, no session, no auth freeze. If Komodo or RCON aren't configured,
|
No token, no session, no auth freeze. If Komodo or RCON aren't configured,
|
||||||
the sweep marks pending rows `manual` so nothing hangs forever.
|
the sweep marks pending rows `manual` so nothing hangs forever.
|
||||||
@@ -167,8 +177,8 @@ page tracks both actions. **Fill missing servers** remains the default.
|
|||||||
All jobs and capacity reservations are committed together before provisioning;
|
All jobs and capacity reservations are committed together before provisioning;
|
||||||
one collection page tracks each server's independent progress and failures.
|
one collection page tracks each server's independent progress and failures.
|
||||||
New servers start one at a time by default across all collections and individual
|
New servers start one at a time by default across all collections and individual
|
||||||
launches. A startup slot stays occupied until Komodo and RCON are healthy and
|
launches. A startup slot stays occupied until Komodo is healthy and RCON or
|
||||||
RCON reports the selected map; an unready server holds the queue until it recovers
|
fresh telemetry reports the selected map; an unready server holds the queue until it recovers
|
||||||
or is removed. The queue survives application restarts. The collection preview
|
or is removed. The queue survives application restarts. The collection preview
|
||||||
shows the startup limit, and its progress page shows active startups and each
|
shows the startup limit, and its progress page shows active startups and each
|
||||||
server's place in the shared queue. Set `DEPLOYMENT_CREATE_CONCURRENCY` to a
|
server's place in the shared queue. Set `DEPLOYMENT_CREATE_CONCURRENCY` to a
|
||||||
@@ -274,7 +284,7 @@ Rotations use `execgts <ruleset>.cfg map <engine-map>`; Plutainer seeds the requ
|
|||||||
baseline. Town, Farm, and Bus Depot share `zm_transit`, so EC keeps their selection
|
baseline. Town, Farm, and Bus Depot share `zm_transit`, so EC keeps their selection
|
||||||
in `map_slug` through creation, beta, promotion, and drift repair. Map-scoped
|
in `map_slug` through creation, beta, promotion, and drift repair. Map-scoped
|
||||||
artifacts target the engine map and therefore apply to all its variants.
|
artifacts target the engine map and therefore apply to all its variants.
|
||||||
Creation health checks verify the engine map reported by RCON; this does not
|
Creation health checks verify the engine map reported by RCON or fresh telemetry; this does not
|
||||||
independently verify the start location within that map.
|
independently verify the start location within that map.
|
||||||
|
|
||||||
Both games share the fleet's port allocator (`SERVER_PORT_MIN`/`SERVER_PORT_MAX`)
|
Both games share the fleet's port allocator (`SERVER_PORT_MIN`/`SERVER_PORT_MAX`)
|
||||||
@@ -365,9 +375,9 @@ RCON parsing (fake UDP socket).
|
|||||||
|
|
||||||
Campaigns (`/campaigns`) tracks script/config updates, drift reconciliation, restart progress, file diffs, and rollbacks. Existing campaign records are preserved in the original database tables. The campaign API lives at `/api/campaigns`; legacy detail API endpoints remain available for compatibility.
|
Campaigns (`/campaigns`) tracks script/config updates, drift reconciliation, restart progress, file diffs, and rollbacks. Existing campaign records are preserved in the original database tables. The campaign API lives at `/api/campaigns`; legacy detail API endpoints remain available for compatibility.
|
||||||
|
|
||||||
Provisioning (`/provisioning`, `/api/provisioning`) owns Create, Start, Restart, Rebuild, Stop, and Destroy: request persistence, eligibility and occupancy checks, execution, recovery, and history. Campaigns, betas, and scheduled maintenance request lifecycle work through Provisioning and link to the resulting operation. They retain their file, test-evidence, and version-verification workflows. Pending automated restarts appear in Provisioning before execution. History survives server removal. New creation requests return HTTP 202 with an operation ID and open the provisioning detail page. Creation is complete only when Komodo reports healthy, RCON responds successfully, and the RCON map exactly matches the selected map. Queued requests reserve their server name and port; queued creation and pending health checks survive application restarts. The detail page records startup progress and explains any health or map mismatch that is still blocking completion.
|
Provisioning (`/provisioning`, `/api/provisioning`) owns Create, Start, Restart, Rebuild, Stop, and Destroy: request persistence, eligibility and occupancy checks, execution, recovery, and history. Campaigns, betas, and scheduled maintenance request lifecycle work through Provisioning and link to the resulting operation. They retain their file, test-evidence, and version-verification workflows. Pending automated restarts appear in Provisioning before execution. History survives server removal. New creation requests return HTTP 202 with an operation ID and open the provisioning detail page. Creation is complete only when Komodo reports healthy and RCON or fresh telemetry verifies game status with an engine map that exactly matches the selected map. Queued requests reserve their server name and port; queued creation and pending health checks survive application restarts. The detail page records startup progress and explains any health or map mismatch that is still blocking completion.
|
||||||
|
|
||||||
Each server deployment opens a detail page with its flow and timestamped activity. Destroy uses an in-app confirmation modal and immediately returns a queued deployment (`DELETE /api/servers/:name` returns HTTP 202 with `deploymentId`). The worker waits for a fresh RCON player count of zero before teardown; occupied or unreachable servers remain queued and are checked each worker tick. Waiting requests survive application restarts, while interrupted teardown is marked for inspection. Completed destruction stays on the provisioning detail page.
|
Each server deployment opens a detail page with its flow and timestamped activity. Destroy uses an in-app confirmation modal and immediately returns a queued deployment (`DELETE /api/servers/:name` returns HTTP 202 with `deploymentId`). The worker waits for a fresh zero-player RCON or telemetry report, or a verified stopped container, before teardown. Occupied reports from either source block teardown; servers without current verification remain queued and are checked each worker tick. Waiting requests survive application restarts, while interrupted teardown is marked for inspection. Completed destruction stays on the provisioning detail page.
|
||||||
|
|
||||||
|
|
||||||
Use **Provisioning → Manage server**, or a server’s **Server actions**, to start,
|
Use **Provisioning → Manage server**, or a server’s **Server actions**, to start,
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { parseServerName, zombieMapBySlug } from './map-aliases.js';
|
import { parseServerName, zombieMapBySlug } from './map-aliases.js';
|
||||||
|
import { freshTelemetry, type GameObservation } from './server-observation.js';
|
||||||
import { freshStatusTimestamp } from './server-freshness.js';
|
import { freshStatusTimestamp } from './server-freshness.js';
|
||||||
import type { ZombieGame } from './map-aliases.js';
|
import type { ZombieGame } from './map-aliases.js';
|
||||||
import type { GameTelemetry } from './telemetry.js';
|
import type { GameTelemetry } from './telemetry.js';
|
||||||
@@ -6,6 +7,13 @@ import type { Health, RosterPlayer } from './ui/stores/servers.js';
|
|||||||
import { mapTitle, serverTitle } from './ui/server-labels.js';
|
import { mapTitle, serverTitle } from './ui/server-labels.js';
|
||||||
|
|
||||||
export interface FleetServer {
|
export interface FleetServer {
|
||||||
|
container_state?: string | null;
|
||||||
|
container_checked_at?: string | null;
|
||||||
|
last_checked_at?: string | null;
|
||||||
|
telemetry_after?: string | null;
|
||||||
|
rcon_status?: GameObservation;
|
||||||
|
telemetry?: GameTelemetry;
|
||||||
|
status_source?: 'telemetry' | 'rcon' | null;
|
||||||
name: string;
|
name: string;
|
||||||
game: string | null;
|
game: string | null;
|
||||||
map: string | null;
|
map: string | null;
|
||||||
@@ -64,11 +72,11 @@ export function fleetIdentity(server: FleetServer) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function currentRound(server: FleetServer, report: GameTelemetry | undefined, now: number): number | null {
|
export function currentRound(server: FleetServer, report: GameTelemetry | undefined, now: number): number | null {
|
||||||
|
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
|
||||||
if (!report || (report.event !== 'round_change' && report.event !== 'heartbeat') || !freshServer(server, now) || !server.active_since || !server.game_started_at
|
if (!report || (report.event !== 'round_change' && report.event !== 'heartbeat') || !freshServer(server, now) || !server.active_since || !server.game_started_at
|
||||||
|| server.game_ended_at || (server.last_players ?? 0) === 0) return null;
|
|| server.game_ended_at || (server.last_players ?? 0) === 0) return null;
|
||||||
const received = Date.parse(report.receivedAt);
|
const received = Date.parse(report.receivedAt);
|
||||||
const age = now - received;
|
return freshTelemetry(report, now, server.telemetry_after) && received >= Date.parse(server.game_started_at)
|
||||||
return age >= 0 && age <= 120_000 && received >= Date.parse(server.game_started_at)
|
|
||||||
&& received >= Date.parse(server.active_since) ? report.round : null;
|
&& received >= Date.parse(server.active_since) ? report.round : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
import { freshStatusTimestamp } from './server-freshness.js';
|
||||||
|
import { parseGameEvent, type GameTelemetry } from './telemetry.js';
|
||||||
|
|
||||||
|
export interface StatusPlayer {
|
||||||
|
num: number | null;
|
||||||
|
score: number | null;
|
||||||
|
ping: number | null;
|
||||||
|
guid: string | null;
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
export interface GameObservation {
|
||||||
|
last_players: number | null;
|
||||||
|
last_map: string | null;
|
||||||
|
last_gametype: string | null;
|
||||||
|
last_seen_at: string | null;
|
||||||
|
last_error: string | null;
|
||||||
|
roster: StatusPlayer[] | null;
|
||||||
|
}
|
||||||
|
export interface ServerObservation extends GameObservation {
|
||||||
|
name: string;
|
||||||
|
health: 'online' | 'stale' | 'offline' | 'starting' | 'provisioning';
|
||||||
|
container_state?: string | null;
|
||||||
|
container_checked_at?: string | null;
|
||||||
|
last_checked_at?: string | null;
|
||||||
|
telemetry_after?: string | null;
|
||||||
|
status_source?: 'telemetry' | 'rcon' | null;
|
||||||
|
rcon_status?: GameObservation;
|
||||||
|
telemetry?: GameTelemetry;
|
||||||
|
active_since?: string | null;
|
||||||
|
inactive_since?: string | null;
|
||||||
|
game_started_at?: string | null;
|
||||||
|
game_ended_at?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function freshTelemetry(report: GameTelemetry | undefined, now: number, after?: string | null): report is GameTelemetry {
|
||||||
|
if (!report || !freshStatusTimestamp(report.receivedAt, now)) return false;
|
||||||
|
if (after && (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))) return false;
|
||||||
|
try { parseGameEvent(report); return true; } catch { return false; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Display policy: use one current roster/count source, keeping raw RCON for diagnostics/fallback. */
|
||||||
|
export function selectServerObservation<T extends ServerObservation>(server: T, report: GameTelemetry | undefined, now: number): T {
|
||||||
|
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
|
||||||
|
const raw = server.rcon_status ?? server;
|
||||||
|
const result: T = { ...server, ...raw, telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
|
||||||
|
if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running') || (!server.container_state && server.health === 'offline')) {
|
||||||
|
return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null };
|
||||||
|
}
|
||||||
|
const checked = server.container_checked_at ?? server.last_checked_at ?? raw.last_seen_at;
|
||||||
|
const containerFresh = server.container_state === 'running' && freshStatusTimestamp(checked ?? null, now);
|
||||||
|
if (containerFresh && report?.server === server.name && freshTelemetry(report, now, server.telemetry_after)) {
|
||||||
|
const players = report.players.length;
|
||||||
|
return { ...result, health: 'online', status_source: 'telemetry', last_players: players,
|
||||||
|
last_map: report.map, last_seen_at: report.receivedAt, last_error: null,
|
||||||
|
roster: report.players.map(player => ({ num: null, score: 'score' in player ? player.score : null,
|
||||||
|
ping: null, guid: player.guid, name: player.name })),
|
||||||
|
active_since: players > 0 ? server.active_since ?? report.receivedAt : null,
|
||||||
|
inactive_since: players === 0 ? server.inactive_since ?? report.receivedAt : null,
|
||||||
|
game_started_at: players === 0 ? null : report.event === 'game_start' ? report.receivedAt : server.game_started_at,
|
||||||
|
game_ended_at: players === 0 || report.event === 'game_start' ? null : report.event === 'game_end' ? report.receivedAt : server.game_ended_at };
|
||||||
|
}
|
||||||
|
const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
|
||||||
|
const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !raw.last_error;
|
||||||
|
const containerExpired = !!server.container_checked_at && !containerFresh;
|
||||||
|
const age = now - Date.parse(raw.last_seen_at ?? '');
|
||||||
|
return { ...result, health: fresh && !containerExpired ? 'online'
|
||||||
|
: after && age >= 0 && age <= 120_000 ? 'stale' : 'offline' };
|
||||||
|
}
|
||||||
@@ -15,6 +15,7 @@ let _db: Database.Database | null = null;
|
|||||||
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
|
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
|
||||||
// backfill columns; this can.
|
// backfill columns; this can.
|
||||||
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
|
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
|
||||||
|
{ name: 'container_checked_at', ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT' },
|
||||||
{ name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
|
{ name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
|
||||||
{ name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
|
{ name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
|
||||||
// Base columns that shipped in the very first schema stay here for parity
|
// Base columns that shipped in the very first schema stay here for parity
|
||||||
|
|||||||
@@ -63,6 +63,7 @@ CREATE TABLE IF NOT EXISTS servers (
|
|||||||
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
|
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
|
||||||
komodo_name TEXT NOT NULL,
|
komodo_name TEXT NOT NULL,
|
||||||
komodo_server TEXT,
|
komodo_server TEXT,
|
||||||
|
container_checked_at TEXT,
|
||||||
container_state TEXT, -- last observed Komodo runtime state
|
container_state TEXT, -- last observed Komodo runtime state
|
||||||
desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped')),
|
desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped')),
|
||||||
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
|
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { getServer } from './servers.js';
|
||||||
|
import { serverObservation } from './status.js';
|
||||||
import { randomUUID } from 'node:crypto';
|
import { randomUUID } from 'node:crypto';
|
||||||
import { db } from '../db/index.js';
|
import { db } from '../db/index.js';
|
||||||
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
|
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
|
||||||
@@ -52,17 +54,19 @@ export function recordGameEvent(event: GameTelemetry): void {
|
|||||||
|
|
||||||
export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
|
export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
|
||||||
const m = readMetrics(run);
|
const m = readMetrics(run);
|
||||||
const server = db().prepare('SELECT last_checked_at, last_seen_at, last_error, container_state FROM servers WHERE name = ?').get(run.server) as {
|
const row = getServer(run.server);
|
||||||
last_checked_at: string | null; last_seen_at: string | null; last_error: string | null; container_state: string | null };
|
const server = row ? serverObservation(row) : null;
|
||||||
const sample = server.last_checked_at ? Date.parse(server.last_checked_at) : 0;
|
const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
|
||||||
const freshSample = sample >= Date.parse(run.observing_at ?? run.created_at) && Date.now() - sample <= 60_000;
|
const sample = Date.parse(timestamp ?? '');
|
||||||
const healthy = freshSample && !server.last_error && server.container_state === 'running' && server.last_seen_at === server.last_checked_at;
|
const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
|
||||||
const at = freshSample && server.container_state === 'running' ? sample : Date.now();
|
&& Date.now() - sample >= 0 && Date.now() - sample <= 45_000;
|
||||||
|
const healthy = freshSample && server?.health === 'online';
|
||||||
|
const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
|
||||||
const before = m.wasHealthy;
|
const before = m.wasHealthy;
|
||||||
const lastSample = m.lastSample;
|
const lastSample = m.lastSample;
|
||||||
applySample(plan, m, at, healthy);
|
applySample(plan, m, at, healthy);
|
||||||
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: server.last_checked_at });
|
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, source: server?.status_source });
|
||||||
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server.last_error });
|
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
|
||||||
saveMetrics(run, m);
|
saveMetrics(run, m);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import { creationQueue, destroyServer, nextAvailablePort, nextLobbyNumber, queue
|
|||||||
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
|
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
|
||||||
import { housePartyTargets } from './house-party.js';
|
import { housePartyTargets } from './house-party.js';
|
||||||
import { getServerKey } from './server-keys.js';
|
import { getServerKey } from './server-keys.js';
|
||||||
import { classifyHealth } from './health.js';
|
import { serverObservation } from './status.js';
|
||||||
import { getServer } from './servers.js';
|
import { getServer } from './servers.js';
|
||||||
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
|
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
|
||||||
|
|
||||||
@@ -37,8 +37,8 @@ function coverage(entries: BlueprintEntry[]) {
|
|||||||
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
|
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
|
||||||
const server = getServer(name);
|
const server = getServer(name);
|
||||||
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
|
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
|
||||||
: server?.container_state && !['running', 'starting', 'restarting'].includes(server.container_state) ? 'stopped'
|
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
|
||||||
: classifyHealth(server?.last_seen_at ?? null, server?.last_error ?? null, server?.container_state),
|
: server ? serverObservation(server).health : 'offline',
|
||||||
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
|
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
|
||||||
});
|
});
|
||||||
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
|
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
|
||||||
|
|||||||
@@ -68,6 +68,7 @@ export interface ServerRow {
|
|||||||
komodo_name: string;
|
komodo_name: string;
|
||||||
komodo_server: string | null;
|
komodo_server: string | null;
|
||||||
container_state: string | null;
|
container_state: string | null;
|
||||||
|
container_checked_at: string | null;
|
||||||
desired_state: 'running' | 'stopped';
|
desired_state: 'running' | 'stopped';
|
||||||
game_port: number | null;
|
game_port: number | null;
|
||||||
rcon_host: string | null;
|
rcon_host: string | null;
|
||||||
@@ -162,6 +163,7 @@ export function setServerField(
|
|||||||
'komodo_name',
|
'komodo_name',
|
||||||
'komodo_server',
|
'komodo_server',
|
||||||
'container_state',
|
'container_state',
|
||||||
|
'container_checked_at',
|
||||||
'game_port',
|
'game_port',
|
||||||
'rcon_host',
|
'rcon_host',
|
||||||
'game',
|
'game',
|
||||||
|
|||||||
@@ -1,28 +1,43 @@
|
|||||||
import { db } from '../db/index.js';
|
import { db } from '../db/index.js';
|
||||||
import { parseGameEvent } from '../../telemetry.js';
|
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
|
||||||
import { ONLINE_MAX_MS } from '../../server-freshness.js';
|
import { ONLINE_MAX_MS } from '../../server-freshness.js';
|
||||||
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
|
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
|
||||||
|
|
||||||
export interface TelemetryEvidence { players: number; map: string; receivedAt: string }
|
export interface TelemetryEvidence { players: number; map: string; receivedAt: string }
|
||||||
|
|
||||||
/** Only validated, recently received reports can establish current game state. */
|
/** Reports from before the latest lifecycle submission describe the old process. */
|
||||||
export function telemetryEvidence(server: string, after?: string | null): TelemetryEvidence | null {
|
export function latestGameSubmission(server: string): string | null {
|
||||||
|
return (db().prepare(`SELECT MAX(at) AS at FROM (
|
||||||
|
SELECT submitted_at AS at FROM server_deployments WHERE server=? AND action IN ('create','start','restart','rebuild')
|
||||||
|
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function telemetryReport(server: string): GameTelemetry | null {
|
||||||
const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as
|
const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as
|
||||||
{ received_at: string; payload: string | null } | undefined;
|
{ received_at: string; payload: string | null } | undefined;
|
||||||
if (!row?.payload) return null;
|
if (!row?.payload) return null;
|
||||||
const received = Date.parse(row.received_at);
|
try {
|
||||||
|
const saved = JSON.parse(row.payload) as GameTelemetry;
|
||||||
|
const event = parseGameEvent(saved);
|
||||||
|
return event.server === server ? { ...event, receivedAt: row.received_at, timing: saved.timing } : null;
|
||||||
|
} catch { return null; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Only validated, recently received reports can establish current game state. */
|
||||||
|
export function telemetryEvidence(server: string, after: string | null = latestGameSubmission(server)): TelemetryEvidence | null {
|
||||||
|
const report = telemetryReport(server);
|
||||||
|
if (!report) return null;
|
||||||
|
const received = Date.parse(report.receivedAt);
|
||||||
const age = Date.now() - received;
|
const age = Date.now() - received;
|
||||||
if (!Number.isFinite(age) || age < 0 || age > ONLINE_MAX_MS) return null;
|
if (!Number.isFinite(age) || age < 0 || age > ONLINE_MAX_MS) return null;
|
||||||
if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after))) return null;
|
if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after))) return null;
|
||||||
try {
|
return { players: report.players.length, map: report.map, receivedAt: report.receivedAt };
|
||||||
const event = parseGameEvent(JSON.parse(row.payload));
|
|
||||||
if (event.server !== server) return null;
|
|
||||||
return { players: event.players.length, map: event.map, receivedAt: row.received_at };
|
|
||||||
} catch { return null; }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Positive occupancy from either source always vetoes an empty-server claim. */
|
/** Positive occupancy from either source always vetoes an empty-server claim. */
|
||||||
export async function probePlayerCount(server: string): Promise<number | null> {
|
export async function probePlayerCount(server: string): Promise<number | null> {
|
||||||
|
const occupied = telemetryEvidence(server);
|
||||||
|
if (occupied && occupied.players > 0) return occupied.players;
|
||||||
const players = await rconGetPlayerCount(server).catch(() => null);
|
const players = await rconGetPlayerCount(server).catch(() => null);
|
||||||
// Read after the network request: a player may join while RCON is pending.
|
// Read after the network request: a player may join while RCON is pending.
|
||||||
const telemetry = telemetryEvidence(server);
|
const telemetry = telemetryEvidence(server);
|
||||||
@@ -35,10 +50,14 @@ export async function probePlayerCount(server: string): Promise<number | null> {
|
|||||||
export async function probeServerStatus(server: string, after?: string | null): Promise<
|
export async function probeServerStatus(server: string, after?: string | null): Promise<
|
||||||
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry' } | { ok: false; reason: string }
|
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry' } | { ok: false; reason: string }
|
||||||
> {
|
> {
|
||||||
|
const current = telemetryEvidence(server, after);
|
||||||
|
if (current) return { ok: true, ...current, source: 'telemetry' };
|
||||||
let reason = 'RCON unavailable';
|
let reason = 'RCON unavailable';
|
||||||
try {
|
try {
|
||||||
const rcon = await rconStatusResult(server);
|
const rcon = await rconStatusResult(server);
|
||||||
if (rcon.ok) return { ok: true, players: Math.max(rcon.status.players, telemetryEvidence(server, after)?.players ?? 0), map: rcon.status.map, source: 'rcon' };
|
const latest = telemetryEvidence(server, after);
|
||||||
|
if (latest) return { ok: true, ...latest, source: 'telemetry' };
|
||||||
|
if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon' };
|
||||||
reason = rcon.reason;
|
reason = rcon.reason;
|
||||||
} catch (err) { reason = (err as Error).message; }
|
} catch (err) { reason = (err as Error).message; }
|
||||||
const telemetry = telemetryEvidence(server, after);
|
const telemetry = telemetryEvidence(server, after);
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
import { selectServerObservation, type GameObservation } from '../../server-observation.js';
|
||||||
|
import { classifyHealth } from './health.js';
|
||||||
|
import { isProvisioning } from './provisioning.js';
|
||||||
|
import { restartSettling } from '../provisioning/restarts.js';
|
||||||
|
import { type ServerRow } from './servers.js';
|
||||||
|
import { telemetryReport, latestGameSubmission } from './status-evidence.js';
|
||||||
|
|
||||||
|
/** Shared snapshot for REST, status events, metrics, and other server-side readers. */
|
||||||
|
export function serverObservation(server: ServerRow) {
|
||||||
|
let roster = null;
|
||||||
|
try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
|
||||||
|
const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
|
||||||
|
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
|
||||||
|
const provisioning = isProvisioning(server.name);
|
||||||
|
const state = restartSettling(server.name) ? 'restarting' : server.container_state;
|
||||||
|
const ready = !provisioning && (!state || state === 'running');
|
||||||
|
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
|
||||||
|
return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
|
||||||
|
status_source: null as 'telemetry' | 'rcon' | null,
|
||||||
|
telemetry: telemetryReport(server.name) ?? undefined,
|
||||||
|
telemetry_after: latestGameSubmission(server.name),
|
||||||
|
last_error: ready ? server.last_error : null,
|
||||||
|
replySnippet: ready ? server.last_reply_snippet : null,
|
||||||
|
active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null,
|
||||||
|
game_started_at: ready ? server.game_started_at : null, game_ended_at: ready ? server.game_ended_at : null
|
||||||
|
}, undefined, Date.now());
|
||||||
|
}
|
||||||
@@ -257,85 +257,62 @@ export type KomodoHealthResult =
|
|||||||
| { ok: true; state: string }
|
| { ok: true; state: string }
|
||||||
| { ok: false; reason: string; state?: string; snippet?: string };
|
| { ok: false; reason: string; state?: string; snippet?: string };
|
||||||
|
|
||||||
// Shape-agnostic scan for the container's running-state. Walks the entire
|
// Komodo's container summary has { state: "running", container: { status:
|
||||||
// payload looking for anything that resembles a state field. Covered shapes:
|
// "Up 35 seconds (health: starting)" } }. Inspect responses instead have
|
||||||
// - Docker inspect: { State: { Status: "running", Running: true, Health: { Status: "healthy" } } }
|
// State.Health.Status (or snake/lowercase equivalents). Runtime health wins
|
||||||
// - Komodo GetDeployment: top-level or nested `state`, `Status`, `container_state`
|
// over a coarse running flag. Unknown/configuration strings are not readiness.
|
||||||
// - Komodo GetDeploymentStatus: { state: "Running" }
|
|
||||||
// - Anything with `Running: true` at any depth
|
|
||||||
// When it can't find any of these, callers get the raw JSON in the error so
|
|
||||||
// we can spot which field to add here.
|
|
||||||
const STATE_KEYS = new Set([
|
|
||||||
'state',
|
|
||||||
'State',
|
|
||||||
'status',
|
|
||||||
'Status',
|
|
||||||
'container_state',
|
|
||||||
'containerState',
|
|
||||||
'deployment_state',
|
|
||||||
'deploymentState',
|
|
||||||
'docker_status'
|
|
||||||
]);
|
|
||||||
|
|
||||||
export function extractContainerState(payload: unknown): string | null {
|
export function extractContainerState(payload: unknown): string | null {
|
||||||
const seen = new Set<unknown>();
|
const seen = new Set<unknown>();
|
||||||
let found: string | null = null;
|
const known = new Set(['running','healthy','starting','restarting','unhealthy','stopped','exited','created','dead','paused','removing','notdeployed','unknown']);
|
||||||
|
const stateOf = (value: unknown): string | null => {
|
||||||
const walk = (v: unknown): void => {
|
if (typeof value !== 'string') return null;
|
||||||
if (found || v === null || typeof v !== 'object' || seen.has(v)) return;
|
const state = value.toLowerCase().trim();
|
||||||
seen.add(v);
|
if (known.has(state)) return state;
|
||||||
if (Array.isArray(v)) {
|
if (/^up\b/.test(state)) {
|
||||||
for (const item of v) walk(item);
|
const health = state.match(/\((?:health:\s*)?(starting|unhealthy|healthy)\)/)?.[1];
|
||||||
return;
|
return health && health !== 'healthy' ? health : 'running';
|
||||||
}
|
}
|
||||||
const obj = v as Record<string, unknown>;
|
if (/^exited\b/.test(state)) return 'exited';
|
||||||
|
if (/^restarting\b/.test(state)) return 'restarting';
|
||||||
// Komodo summaries expose health alongside a flat running state.
|
return null;
|
||||||
const flatState = obj['state'] ?? obj['State'] ?? obj['status'] ?? obj['Status'];
|
|
||||||
const flatHealth = obj['health'] ?? obj['Health'];
|
|
||||||
const healthStatus = typeof flatHealth === 'string' ? flatHealth
|
|
||||||
: flatHealth && typeof flatHealth === 'object'
|
|
||||||
? ((flatHealth as Record<string, unknown>)['status'] ?? (flatHealth as Record<string, unknown>)['Status']) : null;
|
|
||||||
if (typeof flatState === 'string' && flatState.toLowerCase() === 'running'
|
|
||||||
&& typeof healthStatus === 'string' && healthStatus.toLowerCase() !== 'healthy') {
|
|
||||||
found = healthStatus;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Docker inspect nested: { State: { Status, Running, Health: { Status } } }
|
|
||||||
const stateObj = obj['State'];
|
|
||||||
if (stateObj && typeof stateObj === 'object') {
|
|
||||||
const so = stateObj as Record<string, unknown>;
|
|
||||||
const status = so['Status'];
|
|
||||||
const running = so['Running'];
|
|
||||||
const health = so['Health'];
|
|
||||||
// A running process can still be starting or unhealthy. Honor an
|
|
||||||
// explicit Docker health check before accepting its running state.
|
|
||||||
if ((running === true || status === 'running') && health && typeof health === 'object') {
|
|
||||||
const hs = (health as Record<string, unknown>)['Status'];
|
|
||||||
if (typeof hs === 'string' && hs.toLowerCase() !== 'healthy') { found = hs; return; }
|
|
||||||
}
|
|
||||||
if (typeof status === 'string') { found = status; return; }
|
|
||||||
if (running === true) { found = 'running'; return; }
|
|
||||||
if (health && typeof health === 'object') {
|
|
||||||
const hs = (health as Record<string, unknown>)['Status'];
|
|
||||||
if (typeof hs === 'string') { found = hs; return; }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Flat string-valued state under any of the recognized keys.
|
|
||||||
for (const key of STATE_KEYS) {
|
|
||||||
const val = obj[key];
|
|
||||||
if (typeof val === 'string' && val.length > 0) {
|
|
||||||
found = val;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const val of Object.values(obj)) walk(val);
|
|
||||||
};
|
};
|
||||||
walk(payload);
|
const walk = (value: unknown): string | null => {
|
||||||
return found;
|
if (!value || typeof value !== 'object' || seen.has(value)) return null;
|
||||||
|
seen.add(value);
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
const states = value.map(walk).filter((state): state is string => !!state);
|
||||||
|
return states.find(state => !['running','healthy'].includes(state)) ?? states[0] ?? null;
|
||||||
|
}
|
||||||
|
const object = value as Record<string, unknown>;
|
||||||
|
const rawState = object.State ?? object.state;
|
||||||
|
const state = stateOf(rawState) ?? stateOf(object.container_state ?? object.containerState ?? object.deployment_state ?? object.deploymentState);
|
||||||
|
// A stopped process is never ready just because its previous health check passed.
|
||||||
|
if (state && !['running','healthy','unknown'].includes(state)) return state;
|
||||||
|
if (rawState && typeof rawState === 'object') {
|
||||||
|
const nested = walk(rawState);
|
||||||
|
if (nested) return nested;
|
||||||
|
}
|
||||||
|
for (const key of ['container','Container','containers','Containers']) {
|
||||||
|
if (object[key] === null && state === 'running') return 'unknown';
|
||||||
|
const nested = walk(object[key]);
|
||||||
|
if (nested) return nested;
|
||||||
|
}
|
||||||
|
const status = stateOf(object.Status ?? object.status ?? object.docker_status);
|
||||||
|
if (status && !['running','healthy'].includes(status)) return status;
|
||||||
|
const health = object.Health ?? object.health;
|
||||||
|
const healthState = stateOf(typeof health === 'object' && health ?
|
||||||
|
(health as Record<string, unknown>).Status ?? (health as Record<string, unknown>).status : health);
|
||||||
|
if (healthState && healthState !== 'healthy') return healthState;
|
||||||
|
if (state || status) return state ?? status;
|
||||||
|
if (object.Running === true || object.running === true) return healthState ?? 'running';
|
||||||
|
for (const [key, item] of Object.entries(object)) {
|
||||||
|
if (['config','Config','labels','Labels'].includes(key)) continue;
|
||||||
|
const nested = walk(item);
|
||||||
|
if (nested) return nested;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
return walk(payload);
|
||||||
}
|
}
|
||||||
|
|
||||||
const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
|
const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
|
import type { StatusPlayer, GameObservation } from '../../server-observation.js';
|
||||||
import { EventEmitter } from 'node:events';
|
import { EventEmitter } from 'node:events';
|
||||||
import type { GameTelemetry } from '../../telemetry.js';
|
import type { GameTelemetry } from '../../telemetry.js';
|
||||||
import type { RconEndpoint, RconFailure, RconPlayer } from '../integrations/rcon.js';
|
import type { RconEndpoint, RconFailure } from '../integrations/rcon.js';
|
||||||
import type { Health } from '../fleet/health.js';
|
import type { Health } from '../fleet/health.js';
|
||||||
|
|
||||||
// One in-process event bus, multiplexed by the SSE endpoints. Kept separate
|
// One in-process event bus, multiplexed by the SSE endpoints. Kept separate
|
||||||
@@ -9,10 +10,15 @@ import type { Health } from '../fleet/health.js';
|
|||||||
|
|
||||||
export interface ServerStatusEntry {
|
export interface ServerStatusEntry {
|
||||||
server: string;
|
server: string;
|
||||||
|
containerState?: string | null;
|
||||||
|
containerCheckedAt?: string | null;
|
||||||
|
telemetryAfter?: string | null;
|
||||||
|
rconStatus?: GameObservation;
|
||||||
|
telemetry?: GameTelemetry;
|
||||||
players: number | null;
|
players: number | null;
|
||||||
map: string | null;
|
map: string | null;
|
||||||
gametype: string | null;
|
gametype: string | null;
|
||||||
roster: RconPlayer[] | null;
|
roster: StatusPlayer[] | null;
|
||||||
health: Health;
|
health: Health;
|
||||||
lastSeenAt: string | null;
|
lastSeenAt: string | null;
|
||||||
lastError: string | null;
|
lastError: string | null;
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { config } from '../config.js';
|
||||||
import { processProvisioning } from '../provisioning/worker.js';
|
import { processProvisioning } from '../provisioning/worker.js';
|
||||||
import { sweep } from './sweep.js';
|
import { sweep } from './sweep.js';
|
||||||
import { betaTick } from './beta-runner.js';
|
import { betaTick } from './beta-runner.js';
|
||||||
@@ -8,7 +9,8 @@ import type { RconFailure } from '../integrations/rcon.js';
|
|||||||
import { getServer, listServers, setServerField } from '../fleet/servers.js';
|
import { getServer, listServers, setServerField } from '../fleet/servers.js';
|
||||||
import { komodoInspectHealthy } from '../integrations/komodo.js';
|
import { komodoInspectHealthy } from '../integrations/komodo.js';
|
||||||
import { restartSettling } from '../provisioning/index.js';
|
import { restartSettling } from '../provisioning/index.js';
|
||||||
import { classifyHealth } from '../fleet/health.js';
|
import { serverObservation } from '../fleet/status.js';
|
||||||
|
import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
|
||||||
import { isProvisioning } from '../fleet/provisioning.js';
|
import { isProvisioning } from '../fleet/provisioning.js';
|
||||||
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
|
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
|
||||||
import { recordMetrics } from '../fleet/metrics.js';
|
import { recordMetrics } from '../fleet/metrics.js';
|
||||||
@@ -68,167 +70,84 @@ export function fanoutStatus(): Promise<void> {
|
|||||||
|
|
||||||
async function refreshStatus(): Promise<void> {
|
async function refreshStatus(): Promise<void> {
|
||||||
const servers = listServers();
|
const servers = listServers();
|
||||||
if (servers.length === 0) return;
|
if (!servers.length) return;
|
||||||
|
|
||||||
const entries: ServerStatusEntry[] = new Array(servers.length);
|
const entries: ServerStatusEntry[] = new Array(servers.length);
|
||||||
const capacities = new Map<string, number | null>();
|
const capacities = new Map<string, number | null>();
|
||||||
let cursor = 0;
|
let cursor = 0;
|
||||||
|
async function pump() {
|
||||||
async function pump(): Promise<void> {
|
while (cursor < servers.length) {
|
||||||
while (true) {
|
|
||||||
const i = cursor++;
|
const i = cursor++;
|
||||||
if (i >= servers.length) return;
|
|
||||||
const srv = servers[i];
|
const srv = servers[i];
|
||||||
// Creation performs its own container, RCON, and map verification.
|
if (!isProvisioning(srv.name)) {
|
||||||
if (isProvisioning(srv.name)) {
|
const epoch = latestGameSubmission(srv.name);
|
||||||
setServerField(srv.name, 'last_error', null);
|
|
||||||
setServerField(srv.name, 'last_reply_snippet', null);
|
|
||||||
entries[i] = {
|
|
||||||
server: srv.name, players: null, map: srv.map, gametype: null,
|
|
||||||
roster: null, health: 'provisioning', lastSeenAt: srv.last_seen_at,
|
|
||||||
lastError: null, endpoint: null, reason: null, replySnippet: null,
|
|
||||||
activeSince: null, inactiveSince: null, gameStartedAt: null, gameEndedAt: null
|
|
||||||
};
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const container = restartSettling(srv.name)
|
const container = restartSettling(srv.name)
|
||||||
? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
|
? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
|
||||||
: await komodoInspectHealthy(srv.name).catch((err) => ({
|
: await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
|
||||||
ok: false as const, reason: (err as Error).message
|
// A lifecycle request may be submitted while an inspection is in flight.
|
||||||
}));
|
if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) {
|
||||||
const containerState = container.ok ? 'running'
|
setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown');
|
||||||
: ('state' in container && typeof container.state === 'string' ? container.state.toLowerCase() : 'unknown');
|
setServerField(srv.name, 'container_checked_at', new Date().toISOString());
|
||||||
setServerField(srv.name, 'container_state', containerState);
|
|
||||||
if (!container.ok) {
|
if (!container.ok) {
|
||||||
// Container readiness precedes RCON readiness. Old replies and
|
|
||||||
// failures describe the previous process, not this startup.
|
|
||||||
setServerField(srv.name, 'last_error', null);
|
setServerField(srv.name, 'last_error', null);
|
||||||
setServerField(srv.name, 'last_reply_snippet', null);
|
setServerField(srv.name, 'last_reply_snippet', null);
|
||||||
entries[i] = {
|
} else if (!telemetryEvidence(srv.name)) {
|
||||||
server: srv.name, players: null, map: srv.map, gametype: null,
|
const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
|
||||||
roster: null, health: classifyHealth(null, null, containerState),
|
reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
|
||||||
lastSeenAt: srv.last_seen_at, lastError: null, endpoint: null,
|
if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) {
|
||||||
reason: null, replySnippet: null, activeSince: null,
|
|
||||||
inactiveSince: null, gameStartedAt: null, gameEndedAt: null
|
|
||||||
};
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const result = await rconStatusResult(srv.name).catch((err) => ({
|
|
||||||
ok: false as const,
|
|
||||||
endpoint: null,
|
|
||||||
reason: 'send-error' as RconFailure,
|
|
||||||
detail: (err as Error).message,
|
|
||||||
rawReply: undefined as string | undefined
|
|
||||||
}));
|
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
|
setServerField(srv.name, 'last_checked_at', now);
|
||||||
|
setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null);
|
||||||
if (result.ok) {
|
if (result.ok) {
|
||||||
capacities.set(srv.name, result.status.capacity ?? null);
|
capacities.set(srv.name, result.status.capacity ?? null);
|
||||||
// Compare against the previously persisted count BEFORE we
|
setServerField(srv.name, 'last_players', result.status.players);
|
||||||
// overwrite last_players — occupancy transitions are anchored
|
|
||||||
// to that boundary.
|
|
||||||
const prevPlayers = srv.last_players;
|
|
||||||
const nextPlayers = result.status.players;
|
|
||||||
let activeSince = srv.active_since;
|
|
||||||
let inactiveSince = srv.inactive_since;
|
|
||||||
// Telemetry can arrive while the RCON request is in flight.
|
|
||||||
const current = getServer(srv.name) ?? srv;
|
|
||||||
const gameStartedAt = nextPlayers === 0 ? null : current.game_started_at;
|
|
||||||
const gameEndedAt = nextPlayers === 0 ? null : current.game_ended_at;
|
|
||||||
if (nextPlayers === 0) {
|
|
||||||
setServerField(srv.name, 'game_started_at', null);
|
|
||||||
setServerField(srv.name, 'game_ended_at', null);
|
|
||||||
}
|
|
||||||
if (nextPlayers > 0 && (activeSince === null || prevPlayers === 0 || prevPlayers === null)) {
|
|
||||||
activeSince = now;
|
|
||||||
inactiveSince = null;
|
|
||||||
setServerField(srv.name, 'active_since', activeSince);
|
|
||||||
setServerField(srv.name, 'inactive_since', null);
|
|
||||||
} else if (nextPlayers === 0 && (inactiveSince === null || (prevPlayers !== null && prevPlayers > 0))) {
|
|
||||||
inactiveSince = now;
|
|
||||||
activeSince = null;
|
|
||||||
setServerField(srv.name, 'inactive_since', inactiveSince);
|
|
||||||
setServerField(srv.name, 'active_since', null);
|
|
||||||
}
|
|
||||||
// Persist last-observed state. last_seen_at is the health anchor.
|
|
||||||
setServerField(srv.name, 'last_players', nextPlayers);
|
|
||||||
setServerField(srv.name, 'last_map', result.status.map);
|
setServerField(srv.name, 'last_map', result.status.map);
|
||||||
setServerField(srv.name, 'last_gametype', result.status.gametype);
|
setServerField(srv.name, 'last_gametype', result.status.gametype);
|
||||||
setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
|
setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
|
||||||
setServerField(srv.name, 'last_seen_at', now);
|
setServerField(srv.name, 'last_seen_at', now);
|
||||||
setServerField(srv.name, 'last_checked_at', now);
|
|
||||||
setServerField(srv.name, 'last_error', null);
|
setServerField(srv.name, 'last_error', null);
|
||||||
setServerField(srv.name, 'last_reply_snippet', result.rawReply);
|
} else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail));
|
||||||
entries[i] = {
|
}
|
||||||
server: srv.name,
|
}
|
||||||
players: nextPlayers,
|
}
|
||||||
map: result.status.map,
|
|
||||||
gametype: result.status.gametype,
|
|
||||||
roster: result.status.roster,
|
|
||||||
health: classifyHealth(now, null),
|
|
||||||
lastSeenAt: now,
|
|
||||||
lastError: null,
|
|
||||||
endpoint: result.endpoint,
|
|
||||||
reason: null,
|
|
||||||
replySnippet: result.rawReply,
|
|
||||||
activeSince,
|
|
||||||
inactiveSince,
|
|
||||||
gameStartedAt,
|
|
||||||
gameEndedAt
|
|
||||||
};
|
|
||||||
} else {
|
} else {
|
||||||
// RCON unreachable — leave last_seen_at, active_since, and
|
setServerField(srv.name, 'last_error', null);
|
||||||
// inactive_since alone (health decays to offline naturally;
|
setServerField(srv.name, 'last_reply_snippet', null);
|
||||||
// occupancy timers survive transient outages). Record the
|
}
|
||||||
// structured failure.
|
const current = getServer(srv.name) ?? srv;
|
||||||
const encoded = encodeError(result.reason, result.detail);
|
const status = serverObservation(current);
|
||||||
const snippet = result.rawReply ?? null;
|
// Occupancy timers and match clearing follow the same source as the UI.
|
||||||
setServerField(srv.name, 'last_checked_at', now);
|
if (status.health === 'online' && status.last_players !== null) {
|
||||||
setServerField(srv.name, 'last_error', encoded);
|
if (status.last_players > 0) {
|
||||||
setServerField(srv.name, 'last_reply_snippet', snippet);
|
setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at);
|
||||||
entries[i] = {
|
setServerField(srv.name, 'inactive_since', null);
|
||||||
server: srv.name,
|
} else {
|
||||||
players: srv.last_players,
|
setServerField(srv.name, 'active_since', null);
|
||||||
map: srv.last_map,
|
setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at);
|
||||||
gametype: srv.last_gametype,
|
setServerField(srv.name, 'game_started_at', null);
|
||||||
roster: srv.last_roster ? safeJson(srv.last_roster) : null,
|
setServerField(srv.name, 'game_ended_at', null);
|
||||||
health: classifyHealth(srv.last_seen_at, encoded),
|
|
||||||
lastSeenAt: srv.last_seen_at,
|
|
||||||
lastError: encoded,
|
|
||||||
endpoint: result.endpoint,
|
|
||||||
reason: result.reason,
|
|
||||||
replySnippet: snippet,
|
|
||||||
activeSince: srv.active_since,
|
|
||||||
inactiveSince: srv.inactive_since,
|
|
||||||
gameStartedAt: srv.game_started_at,
|
|
||||||
gameEndedAt: srv.game_ended_at
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
const final = serverObservation(getServer(srv.name) ?? current);
|
||||||
|
entries[i] = { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map,
|
||||||
|
gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at,
|
||||||
|
lastError: final.last_error, replySnippet: final.replySnippet,
|
||||||
|
endpoint: current.game_port ? { host: current.rcon_host ?? config.rcon.host, port: current.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null,
|
||||||
|
activeSince: final.active_since, inactiveSince: final.inactive_since,
|
||||||
|
gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at,
|
||||||
|
containerState: final.container_state, containerCheckedAt: final.container_checked_at,
|
||||||
|
telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry };
|
||||||
}
|
}
|
||||||
await Promise.all(Array.from({ length: STATUS_CONCURRENCY }, pump));
|
}
|
||||||
|
await Promise.all(Array.from({ length: Math.min(STATUS_CONCURRENCY, servers.length) }, pump));
|
||||||
try {
|
try {
|
||||||
recordMetrics(servers.map((srv, i) => ({
|
recordMetrics(servers.map((srv, i) => ({ server: srv.name, game: srv.game ?? 'unknown',
|
||||||
server: srv.name,
|
|
||||||
game: srv.game ?? 'unknown',
|
|
||||||
map: selectedZombieMap(srv.game, entries[i].map ?? srv.map, srv.map_slug)?.friendly ?? entries[i].map ?? srv.map ?? 'Unknown map',
|
map: selectedZombieMap(srv.game, entries[i].map ?? srv.map, srv.map_slug)?.friendly ?? entries[i].map ?? srv.map ?? 'Unknown map',
|
||||||
players: capacities.has(srv.name) ? entries[i].players : null,
|
players: entries[i].health === 'online' ? entries[i].players : null,
|
||||||
capacity: capacities.get(srv.name) ?? null
|
capacity: capacities.get(srv.name) ?? null })));
|
||||||
})));
|
|
||||||
} catch (err) { console.error('[worker] metrics error:', err); }
|
} catch (err) { console.error('[worker] metrics error:', err); }
|
||||||
|
|
||||||
emit({ type: 'server-status', entries });
|
emit({ type: 'server-status', entries });
|
||||||
}
|
}
|
||||||
|
|
||||||
function safeJson<T>(raw: string): T | null {
|
|
||||||
try {
|
|
||||||
return JSON.parse(raw) as T;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function startWorker(): void {
|
export function startWorker(): void {
|
||||||
if (started) return;
|
if (started) return;
|
||||||
started = true;
|
started = true;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { timeFormat } from '$ui/stores/time-zone.js';
|
import { timeFormat } from '$ui/stores/time-zone.js';
|
||||||
import type { GameTelemetry } from '$lib/telemetry.js';
|
import type { GameTelemetry } from '$lib/telemetry.js';
|
||||||
|
import { freshTelemetry } from '$lib/server-observation.js';
|
||||||
import { uiNow } from '$ui/stores/servers.js';
|
import { uiNow } from '$ui/stores/servers.js';
|
||||||
import { elapsedSince } from '$ui/duration.js';
|
import { elapsedSince } from '$ui/duration.js';
|
||||||
import { mapTitle } from '$ui/server-labels.js';
|
import { mapTitle } from '$ui/server-labels.js';
|
||||||
@@ -9,8 +10,9 @@
|
|||||||
import Badge from './primitives/Badge.svelte';
|
import Badge from './primitives/Badge.svelte';
|
||||||
import EmptyState from './primitives/EmptyState.svelte';
|
import EmptyState from './primitives/EmptyState.svelte';
|
||||||
|
|
||||||
let { report, mapLabel, game }: { report: GameTelemetry | undefined; mapLabel?: string; game: string } = $props();
|
let { report, mapLabel, game, current = true, after = null }: { report: GameTelemetry | undefined; mapLabel?: string; game: string; current?: boolean; after?: string | null } = $props();
|
||||||
const timingNow = $derived(report && (report.event === 'game_end' || $uiNow - Date.parse(report.receivedAt) > 120_000)
|
const fresh = $derived(current && freshTelemetry(report, $uiNow, after));
|
||||||
|
const timingNow = $derived(report && (report.event === 'game_end' || !fresh)
|
||||||
? Date.parse(report.receivedAt) : $uiNow);
|
? Date.parse(report.receivedAt) : $uiNow);
|
||||||
const columns = [
|
const columns = [
|
||||||
{ key: 'score', label: 'Score' },
|
{ key: 'score', label: 'Score' },
|
||||||
@@ -38,11 +40,11 @@
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
{#if !report}
|
{#if !report}
|
||||||
<EmptyState title="Waiting for game data" description="The scoreboard will appear when this server reports a game start, round change, or game end." />
|
<EmptyState title="Waiting for game data" description="The scoreboard will appear when this server reports a game start, round change, heartbeat, or game end." />
|
||||||
{:else}
|
{:else}
|
||||||
<header class="scoreboard-header">
|
<header class="scoreboard-header">
|
||||||
<div class="match">
|
<div class="match">
|
||||||
<span class="eyebrow">{report.event === 'game_end' ? 'Final scoreboard' : 'Latest scoreboard'}</span>
|
<span class="eyebrow">{report.event === 'game_end' ? 'Final scoreboard' : fresh ? 'Live scoreboard' : 'Last reported scoreboard'}</span>
|
||||||
<div class="match-title">
|
<div class="match-title">
|
||||||
<h2 title={report.map}>{mapLabel ?? mapTitle(report.map)}</h2>
|
<h2 title={report.map}>{mapLabel ?? mapTitle(report.map)}</h2>
|
||||||
{#if report.event !== 'game_start'}<Badge variant="accent" size="md">Round {report.round}</Badge>{/if}
|
{#if report.event !== 'game_start'}<Badge variant="accent" size="md">Round {report.round}</Badge>{/if}
|
||||||
@@ -51,7 +53,8 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="report-meta">
|
<div class="report-meta">
|
||||||
<span>{players.length} {players.length === 1 ? 'player' : 'players'}</span>
|
<span>{players.length} {players.length === 1 ? 'player' : 'players'} in this report</span>
|
||||||
|
{#if !fresh}<Badge variant="warning">Historical report</Badge>{/if}
|
||||||
<time datetime={report.receivedAt} title={$timeFormat.dateTime(report.receivedAt)}>
|
<time datetime={report.receivedAt} title={$timeFormat.dateTime(report.receivedAt)}>
|
||||||
Updated {elapsedSince(report.receivedAt, $uiNow)} ago
|
Updated {elapsedSince(report.receivedAt, $uiNow)} ago
|
||||||
</time>
|
</time>
|
||||||
|
|||||||
@@ -1,18 +1,19 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { timeFormat } from '$ui/stores/time-zone.js';
|
import { timeFormat } from '$ui/stores/time-zone.js';
|
||||||
import { telemetryByServer, uiNow } from '$ui/stores/servers.js';
|
import { telemetryByServer, uiNow } from '$ui/stores/servers.js';
|
||||||
|
import { freshStatusTimestamp } from '$lib/server-freshness.js';
|
||||||
import { elapsedSince } from '$ui/duration.js';
|
import { elapsedSince } from '$ui/duration.js';
|
||||||
let { name, since = null }: { name: string; since?: string | null } = $props();
|
import type { GameTelemetry } from '$lib/telemetry.js';
|
||||||
let report = $derived($telemetryByServer.get(name));
|
let { name, since = null, report: observation }: { name: string; since?: string | null; report?: GameTelemetry } = $props();
|
||||||
let fresh = $derived(report && $uiNow - Date.parse(report.receivedAt) >= 0
|
let report = $derived(observation ?? $telemetryByServer.get(name));
|
||||||
&& $uiNow - Date.parse(report.receivedAt) <= 120_000
|
let fresh = $derived(report && freshStatusTimestamp(report.receivedAt, $uiNow)
|
||||||
&& (!since || Date.parse(report.receivedAt) >= Date.parse(since)));
|
&& (!since || Date.parse(report.receivedAt) >= Date.parse(since)));
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
{#if report && (report.event === 'round_change' || report.event === 'heartbeat') && fresh}
|
{#if report && (report.event === 'round_change' || report.event === 'heartbeat') && fresh}
|
||||||
<div class="round" aria-live="polite">
|
<div class="round" aria-live="polite">
|
||||||
<strong>Round {report.round}</strong>
|
<strong>Round {report.round}</strong>
|
||||||
<span title={$timeFormat.dateTime(report.receivedAt)}>Last reported {elapsedSince(report.receivedAt, $uiNow)} ago · {report.players.length} players</span>
|
<span title={$timeFormat.dateTime(report.receivedAt)}>Last reported {elapsedSince(report.receivedAt, $uiNow)} ago</span>
|
||||||
</div>
|
</div>
|
||||||
{/if}
|
{/if}
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,7 @@
|
|||||||
betaId: number;
|
betaId: number;
|
||||||
}
|
}
|
||||||
interface Props {
|
interface Props {
|
||||||
|
telemetry?: import('$lib/telemetry.js').GameTelemetry;
|
||||||
name: string;
|
name: string;
|
||||||
players?: number | null;
|
players?: number | null;
|
||||||
map?: string | null;
|
map?: string | null;
|
||||||
@@ -35,6 +36,7 @@
|
|||||||
mapLabelOverride?: string;
|
mapLabelOverride?: string;
|
||||||
}
|
}
|
||||||
let {
|
let {
|
||||||
|
telemetry,
|
||||||
name,
|
name,
|
||||||
players = null,
|
players = null,
|
||||||
map = null,
|
map = null,
|
||||||
@@ -66,7 +68,7 @@
|
|||||||
let topRoster = $derived(
|
let topRoster = $derived(
|
||||||
(roster ?? [])
|
(roster ?? [])
|
||||||
.slice()
|
.slice()
|
||||||
.sort((a, b) => b.score - a.score)
|
.sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
|
||||||
.slice(0, 4)
|
.slice(0, 4)
|
||||||
);
|
);
|
||||||
let showRoster = $derived(health === 'online' || health === 'stale');
|
let showRoster = $derived(health === 'online' || health === 'stale');
|
||||||
@@ -134,7 +136,7 @@
|
|||||||
{#if topRoster.length}<div class="roster-heading">Last reported players</div>{/if}
|
{#if topRoster.length}<div class="roster-heading">Last reported players</div>{/if}
|
||||||
{/if}
|
{/if}
|
||||||
|
|
||||||
{#if health === 'online' && players !== null && players > 0 && gameInProgress}<RoundTelemetry {name} since={gameStartedAt} />{/if}
|
{#if health === 'online' && players !== null && players > 0 && gameInProgress}<RoundTelemetry {name} report={telemetry} since={gameStartedAt} />{/if}
|
||||||
|
|
||||||
{#if showTiming && !(compactEmpty && players === 0)}
|
{#if showTiming && !(compactEmpty && players === 0)}
|
||||||
<div class="timing">
|
<div class="timing">
|
||||||
@@ -169,8 +171,8 @@
|
|||||||
{#each topRoster as p (p.guid ?? p.name)}
|
{#each topRoster as p (p.guid ?? p.name)}
|
||||||
<li>
|
<li>
|
||||||
<span class="pname">{p.name || '—'}</span>
|
<span class="pname">{p.name || '—'}</span>
|
||||||
<span class="pscore">{p.score}</span>
|
<span class="pscore">{p.score ?? '—'}</span>
|
||||||
<span class="pping">{p.ping}ms</span>
|
<span class="pping">{p.ping === null ? '—' : `${p.ping}ms`}</span>
|
||||||
</li>
|
</li>
|
||||||
{/each}
|
{/each}
|
||||||
</ul>
|
</ul>
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
|
import { selectServerObservation } from '$lib/server-observation.js';
|
||||||
import { derived, writable } from 'svelte/store';
|
import { derived, writable } from 'svelte/store';
|
||||||
import { browser } from '$app/environment';
|
import { browser } from '$app/environment';
|
||||||
import { statusByServer, statusReceivedAt, syncServerTime, matchEventsByServer } from './servers.js';
|
import { statusByServer, statusReceivedAt, syncServerTime, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
|
||||||
import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js';
|
import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js';
|
||||||
|
|
||||||
export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
|
export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
|
||||||
@@ -10,18 +11,24 @@ export const fleetError = writable('');
|
|||||||
export const fleetRefreshing = writable(false);
|
export const fleetRefreshing = writable(false);
|
||||||
export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
|
export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
|
||||||
|
|
||||||
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer], ([snapshot, live, received, matches]) =>
|
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, uiNow], ([snapshot, live, received, matches, reports, now]) =>
|
||||||
snapshot.servers.map(server => {
|
snapshot.servers.map(server => {
|
||||||
const status = received > snapshot.receivedAt ? live.get(server.name) : undefined;
|
const status = received > snapshot.receivedAt ? live.get(server.name) : undefined;
|
||||||
const current = status ? { ...server, last_players: status.players, last_map: status.map,
|
const base = status ? { ...server, last_players: status.players, last_map: status.map,
|
||||||
last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
|
last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
|
||||||
roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
|
roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
|
||||||
inactive_since: status.inactiveSince, game_started_at: status.gameStartedAt,
|
inactive_since: status.inactiveSince, game_started_at: status.gameStartedAt,
|
||||||
game_ended_at: status.gameEndedAt } : server;
|
game_ended_at: status.gameEndedAt, container_state: status.containerState ?? server.container_state,
|
||||||
|
container_checked_at: status.containerCheckedAt ?? server.container_checked_at,
|
||||||
|
telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus,
|
||||||
|
telemetry: status.telemetry ?? server.telemetry } : server;
|
||||||
|
const current = selectServerObservation(base, reports.get(server.name), now);
|
||||||
const match = matches.get(server.name);
|
const match = matches.get(server.name);
|
||||||
// Apply lifecycle events immediately, until a newer status observes them
|
// Apply lifecycle events immediately, until a newer status observes them
|
||||||
// or confirms that the lobby has emptied.
|
// or confirms that the lobby has emptied.
|
||||||
if (!match || Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
|
if (!match || current.health !== 'online'
|
||||||
|
|| (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after))
|
||||||
|
|| Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
|
||||||
|| match.receivedAt < (current.game_started_at ?? '')
|
|| match.receivedAt < (current.game_started_at ?? '')
|
||||||
|| match.receivedAt < (current.game_ended_at ?? '')) return current;
|
|| match.receivedAt < (current.game_ended_at ?? '')) return current;
|
||||||
return { ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
|
return { ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
|
||||||
|
|||||||
@@ -5,13 +5,8 @@ import type { GameTelemetry } from '$lib/telemetry.js';
|
|||||||
export const telemetryByServer = writable<Map<string, GameTelemetry>>(new Map());
|
export const telemetryByServer = writable<Map<string, GameTelemetry>>(new Map());
|
||||||
export const matchEventsByServer = writable<Map<string, GameTelemetry>>(new Map());
|
export const matchEventsByServer = writable<Map<string, GameTelemetry>>(new Map());
|
||||||
|
|
||||||
export interface RosterPlayer {
|
export type { StatusPlayer as RosterPlayer } from '$lib/server-observation.js';
|
||||||
num: number | null;
|
import type { StatusPlayer as RosterPlayer, GameObservation } from '$lib/server-observation.js';
|
||||||
score: number;
|
|
||||||
ping: number;
|
|
||||||
guid: string | null;
|
|
||||||
name: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type Health = 'online' | 'stale' | 'offline' | 'starting' | 'provisioning';
|
export type Health = 'online' | 'stale' | 'offline' | 'starting' | 'provisioning';
|
||||||
export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
|
export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
|
||||||
@@ -23,6 +18,11 @@ export interface RconEndpoint {
|
|||||||
|
|
||||||
export interface ServerStatus {
|
export interface ServerStatus {
|
||||||
server: string;
|
server: string;
|
||||||
|
containerState?: string | null;
|
||||||
|
containerCheckedAt?: string | null;
|
||||||
|
telemetryAfter?: string | null;
|
||||||
|
rconStatus?: GameObservation;
|
||||||
|
telemetry?: GameTelemetry;
|
||||||
players: number | null;
|
players: number | null;
|
||||||
map: string | null;
|
map: string | null;
|
||||||
gametype: string | null;
|
gametype: string | null;
|
||||||
|
|||||||
@@ -204,7 +204,7 @@
|
|||||||
{#if view === 'cards'}
|
{#if view === 'cards'}
|
||||||
<div class="cards">{#each group.servers as server (server.name)}
|
<div class="cards">{#each group.servers as server (server.name)}
|
||||||
<article class="server-tile" class:pinned={pinned.includes(server.name)}>
|
<article class="server-tile" class:pinned={pinned.includes(server.name)}>
|
||||||
<ServerCard name={server.name} players={server.last_players} map={server.last_map ?? server.map}
|
<ServerCard name={server.name} telemetry={server.telemetry} players={server.last_players} map={server.last_map ?? server.map}
|
||||||
gametype={server.last_gametype} roster={server.roster} health={server.health}
|
gametype={server.last_gametype} roster={server.roster} health={server.health}
|
||||||
lastSeenAt={server.last_seen_at} lastError={server.last_error} held={server.held}
|
lastSeenAt={server.last_seen_at} lastError={server.last_error} held={server.held}
|
||||||
activeSince={server.active_since} inactiveSince={server.inactive_since}
|
activeSince={server.active_since} inactiveSince={server.inactive_since}
|
||||||
|
|||||||
@@ -2,38 +2,16 @@ import { db } from '$lib/server/db/index.js';
|
|||||||
import { kick } from '$lib/server/pipeline/worker.js';
|
import { kick } from '$lib/server/pipeline/worker.js';
|
||||||
import { json } from '@sveltejs/kit';
|
import { json } from '@sveltejs/kit';
|
||||||
import { listServers, heldByBeta } from '$lib/server/fleet/servers.js';
|
import { listServers, heldByBeta } from '$lib/server/fleet/servers.js';
|
||||||
import { classifyHealth } from '$lib/server/fleet/health.js';
|
import { serverObservation } from '$lib/server/fleet/status.js';
|
||||||
import { isProvisioning } from '$lib/server/fleet/provisioning.js';
|
|
||||||
import { queueCreateServer } from '$lib/server/provisioning/index.js';
|
import { queueCreateServer } from '$lib/server/provisioning/index.js';
|
||||||
import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
|
import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
|
||||||
import type { RconPlayer } from '$lib/server/integrations/rcon.js';
|
|
||||||
import type { GameMode, ZombieGame } from '$lib/server/fleet/map-aliases.js';
|
import type { GameMode, ZombieGame } from '$lib/server/fleet/map-aliases.js';
|
||||||
|
|
||||||
function safeJson<T>(raw: string | null): T | null {
|
|
||||||
if (!raw) return null;
|
|
||||||
try {
|
|
||||||
return JSON.parse(raw) as T;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function GET() {
|
export async function GET() {
|
||||||
const rconHost = config.rcon.host;
|
const rconHost = config.rcon.host;
|
||||||
const servers = listServers().map((s) => {
|
const servers = listServers().map((s) => {
|
||||||
const provisioning = isProvisioning(s.name);
|
|
||||||
const ready = !provisioning && (!s.container_state || s.container_state === 'running');
|
|
||||||
return {
|
return {
|
||||||
...s,
|
...serverObservation(s),
|
||||||
last_players: ready ? s.last_players : null,
|
|
||||||
last_error: ready ? s.last_error : null,
|
|
||||||
active_since: ready ? s.active_since : null,
|
|
||||||
inactive_since: ready ? s.inactive_since : null,
|
|
||||||
game_started_at: ready ? s.game_started_at : null,
|
|
||||||
game_ended_at: ready ? s.game_ended_at : null,
|
|
||||||
roster: ready ? safeJson<RconPlayer[]>(s.last_roster) : null,
|
|
||||||
health: provisioning ? 'provisioning' : classifyHealth(s.last_seen_at, s.last_error, s.container_state),
|
|
||||||
replySnippet: ready ? s.last_reply_snippet : null,
|
|
||||||
// Derived endpoint — same logic as endpointFor() but without asking
|
// Derived endpoint — same logic as endpointFor() but without asking
|
||||||
// Komodo. If we don't have a port yet, endpoint is null.
|
// Komodo. If we don't have a port yet, endpoint is null.
|
||||||
endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
|
endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
|
||||||
|
|||||||
@@ -2,13 +2,15 @@ import { json } from '@sveltejs/kit';
|
|||||||
import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
|
import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
|
||||||
import { listServers } from '$lib/server/fleet/servers.js';
|
import { listServers } from '$lib/server/fleet/servers.js';
|
||||||
import { listServerKeys } from '$lib/server/fleet/server-keys.js';
|
import { listServerKeys } from '$lib/server/fleet/server-keys.js';
|
||||||
import { classifyHealth } from '$lib/server/fleet/health.js';
|
import { serverObservation } from '$lib/server/fleet/status.js';
|
||||||
|
import { freshStatusTimestamp } from '$lib/server-freshness.js';
|
||||||
import { tcpProbe } from '$lib/server/integrations/netProbe.js';
|
import { tcpProbe } from '$lib/server/integrations/netProbe.js';
|
||||||
import type { SetupCheck, SystemStatus } from '$lib/system-status.js';
|
import type { SetupCheck, SystemStatus } from '$lib/system-status.js';
|
||||||
|
|
||||||
export async function GET({ url }: { url: URL }) {
|
export async function GET({ url }: { url: URL }) {
|
||||||
const servers = listServers();
|
const servers = listServers();
|
||||||
const readyServers = servers.filter(server => classifyHealth(server.last_seen_at, server.last_error, server.container_state) === 'online').length;
|
const readyServers = servers.filter(server => serverObservation(server).health === 'online').length;
|
||||||
|
const rconServers = servers.filter(server => !server.last_error && freshStatusTimestamp(server.last_seen_at, Date.now()) && server.container_state === 'running').length;
|
||||||
const savedKeys = listServerKeys().length;
|
const savedKeys = listServerKeys().length;
|
||||||
const missing = [!config.komodo.address && 'KOMODO_ADDRESS', !config.komodo.apiKey && 'KOMODO_API_KEY', !config.komodo.apiSecret && 'KOMODO_API_SECRET'].filter(Boolean);
|
const missing = [!config.komodo.address && 'KOMODO_ADDRESS', !config.komodo.apiKey && 'KOMODO_API_KEY', !config.komodo.apiSecret && 'KOMODO_API_SECRET'].filter(Boolean);
|
||||||
const connection: SetupCheck = { id: 'komodo', title: 'Komodo connection', state: komodoConfigured() ? 'unverified' : 'unconfigured', detail: komodoConfigured() ? 'Credentials configured. Run connection check to verify host reachability.' : `Set ${missing.join(', ')} in the app environment.`, href: '#setup-guide', action: 'Setup instructions' };
|
const connection: SetupCheck = { id: 'komodo', title: 'Komodo connection', state: komodoConfigured() ? 'unverified' : 'unconfigured', detail: komodoConfigured() ? 'Credentials configured. Run connection check to verify host reachability.' : `Set ${missing.join(', ')} in the app environment.`, href: '#setup-guide', action: 'Setup instructions' };
|
||||||
@@ -28,7 +30,7 @@ export async function GET({ url }: { url: URL }) {
|
|||||||
const checks: SetupCheck[] = [
|
const checks: SetupCheck[] = [
|
||||||
connection,
|
connection,
|
||||||
{ id: 'target', title: 'Deployment host', state: config.komodo.targetServer ? 'ready' : 'unconfigured', detail: config.komodo.targetServer ? `New servers target ${config.komodo.targetServer}.` : 'Set KOMODO_TARGET_SERVER to the exact host name shown in Komodo.', href: '#setup-guide', action: 'Setup instructions' },
|
{ id: 'target', title: 'Deployment host', state: config.komodo.targetServer ? 'ready' : 'unconfigured', detail: config.komodo.targetServer ? `New servers target ${config.komodo.targetServer}.` : 'Set KOMODO_TARGET_SERVER to the exact host name shown in Komodo.', href: '#setup-guide', action: 'Setup instructions' },
|
||||||
{ id: 'rcon', title: 'Game server connection', state: !rconConfigured() ? 'unconfigured' : readyServers ? 'ready' : 'unverified', detail: !rconConfigured() ? 'Set RCON_PASSWORD to the password used by your game servers.' : readyServers ? `${readyServers} servers have fresh successful RCON reports.` : 'RCON password configured. A running server is needed to verify a response.', href: servers.length ? '/' : '#setup-guide', action: servers.length ? 'Review fleet' : 'Setup instructions' },
|
{ id: 'rcon', title: 'Game server connection', state: !rconConfigured() ? 'unconfigured' : rconServers ? 'ready' : 'unverified', detail: !rconConfigured() ? 'Set RCON_PASSWORD to the password used by your game servers.' : rconServers ? `${rconServers} servers have fresh successful RCON reports.` : 'RCON password configured. A running server is needed to verify a response.', href: servers.length ? '/' : '#setup-guide', action: servers.length ? 'Review fleet' : 'Setup instructions' },
|
||||||
{ id: 'game-files', title: 'Game files on the host', state: 'unverified', detail: 'T5 and T6 game-file volumes must be populated on the Komodo host. Their contents cannot be verified from this app.', href: '#game-files', action: 'Game-file instructions' },
|
{ id: 'game-files', title: 'Game files on the host', state: 'unverified', detail: 'T5 and T6 game-file volumes must be populated on the Komodo host. Their contents cannot be verified from this app.', href: '#game-files', action: 'Game-file instructions' },
|
||||||
{ id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Key presence does not confirm validity with Plutonium.`, href: '/server-keys', action: savedKeys ? 'Manage keys' : 'Add a key' },
|
{ id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Key presence does not confirm validity with Plutonium.`, href: '/server-keys', action: savedKeys ? 'Manage keys' : 'Add a key' },
|
||||||
{ id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Servers are registered but none has a fresh healthy report. Open diagnostics to investigate.' : 'Create your first server after completing the setup checks.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' }
|
{ id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Servers are registered but none has a fresh healthy report. Open diagnostics to investigate.' : 'Create your first server after completing the setup checks.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' }
|
||||||
|
|||||||
@@ -70,6 +70,10 @@
|
|||||||
status?: { players: number; map: string | null; gametype: string | null; roster: RosterPlayer[] };
|
status?: { players: number; map: string | null; gametype: string | null; roster: RosterPlayer[] };
|
||||||
}
|
}
|
||||||
interface Server {
|
interface Server {
|
||||||
|
telemetry_after?: string | null;
|
||||||
|
status_source?: 'telemetry' | 'rcon' | null;
|
||||||
|
rcon_status?: import('$lib/server-observation.js').GameObservation;
|
||||||
|
telemetry?: import('$lib/telemetry.js').GameTelemetry;
|
||||||
name: string;
|
name: string;
|
||||||
komodo_type: string; komodo_name: string; komodo_server: string | null;
|
komodo_type: string; komodo_name: string; komodo_server: string | null;
|
||||||
game_port: number | null; rcon_host: string | null;
|
game_port: number | null; rcon_host: string | null;
|
||||||
@@ -145,10 +149,11 @@
|
|||||||
let effectivePlayers = $derived(fresh ? server?.last_players ?? null : null);
|
let effectivePlayers = $derived(fresh ? server?.last_players ?? null : null);
|
||||||
let effectiveMap = $derived(server?.last_map ?? null);
|
let effectiveMap = $derived(server?.last_map ?? null);
|
||||||
let effectiveGametype = $derived(server?.last_gametype ?? null);
|
let effectiveGametype = $derived(server?.last_gametype ?? null);
|
||||||
const scoreboard = $derived($telemetryByServer.get(name));
|
const scoreboard = $derived(server?.telemetry ?? $telemetryByServer.get(name));
|
||||||
let effectiveHealth = $derived<Health>(server?.health === 'online' && !fresh ? 'stale' : server?.health ?? 'offline');
|
let effectiveHealth = $derived<Health>(server?.health === 'online' && !fresh ? 'stale' : server?.health ?? 'offline');
|
||||||
let effectiveSeen = $derived(server?.last_seen_at ?? null);
|
let effectiveSeen = $derived(server?.last_seen_at ?? null);
|
||||||
let effectiveErr = $derived(server?.last_error ?? null);
|
let effectiveErr = $derived(effectiveHealth === 'starting' || effectiveHealth === 'provisioning'
|
||||||
|
? null : server?.rcon_status?.last_error ?? server?.last_error ?? null);
|
||||||
let effectiveEndpoint = $derived<RconEndpoint | null>(server?.endpoint ?? null);
|
let effectiveEndpoint = $derived<RconEndpoint | null>(server?.endpoint ?? null);
|
||||||
let decoded = $derived(decodeRconError(effectiveErr));
|
let decoded = $derived(decodeRconError(effectiveErr));
|
||||||
let effectiveReason = $derived<RconFailure | null>(decoded?.reason ?? null);
|
let effectiveReason = $derived<RconFailure | null>(decoded?.reason ?? null);
|
||||||
@@ -217,7 +222,7 @@
|
|||||||
}
|
}
|
||||||
// Tab config — Diagnostics gets a warning badge when there's an issue.
|
// Tab config — Diagnostics gets a warning badge when there's an issue.
|
||||||
let tabs = $derived([
|
let tabs = $derived([
|
||||||
{ id: 'scoreboard', label: 'Scoreboard', badge: scoreboard?.players.length ?? null },
|
{ id: 'scoreboard', label: 'Scoreboard', badge: null },
|
||||||
{ id: 'chat', label: 'Chat' },
|
{ id: 'chat', label: 'Chat' },
|
||||||
{ id: 'matches', label: 'Matches' },
|
{ id: 'matches', label: 'Matches' },
|
||||||
{ id: 'logs', label: 'Logs' },
|
{ id: 'logs', label: 'Logs' },
|
||||||
@@ -296,7 +301,7 @@
|
|||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{#if effectiveHealth !== 'starting' && effectiveHealth !== 'provisioning'}<RoundTelemetry {name} />{/if}
|
{#if effectiveHealth === 'online'}<RoundTelemetry {name} report={scoreboard} since={server?.telemetry_after ?? null} />{/if}
|
||||||
|
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
@@ -315,7 +320,7 @@
|
|||||||
{#if tabError}<div class="ui-error" role="alert">{tabError}<Button size="sm" onclick={() => loadTab(name, activeTab)}>Retry</Button></div>{/if}
|
{#if tabError}<div class="ui-error" role="alert">{tabError}<Button size="sm" onclick={() => loadTab(name, activeTab)}>Retry</Button></div>{/if}
|
||||||
{#if activeTab === 'scoreboard'}
|
{#if activeTab === 'scoreboard'}
|
||||||
<Card padding="lg">
|
<Card padding="lg">
|
||||||
{#key name}<PlayerScoreboard game={fleetIdentity(server).game} report={scoreboard} mapLabel={scoreboard ? fleetIdentity({ ...server, last_map: scoreboard.map }).map : undefined} />{/key}
|
{#key name}<PlayerScoreboard current={fresh} after={server?.telemetry_after ?? null} game={fleetIdentity(server).game} report={scoreboard} mapLabel={scoreboard ? fleetIdentity({ ...server, last_map: scoreboard.map }).map : undefined} />{/key}
|
||||||
</Card>
|
</Card>
|
||||||
{:else if activeTab === 'chat'}
|
{:else if activeTab === 'chat'}
|
||||||
<Card padding="lg">
|
<Card padding="lg">
|
||||||
@@ -373,6 +378,7 @@
|
|||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<p class="sub">Current player count: {server.status_source === 'telemetry' ? 'game telemetry' : server.status_source === 'rcon' ? 'RCON' : 'unavailable'}. RCON probes below show that connection's separate results.</p>
|
||||||
{#if integrations}
|
{#if integrations}
|
||||||
<div class="diag-grid">
|
<div class="diag-grid">
|
||||||
<div class="kv"><span class="k">Endpoint</span><span class="v mono">{endpointLabel(effectiveEndpoint)}</span></div>
|
<div class="kv"><span class="k">Endpoint</span><span class="v mono">{endpointLabel(effectiveEndpoint)}</span></div>
|
||||||
@@ -422,7 +428,7 @@
|
|||||||
<div class="alert alert-ok">
|
<div class="alert alert-ok">
|
||||||
<Badge variant="online" uppercase>Connected</Badge>
|
<Badge variant="online" uppercase>Connected</Badge>
|
||||||
<span class="alert-detail">
|
<span class="alert-detail">
|
||||||
{rconProbe.status?.players ?? '?'} players
|
RCON: {rconProbe.status?.players ?? '?'} players
|
||||||
{#if rconProbe.status?.map} · map <strong>{rconProbe.status.map}</strong>{/if}
|
{#if rconProbe.status?.map} · map <strong>{rconProbe.status.map}</strong>{/if}
|
||||||
{#if rconProbe.status?.gametype} · {rconProbe.status.gametype}{/if}
|
{#if rconProbe.status?.gametype} · {rconProbe.status.gametype}{/if}
|
||||||
</span>
|
</span>
|
||||||
|
|||||||
@@ -155,6 +155,9 @@ describe('durable beta test plans', () => {
|
|||||||
const { getBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
const { getBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||||
expect(getBeta(id)?.state).toBe('ready-promote');
|
expect(getBeta(id)?.state).toBe('ready-promote');
|
||||||
await tick(15, 0, false);
|
await tick(15, 0, false);
|
||||||
|
// Fresh occupied telemetry remains healthy despite an RCON timeout.
|
||||||
|
expect(getBeta(id)?.state).toBe('ready-promote');
|
||||||
|
await tick(31, 0, false);
|
||||||
expect(getBeta(id)?.state).toBe('running');
|
expect(getBeta(id)?.state).toBe('running');
|
||||||
await expect(promoteBeta(id)).rejects.toThrow();
|
await expect(promoteBeta(id)).rejects.toThrow();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,6 +2,21 @@ import { describe, expect, it } from 'vitest';
|
|||||||
import { extractUdpPort, extractContainerState } from '../../src/lib/server/integrations/komodo.js';
|
import { extractUdpPort, extractContainerState } from '../../src/lib/server/integrations/komodo.js';
|
||||||
|
|
||||||
describe('container readiness', () => {
|
describe('container readiness', () => {
|
||||||
|
it.each([
|
||||||
|
['Up 35 seconds (health: starting)', 'starting'],
|
||||||
|
['Up about a minute (healthy)', 'running'],
|
||||||
|
['Up 5 minutes (unhealthy)', 'unhealthy'],
|
||||||
|
['Up 2 hours', 'running'],
|
||||||
|
['Exited (1) 5 seconds ago', 'exited']
|
||||||
|
])('reads nested Komodo container status: %s', (status, expected) => {
|
||||||
|
expect(extractContainerState({ state: 'Running', container: { state: 'running', status } })).toBe(expected);
|
||||||
|
});
|
||||||
|
it('reads lowercase inspect health and does not trust missing containers or configuration', () => {
|
||||||
|
expect(extractContainerState({ state: { status: 'running', running: true, health: { status: 'starting' } } })).toBe('starting');
|
||||||
|
expect(extractContainerState({ state: 'Running', container: null })).toBe('unknown');
|
||||||
|
expect(extractContainerState({ config: { state: 'running' }, status: 'ok' })).toBeNull();
|
||||||
|
expect(extractContainerState({ state: 'Exited', container: { status: 'Up 1 minute (healthy)' } })).toBe('exited');
|
||||||
|
});
|
||||||
it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => {
|
it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => {
|
||||||
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health);
|
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health);
|
||||||
expect(extractContainerState({ state: 'running', health })).toBe(health);
|
expect(extractContainerState({ state: 'running', health })).toBe(health);
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { freshTelemetry, selectServerObservation, type ServerObservation } from '../../src/lib/server-observation.js';
|
||||||
|
import { gameEvent } from '../helpers/telemetry.js';
|
||||||
|
|
||||||
|
const now = Date.parse('2026-09-27T12:00:00Z');
|
||||||
|
const at = (offset = 0) => new Date(now + offset).toISOString();
|
||||||
|
const report = (offset = 0, players = 3) => ({ ...gameEvent('heartbeat', 15, players, 'managed'), receivedAt: at(offset) });
|
||||||
|
function server(changes: Partial<ServerObservation> = {}): ServerObservation {
|
||||||
|
return { name: 'managed', health: 'online', container_state: 'running', container_checked_at: at(),
|
||||||
|
last_players: 4, last_map: 'zombie_theater', last_gametype: 'zom', last_seen_at: at(),
|
||||||
|
last_error: null, roster: [], ...changes };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('shared server observation', () => {
|
||||||
|
it('selects the telemetry count and roster together and keeps raw RCON evidence', () => {
|
||||||
|
const raw = server({ last_error: 'timeout' });
|
||||||
|
const selected = selectServerObservation(server({ rcon_status: raw }), report(), now);
|
||||||
|
expect(selected).toMatchObject({ health: 'online', status_source: 'telemetry', last_players: 3, last_error: null });
|
||||||
|
expect(selected.roster).toHaveLength(3);
|
||||||
|
expect(selected.roster?.[0]).toMatchObject({ ping: null, score: 5000 });
|
||||||
|
expect(selected.rcon_status).toBe(raw);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the latest HTTP or streamed telemetry and falls back without extending freshness', () => {
|
||||||
|
const original = server();
|
||||||
|
const selected = selectServerObservation(server({ rcon_status: original, telemetry: report(0, 2) }), report(-1000), now);
|
||||||
|
expect(selected.last_players).toBe(2);
|
||||||
|
const fallback = selectServerObservation({ ...selected, container_checked_at: at(46_000),
|
||||||
|
rcon_status: { ...original, last_seen_at: at(40_000) } }, undefined, now + 46_000);
|
||||||
|
expect(fallback).toMatchObject({ status_source: 'rcon', health: 'online', last_players: 4, last_seen_at: at(40_000) });
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(['starting', 'restarting', 'unhealthy', 'exited', 'unknown'])('telemetry cannot override container state %s', state => {
|
||||||
|
const selected = selectServerObservation(server({ health: state === 'starting' || state === 'restarting' ? 'starting' : 'offline', container_state: state }), report(), now);
|
||||||
|
expect(selected.health).not.toBe('online');
|
||||||
|
expect(selected.last_players).toBeNull();
|
||||||
|
expect(selected.roster).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('requires fresh container evidence and reports after the latest lifecycle submission', () => {
|
||||||
|
expect(selectServerObservation(server({ container_checked_at: at(-46_000) }), report(), now).health).not.toBe('online');
|
||||||
|
expect(selectServerObservation(server({ health: 'provisioning' }), report(), now).health).toBe('provisioning');
|
||||||
|
expect(selectServerObservation(server({ telemetry_after: at() }), report(-1), now).health).toBe('offline');
|
||||||
|
expect(freshTelemetry(report(), now, 'invalid')).toBe(false);
|
||||||
|
expect(freshTelemetry(report(2000), now)).toBe(false);
|
||||||
|
expect(freshTelemetry(report(-46_000), now)).toBe(false);
|
||||||
|
expect(freshTelemetry({ ...report(), players: [{}] } as never, now)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses game-start rosters without inventing scores and clears match times on an empty report', () => {
|
||||||
|
const start = { ...gameEvent('game_start', 0, 2, 'managed'), receivedAt: at() };
|
||||||
|
const active = selectServerObservation(server(), start, now);
|
||||||
|
expect(active.roster?.[0]).toMatchObject({ score: null, ping: null });
|
||||||
|
expect(active.game_started_at).toBe(at());
|
||||||
|
expect(selectServerObservation({ ...active, game_ended_at: at() }, report(1, 0), now + 1))
|
||||||
|
.toMatchObject({ last_players: 0, roster: [], game_started_at: null, game_ended_at: null });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -76,6 +76,9 @@ describe('database fleet and container readiness', () => {
|
|||||||
await postTelemetry(gameEvent('game_end', 10, 1, 'managed'));
|
await postTelemetry(gameEvent('game_end', 10, 1, 'managed'));
|
||||||
rcon.mockResolvedValue(reply(0));
|
rcon.mockResolvedValue(reply(0));
|
||||||
await fanoutStatus();
|
await fanoutStatus();
|
||||||
|
expect(getServer('managed')?.game_ended_at).not.toBeNull();
|
||||||
|
await postTelemetry(gameEvent('heartbeat', 0, 0, 'managed'));
|
||||||
|
await fanoutStatus();
|
||||||
expect(getServer('managed')).toMatchObject({ game_started_at: null, game_ended_at: null });
|
expect(getServer('managed')).toMatchObject({ game_started_at: null, game_ended_at: null });
|
||||||
rcon.mockResolvedValue(reply(1));
|
rcon.mockResolvedValue(reply(1));
|
||||||
await fanoutStatus();
|
await fanoutStatus();
|
||||||
@@ -118,6 +121,41 @@ describe('database fleet and container readiness', () => {
|
|||||||
expect(snapshot.serverNow).toBeLessThanOrEqual(Date.now());
|
expect(snapshot.serverNow).toBeLessThanOrEqual(Date.now());
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('uses telemetry consistently in REST, status events, and metrics without polling RCON', async () => {
|
||||||
|
const { db, inspect, rcon, fanoutStatus } = await setup();
|
||||||
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
||||||
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
||||||
|
db.prepare("UPDATE servers SET last_players = 4, last_seen_at = ? WHERE name = 'managed'").run(new Date().toISOString());
|
||||||
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
||||||
|
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
||||||
|
const events: unknown[] = [];
|
||||||
|
const unsubscribe = subscribe(event => events.push(event));
|
||||||
|
try { await fanoutStatus(); } finally { unsubscribe(); }
|
||||||
|
expect(rcon).not.toHaveBeenCalled();
|
||||||
|
const server = (await (await GET()).json()).servers[0];
|
||||||
|
expect(server).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry', last_error: null,
|
||||||
|
rcon_status: { last_players: 4, last_error: 'timeout:old process' } });
|
||||||
|
expect(server.roster).toHaveLength(3);
|
||||||
|
expect(server.roster[0]).toMatchObject({ ping: null, score: 5000 });
|
||||||
|
expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
|
||||||
|
health: 'online', players: 3, roster: server.roster, lastSeenAt: server.last_seen_at
|
||||||
|
})] }));
|
||||||
|
expect(db.prepare("SELECT players FROM metric_samples WHERE server = 'managed'").get()).toEqual({ players: 3 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to RCON when telemetry expires and does not renew old reports', async () => {
|
||||||
|
const { db, inspect, rcon, fanoutStatus } = await setup();
|
||||||
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
||||||
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
||||||
|
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
||||||
|
db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 46_000).toISOString());
|
||||||
|
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
||||||
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
||||||
|
await fanoutStatus();
|
||||||
|
expect(rcon).toHaveBeenCalledTimes(1);
|
||||||
|
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_players: 0, status_source: 'rcon' });
|
||||||
|
});
|
||||||
|
|
||||||
it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => {
|
it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => {
|
||||||
const { db } = await setup();
|
const { db } = await setup();
|
||||||
const { allServers, listServers } = await import('../../src/lib/server/fleet/servers.js');
|
const { allServers, listServers } = await import('../../src/lib/server/fleet/servers.js');
|
||||||
@@ -151,8 +189,11 @@ describe('database fleet and container readiness', () => {
|
|||||||
it.each(['unhealthy', 'exited', undefined])('does not poll RCON with container state %s', async (state) => {
|
it.each(['unhealthy', 'exited', undefined])('does not poll RCON with container state %s', async (state) => {
|
||||||
const { inspect, rcon, fanoutStatus } = await setup();
|
const { inspect, rcon, fanoutStatus } = await setup();
|
||||||
inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' });
|
inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' });
|
||||||
|
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
||||||
await fanoutStatus();
|
await fanoutStatus();
|
||||||
expect(rcon).not.toHaveBeenCalled();
|
expect(rcon).not.toHaveBeenCalled();
|
||||||
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
||||||
|
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'offline', last_players: null });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('waits for an EC restart request to settle before trusting the old healthy container', async () => {
|
it('waits for an EC restart request to settle before trusting the old healthy container', async () => {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|||||||
import { get } from 'svelte/store';
|
import { get } from 'svelte/store';
|
||||||
import { freshServer, matchesFleet, type FleetServer } from '../../src/lib/fleet-overview.js';
|
import { freshServer, matchesFleet, type FleetServer } from '../../src/lib/fleet-overview.js';
|
||||||
import type { ServerStatus } from '../../src/lib/ui/stores/servers.js';
|
import type { ServerStatus } from '../../src/lib/ui/stores/servers.js';
|
||||||
|
import { gameEvent } from '../helpers/telemetry.js';
|
||||||
|
|
||||||
vi.mock('$app/environment', () => ({ browser: true }));
|
vi.mock('$app/environment', () => ({ browser: true }));
|
||||||
|
|
||||||
@@ -65,6 +66,20 @@ async function setup() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe('Fleet report clock', () => {
|
describe('Fleet report clock', () => {
|
||||||
|
it('updates count and roster immediately from telemetry and expires them together', async () => {
|
||||||
|
snapshot.servers = [server(epoch, { container_state: 'running', container_checked_at: new Date(epoch).toISOString(), last_players: 4 })];
|
||||||
|
const { fleetServers, uiNow } = await setup();
|
||||||
|
await vi.advanceTimersByTimeAsync(1000);
|
||||||
|
source.dispatchEvent(new MessageEvent('server-telemetry', { data: JSON.stringify({ serverNow: epoch + 1000,
|
||||||
|
entries: [{ ...gameEvent('heartbeat', 15, 3), receivedAt: new Date(epoch + 1000).toISOString() }] }) }));
|
||||||
|
expect(get(fleetServers)[0]).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry' });
|
||||||
|
expect(get(fleetServers)[0].roster).toHaveLength(3);
|
||||||
|
expect(matchesFleet(get(fleetServers)[0], 'Player 3', 'occupied', '', [], [], get(uiNow))).toBe(true);
|
||||||
|
await vi.advanceTimersByTimeAsync(46_000);
|
||||||
|
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
|
||||||
|
expect(matchesFleet(get(fleetServers)[0], '', 'occupied', '', [], [], get(uiNow))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it('applies game lifecycle events immediately and clears them after an empty report', async () => {
|
it('applies game lifecycle events immediately and clears them after an empty report', async () => {
|
||||||
const { fleetServers } = await setup();
|
const { fleetServers } = await setup();
|
||||||
const report = (event: string, time: number) => source.dispatchEvent(new MessageEvent('server-telemetry', {
|
const report = (event: string, time: number) => source.dispatchEvent(new MessageEvent('server-telemetry', {
|
||||||
|
|||||||
Reference in New Issue
Block a user