josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: honor Komodo health and prefer fresh telemetry across fleet status

Browse Source
This commit is contained in:
josh
2026-09-27 15:34:30 -04:00
parent 46a89ea4b0
commit 8ea696417f
27 changed files with 488 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/beta-tests.ts
+12 -8
View File
@@ -1,3 +1,5 @@
import { getServer } from './servers.js';
import { serverObservation } from './status.js';
import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
@@ -52,17 +54,19 @@ export function recordGameEvent(event: GameTelemetry): void {
export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
const m = readMetrics(run);
const server = db().prepare('SELECT last_checked_at, last_seen_at, last_error, container_state FROM servers WHERE name = ?').get(run.server) as {
last_checked_at: string | null; last_seen_at: string | null; last_error: string | null; container_state: string | null };
const sample = server.last_checked_at ? Date.parse(server.last_checked_at) : 0;
const freshSample = sample >= Date.parse(run.observing_at ?? run.created_at) && Date.now() - sample <= 60_000;
const healthy = freshSample && !server.last_error && server.container_state === 'running' && server.last_seen_at === server.last_checked_at;
const at = freshSample && server.container_state === 'running' ? sample : Date.now();
const row = getServer(run.server);
const server = row ? serverObservation(row) : null;
const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
const sample = Date.parse(timestamp ?? '');
const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
&& Date.now() - sample >= 0 && Date.now() - sample <= 45_000;
const healthy = freshSample && server?.health === 'online';
const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
const before = m.wasHealthy;
const lastSample = m.lastSample;
applySample(plan, m, at, healthy);
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: server.last_checked_at });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server.last_error });
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, source: server?.status_source });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
saveMetrics(run, m);
}
src/lib/server/fleet/collections.ts
+3 -3
View File
@@ -7,7 +7,7 @@ import { creationQueue, destroyServer, nextAvailablePort, nextLobbyNumber, queue
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
import { housePartyTargets } from './house-party.js';
import { getServerKey } from './server-keys.js';
import { classifyHealth } from './health.js';
import { serverObservation } from './status.js';
import { getServer } from './servers.js';
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
@@ -37,8 +37,8 @@ function coverage(entries: BlueprintEntry[]) {
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
: server?.container_state && !['running', 'starting', 'restarting'].includes(server.container_state) ? 'stopped'
: classifyHealth(server?.last_seen_at ?? null, server?.last_error ?? null, server?.container_state),
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
: server ? serverObservation(server).health : 'offline',
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
});
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
src/lib/server/fleet/servers.ts
+2
View File
@@ -68,6 +68,7 @@ export interface ServerRow {
komodo_name: string;
komodo_server: string | null;
container_state: string | null;
container_checked_at: string | null;
desired_state: 'running' | 'stopped';
game_port: number | null;
rcon_host: string | null;
@@ -162,6 +163,7 @@ export function setServerField(
'komodo_name',
'komodo_server',
'container_state',
'container_checked_at',
'game_port',
'rcon_host',
'game',
src/lib/server/fleet/status-evidence.ts
+29 -10
View File
@@ -1,28 +1,43 @@
import { db } from '../db/index.js';
import { parseGameEvent } from '../../telemetry.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
export interface TelemetryEvidence { players: number; map: string; receivedAt: string }
/** Only validated, recently received reports can establish current game state. */
export function telemetryEvidence(server: string, after?: string | null): TelemetryEvidence | null {
/** Reports from before the latest lifecycle submission describe the old process. */
export function latestGameSubmission(server: string): string | null {
return (db().prepare(`SELECT MAX(at) AS at FROM (
SELECT submitted_at AS at FROM server_deployments WHERE server=? AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
}
export function telemetryReport(server: string): GameTelemetry | null {
const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as
{ received_at: string; payload: string | null } | undefined;
if (!row?.payload) return null;
const received = Date.parse(row.received_at);
try {
const saved = JSON.parse(row.payload) as GameTelemetry;
const event = parseGameEvent(saved);
return event.server === server ? { ...event, receivedAt: row.received_at, timing: saved.timing } : null;
} catch { return null; }
}
/** Only validated, recently received reports can establish current game state. */
export function telemetryEvidence(server: string, after: string | null = latestGameSubmission(server)): TelemetryEvidence | null {
const report = telemetryReport(server);
if (!report) return null;
const received = Date.parse(report.receivedAt);
const age = Date.now() - received;
if (!Number.isFinite(age) || age < 0 || age > ONLINE_MAX_MS) return null;
if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after))) return null;
try {
const event = parseGameEvent(JSON.parse(row.payload));
if (event.server !== server) return null;
return { players: event.players.length, map: event.map, receivedAt: row.received_at };
} catch { return null; }
return { players: report.players.length, map: report.map, receivedAt: report.receivedAt };
}
/** Positive occupancy from either source always vetoes an empty-server claim. */
export async function probePlayerCount(server: string): Promise<number | null> {
const occupied = telemetryEvidence(server);
if (occupied && occupied.players > 0) return occupied.players;
const players = await rconGetPlayerCount(server).catch(() => null);
// Read after the network request: a player may join while RCON is pending.
const telemetry = telemetryEvidence(server);
@@ -35,10 +50,14 @@ export async function probePlayerCount(server: string): Promise<number | null> {
export async function probeServerStatus(server: string, after?: string | null): Promise<
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry' } | { ok: false; reason: string }
> {
const current = telemetryEvidence(server, after);
if (current) return { ok: true, ...current, source: 'telemetry' };
let reason = 'RCON unavailable';
try {
const rcon = await rconStatusResult(server);
if (rcon.ok) return { ok: true, players: Math.max(rcon.status.players, telemetryEvidence(server, after)?.players ?? 0), map: rcon.status.map, source: 'rcon' };
const latest = telemetryEvidence(server, after);
if (latest) return { ok: true, ...latest, source: 'telemetry' };
if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon' };
reason = rcon.reason;
} catch (err) { reason = (err as Error).message; }
const telemetry = telemetryEvidence(server, after);
src/lib/server/fleet/status.ts
+27
View File
@@ -0,0 +1,27 @@
import { selectServerObservation, type GameObservation } from '../../server-observation.js';
import { classifyHealth } from './health.js';
import { isProvisioning } from './provisioning.js';
import { restartSettling } from '../provisioning/restarts.js';
import { type ServerRow } from './servers.js';
import { telemetryReport, latestGameSubmission } from './status-evidence.js';
/** Shared snapshot for REST, status events, metrics, and other server-side readers. */
export function serverObservation(server: ServerRow) {
let roster = null;
try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
const provisioning = isProvisioning(server.name);
const state = restartSettling(server.name) ? 'restarting' : server.container_state;
const ready = !provisioning && (!state || state === 'running');
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
status_source: null as 'telemetry' | 'rcon' | null,
telemetry: telemetryReport(server.name) ?? undefined,
telemetry_after: latestGameSubmission(server.name),
last_error: ready ? server.last_error : null,
replySnippet: ready ? server.last_reply_snippet : null,
active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null,
game_started_at: ready ? server.game_started_at : null, game_ended_at: ready ? server.game_ended_at : null
}, undefined, Date.now());
}