josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify telemetry to three game events

Browse Source
This commit is contained in:
josh
2026-09-18 15:35:21 -04:00
parent 6d903adb31
commit 5bb49a776e
29 changed files with 658 additions and 1314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/beta-plan-runner.ts
+1 -1
View File
@@ -112,7 +112,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
if (oldRun) {
const previous = readMetrics(oldRun);
const metrics = readMetrics(run);
metrics.outages = previous.outages; metrics.logErrors = previous.logErrors; metrics.restarts = previous.restarts;
metrics.outages = previous.outages;
saveMetrics(run, metrics);
}
const restart = await restartInRollout(server.server);
src/lib/server/pipeline/beta-runner.ts
+3 -5
View File
@@ -3,7 +3,6 @@ import { emit } from './bus.js';
import type { BetaStage } from './bus.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { scanForMatchEnd } from '../fleet/match-log.js';
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
import { plannedBetaTick } from './beta-plan-runner.js';
@@ -12,6 +11,7 @@ interface BetaServerActiveRow {
server: string;
stage: BetaStage;
healthy_at: string | null;
match_seen_at: string | null;
wait_for_health: number;
wait_for_match: number;
}
@@ -101,7 +101,7 @@ async function runBetaTick(): Promise<{ processed: number }> {
const rows = db()
.prepare(
`SELECT bs.beta_id AS beta_id, bs.server AS server, bs.stage AS stage,
bs.healthy_at AS healthy_at,
bs.healthy_at AS healthy_at, bs.match_seen_at AS match_seen_at,
b.wait_for_health AS wait_for_health, b.wait_for_match AS wait_for_match
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
@@ -181,9 +181,7 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
return;
}
case 'awaiting-match': {
const since = row.healthy_at ?? nowIso();
const result = await scanForMatchEnd(row.server, since);
if (!result.ended) return;
if (!row.match_seen_at) return;
transition(row.beta_id, row.server, 'ready');
return;
}
src/lib/server/pipeline/bus.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import { EventEmitter } from 'node:events';
import type { RoundTelemetry } from '../../telemetry.js';
import type { GameTelemetry } from '../../telemetry.js';
import type { RconEndpoint, RconFailure, RconPlayer } from '../integrations/rcon.js';
import type { Health } from '../fleet/health.js';
@@ -40,7 +40,7 @@ export type BetaStage =
export type BetaState = 'running' | 'ready-promote' | 'promoted' | 'rolled-back';
export type PipelineEvent =
| { type: 'server-telemetry'; entries: RoundTelemetry[] }
| { type: 'server-telemetry'; entries: GameTelemetry[] }
| { type: 'deployment-opened'; deploymentId: string; action: string; servers: string[] }
| { type: 'server-changed'; deploymentId: string; server: string; files: number }
| {
src/lib/server/pipeline/restart-rollout.ts
+2 -1
View File
@@ -56,7 +56,8 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
if (waitReason) return { kind: 'queued', message: waitReason };
requestsInFlight.add(server);
db().prepare("UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL WHERE name = ?").run(server);
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
try {
const result = await komodoRestart(server);
if (result.ok) return { kind: 'started' };
src/lib/server/pipeline/worker.ts
+2 -66
View File
@@ -10,7 +10,6 @@ import { komodoInspectHealthy } from '../integrations/komodo.js';
import { restartSettling } from './restart-rollout.js';
import { classifyHealth } from '../fleet/health.js';
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
import { scanMatchState } from '../fleet/match-log.js';
const TICK_MS = 15_000;
// Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
@@ -44,61 +43,6 @@ async function runTick(): Promise<void> {
} catch (err) {
console.error('[worker] beta-runner error:', err);
}
try {
await scanMatches();
} catch (err) {
console.error('[worker] match-scan error:', err);
}
}
/**
* Read every server's games_sp.log tail to advance game_started_at /
* game_ended_at. Concurrency-capped so 15 servers don't hammer disk at once.
*/
async function scanMatches(): Promise<void> {
const servers = listServers();
if (servers.length === 0) return;
let cursor = 0;
async function pump(): Promise<void> {
while (true) {
const i = cursor++;
if (i >= servers.length) return;
const srv = servers[i];
const result = await scanMatchState(srv.name).catch((err) => {
console.error(`[worker] scanMatchState(${srv.name}) failed:`, err);
return { kind: 'no-signal' as const };
});
const persistedInProgress = !!srv.game_started_at && !srv.game_ended_at;
const persistedKey = srv.game_cycle_key;
const now = new Date().toISOString();
if (result.kind === 'in-progress' && result.cycleKey) {
// New cycle when persisted key differs (fresh start, or a game
// restart with the same players — new J; line = new key).
if (result.cycleKey !== persistedKey) {
setServerField(srv.name, 'game_started_at', now);
setServerField(srv.name, 'game_ended_at', null);
setServerField(srv.name, 'game_cycle_key', result.cycleKey);
} else if (!persistedInProgress) {
// Same key but previous cycle was ended — treat as new run
// (shouldn't happen in practice since key would differ,
// but be defensive).
setServerField(srv.name, 'game_started_at', now);
setServerField(srv.name, 'game_ended_at', null);
}
} else if (result.kind === 'ended' && result.cycleKey) {
// Only stamp the end if it's the cycle we've been tracking and
// we haven't already stamped it.
if (result.cycleKey === persistedKey && persistedInProgress) {
setServerField(srv.name, 'game_ended_at', now);
}
}
// 'idle' / 'no-signal': leave state alone. UI hides the timer when
// the server has no players (see ServerCard/detail page).
}
}
await Promise.all(Array.from({ length: STATUS_CONCURRENCY }, pump));
}
/**
@@ -169,8 +113,8 @@ async function refreshStatus(): Promise<void> {
const nextPlayers = result.status.players;
let activeSince = srv.active_since;
let inactiveSince = srv.inactive_since;
let gameStartedAt = srv.game_started_at;
let gameEndedAt = srv.game_ended_at;
const gameStartedAt = srv.game_started_at;
const gameEndedAt = srv.game_ended_at;
if (nextPlayers > 0 && (activeSince === null || prevPlayers === 0 || prevPlayers === null)) {
activeSince = now;
inactiveSince = null;
@@ -181,14 +125,6 @@ async function refreshStatus(): Promise<void> {
activeSince = null;
setServerField(srv.name, 'inactive_since', inactiveSince);
setServerField(srv.name, 'active_since', null);
// Game timer cannot run while inactive. Drop any tracked
// match state so the next active session anchors fresh
// when a new J; line fires.
gameStartedAt = null;
gameEndedAt = null;
setServerField(srv.name, 'game_started_at', null);
setServerField(srv.name, 'game_ended_at', null);
setServerField(srv.name, 'game_cycle_key', null);
}
// Persist last-observed state. last_seen_at is the health anchor.
setServerField(srv.name, 'last_players', nextPlayers);