josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: honor Komodo health and prefer fresh telemetry across fleet status

Browse Source
This commit is contained in:
josh
2026-09-27 15:34:30 -04:00
parent 46a89ea4b0
commit 8ea696417f
27 changed files with 488 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/beta-tests.ts
+12 -8
View File
@@ -1,3 +1,5 @@
import { getServer } from './servers.js';
import { serverObservation } from './status.js';
import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
@@ -52,17 +54,19 @@ export function recordGameEvent(event: GameTelemetry): void {
export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
const m = readMetrics(run);
const server = db().prepare('SELECT last_checked_at, last_seen_at, last_error, container_state FROM servers WHERE name = ?').get(run.server) as {
last_checked_at: string | null; last_seen_at: string | null; last_error: string | null; container_state: string | null };
const sample = server.last_checked_at ? Date.parse(server.last_checked_at) : 0;
const freshSample = sample >= Date.parse(run.observing_at ?? run.created_at) && Date.now() - sample <= 60_000;
const healthy = freshSample && !server.last_error && server.container_state === 'running' && server.last_seen_at === server.last_checked_at;
const at = freshSample && server.container_state === 'running' ? sample : Date.now();
const row = getServer(run.server);
const server = row ? serverObservation(row) : null;
const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
const sample = Date.parse(timestamp ?? '');
const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
&& Date.now() - sample >= 0 && Date.now() - sample <= 45_000;
const healthy = freshSample && server?.health === 'online';
const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
const before = m.wasHealthy;
const lastSample = m.lastSample;
applySample(plan, m, at, healthy);
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: server.last_checked_at });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server.last_error });
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, source: server?.status_source });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
saveMetrics(run, m);
}