Preserve health during empty-server monitoring handoff (Refs #148)

This commit is contained in:
2026-10-12 19:02:54 -04:00
parent 2f52dcb1ef
commit f91ec4d152
6 changed files with 224 additions and 6 deletions
+4 -1
View File
@@ -1,6 +1,7 @@
import type { BetaPlan, RequirementProgress } from '../../beta-plan.js';
import type { GameEvent } from '../../telemetry.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
export interface MatchEvidence {
map: string;
@@ -84,7 +85,9 @@ export function applyGameEvent(
export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: boolean): void {
if (m.lastSample !== undefined && at <= m.lastSample) return;
const gap = m.lastSample === undefined ? Infinity : at - m.lastSample;
const contiguous = gap <= 60_000;
// Health evidence remains valid for the shared freshness window, including
// the polling and agent response delay when telemetry hands off to RCON.
const contiguous = gap <= ONLINE_MAX_MS;
const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
if (!healthy && m.wasHealthy === true) m.outages++;
for (const r of plan.requirements) {
+16 -2
View File
@@ -132,8 +132,22 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
const m = readMetrics(run);
const row = getServer(run.server);
const server = row ? serverObservation(row) : null;
const timestamp =
let timestamp =
server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
let source = server?.status_source;
// Telemetry owns the displayed roster until it expires, but a newer
// successful RCON reply is already health evidence during the handoff.
const rconSeen = server?.rcon_status?.last_seen_at;
if (
server?.health === 'online' &&
server.status_source === 'telemetry' &&
rconSeen &&
Date.parse(rconSeen) > Date.parse(timestamp ?? '') &&
Date.parse(rconSeen) <= Date.now()
) {
timestamp = rconSeen;
source = 'rcon';
}
const sample = Date.parse(timestamp ?? '');
const freshSample =
Number.isFinite(sample) &&
@@ -149,7 +163,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
evidence(run, `sample:${sample}`, 'status-sample', {
healthy,
checkedAt: timestamp,
source: server?.status_source
source
});
if (before !== m.wasHealthy)
evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
+10 -1
View File
@@ -5,7 +5,7 @@ import type { Host } from '../../hosts.js';
import type { NativeInstance } from '../backends/native.js';
import type { ServerRow } from './servers.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { ONLINE_MAX_MS, STATUS_REFRESH_MS } from '../../server-freshness.js';
import { serverObservation } from './status.js';
import { recoverySettling } from '../provisioning/restarts.js';
@@ -172,6 +172,15 @@ export class MonitoringSnapshot {
? report
: null;
}
canSkipRcon(server: string): boolean {
const report = this.telemetryEvidence(server);
// Reserve a polling interval plus a response interval before expiry.
// Host status batches allow 25 seconds for agent delivery and replies.
return (
report?.players.length === 0 &&
Date.now() - Date.parse(report.receivedAt) < ONLINE_MAX_MS - 2 * STATUS_REFRESH_MS
);
}
observe = (server: ServerRow) =>
serverObservation(server, {
provisioning: this.provisioning(server.name),
+2 -2
View File
@@ -215,7 +215,7 @@ async function statusPass(): Promise<void> {
!pendingStatus.has(server.name) &&
!snapshot.provisioning(server.name) &&
!snapshot.settling(server.name) &&
snapshot.telemetryEvidence(server.name)?.players.length !== 0
!snapshot.canSkipRcon(server.name)
)
.map((server) => server.name),
snapshot
@@ -332,7 +332,7 @@ async function refreshServerStatus(
updates.last_error = null;
updates.rcon_failures = 0;
updates.last_reply_snippet = null;
} else if (unavailable || snapshot.telemetryEvidence(srv.name)?.players.length !== 0) {
} else if (unavailable || !snapshot.canSkipRcon(srv.name)) {
// Occupied telemetry rosters still need RCON for player pings.
flush();
const probeStartedAt = new Date().toISOString();