Preserve health during empty-server monitoring handoff (Refs #148)
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import type { BetaPlan, RequirementProgress } from '../../beta-plan.js';
|
||||
import type { GameEvent } from '../../telemetry.js';
|
||||
import { MIN_RECORDED_ROUND } from '../../match-history.js';
|
||||
import { ONLINE_MAX_MS } from '../../server-freshness.js';
|
||||
|
||||
export interface MatchEvidence {
|
||||
map: string;
|
||||
@@ -84,7 +85,9 @@ export function applyGameEvent(
|
||||
export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: boolean): void {
|
||||
if (m.lastSample !== undefined && at <= m.lastSample) return;
|
||||
const gap = m.lastSample === undefined ? Infinity : at - m.lastSample;
|
||||
const contiguous = gap <= 60_000;
|
||||
// Health evidence remains valid for the shared freshness window, including
|
||||
// the polling and agent response delay when telemetry hands off to RCON.
|
||||
const contiguous = gap <= ONLINE_MAX_MS;
|
||||
const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
|
||||
if (!healthy && m.wasHealthy === true) m.outages++;
|
||||
for (const r of plan.requirements) {
|
||||
|
||||
@@ -132,8 +132,22 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
|
||||
const m = readMetrics(run);
|
||||
const row = getServer(run.server);
|
||||
const server = row ? serverObservation(row) : null;
|
||||
const timestamp =
|
||||
let timestamp =
|
||||
server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
|
||||
let source = server?.status_source;
|
||||
// Telemetry owns the displayed roster until it expires, but a newer
|
||||
// successful RCON reply is already health evidence during the handoff.
|
||||
const rconSeen = server?.rcon_status?.last_seen_at;
|
||||
if (
|
||||
server?.health === 'online' &&
|
||||
server.status_source === 'telemetry' &&
|
||||
rconSeen &&
|
||||
Date.parse(rconSeen) > Date.parse(timestamp ?? '') &&
|
||||
Date.parse(rconSeen) <= Date.now()
|
||||
) {
|
||||
timestamp = rconSeen;
|
||||
source = 'rcon';
|
||||
}
|
||||
const sample = Date.parse(timestamp ?? '');
|
||||
const freshSample =
|
||||
Number.isFinite(sample) &&
|
||||
@@ -149,7 +163,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
|
||||
evidence(run, `sample:${sample}`, 'status-sample', {
|
||||
healthy,
|
||||
checkedAt: timestamp,
|
||||
source: server?.status_source
|
||||
source
|
||||
});
|
||||
if (before !== m.wasHealthy)
|
||||
evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
|
||||
|
||||
@@ -5,7 +5,7 @@ import type { Host } from '../../hosts.js';
|
||||
import type { NativeInstance } from '../backends/native.js';
|
||||
import type { ServerRow } from './servers.js';
|
||||
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
|
||||
import { ONLINE_MAX_MS } from '../../server-freshness.js';
|
||||
import { ONLINE_MAX_MS, STATUS_REFRESH_MS } from '../../server-freshness.js';
|
||||
import { serverObservation } from './status.js';
|
||||
import { recoverySettling } from '../provisioning/restarts.js';
|
||||
|
||||
@@ -172,6 +172,15 @@ export class MonitoringSnapshot {
|
||||
? report
|
||||
: null;
|
||||
}
|
||||
canSkipRcon(server: string): boolean {
|
||||
const report = this.telemetryEvidence(server);
|
||||
// Reserve a polling interval plus a response interval before expiry.
|
||||
// Host status batches allow 25 seconds for agent delivery and replies.
|
||||
return (
|
||||
report?.players.length === 0 &&
|
||||
Date.now() - Date.parse(report.receivedAt) < ONLINE_MAX_MS - 2 * STATUS_REFRESH_MS
|
||||
);
|
||||
}
|
||||
observe = (server: ServerRow) =>
|
||||
serverObservation(server, {
|
||||
provisioning: this.provisioning(server.name),
|
||||
|
||||
Reference in New Issue
Block a user