From f91ec4d15201a91923e34501e067e506b685b4f2 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 12 Oct 2026 19:02:54 -0400 Subject: [PATCH] Preserve health during empty-server monitoring handoff (Refs #148) --- src/lib/server/fleet/beta-evaluation.ts | 5 +- src/lib/server/fleet/beta-tests.ts | 18 +- src/lib/server/fleet/monitoring-snapshot.ts | 11 +- src/lib/server/pipeline/worker.ts | 4 +- tests/unit/beta-evaluation.test.ts | 10 ++ tests/unit/host-status-batch.test.ts | 182 ++++++++++++++++++++ 6 files changed, 224 insertions(+), 6 deletions(-) diff --git a/src/lib/server/fleet/beta-evaluation.ts b/src/lib/server/fleet/beta-evaluation.ts index 4498fcd..563889a 100644 --- a/src/lib/server/fleet/beta-evaluation.ts +++ b/src/lib/server/fleet/beta-evaluation.ts @@ -1,6 +1,7 @@ import type { BetaPlan, RequirementProgress } from '../../beta-plan.js'; import type { GameEvent } from '../../telemetry.js'; import { MIN_RECORDED_ROUND } from '../../match-history.js'; +import { ONLINE_MAX_MS } from '../../server-freshness.js'; export interface MatchEvidence { map: string; @@ -84,7 +85,9 @@ export function applyGameEvent( export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: boolean): void { if (m.lastSample !== undefined && at <= m.lastSample) return; const gap = m.lastSample === undefined ? Infinity : at - m.lastSample; - const contiguous = gap <= 60_000; + // Health evidence remains valid for the shared freshness window, including + // the polling and agent response delay when telemetry hands off to RCON. + const contiguous = gap <= ONLINE_MAX_MS; const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0; if (!healthy && m.wasHealthy === true) m.outages++; for (const r of plan.requirements) { diff --git a/src/lib/server/fleet/beta-tests.ts b/src/lib/server/fleet/beta-tests.ts index 378afb1..85f5dc5 100644 --- a/src/lib/server/fleet/beta-tests.ts +++ b/src/lib/server/fleet/beta-tests.ts @@ -132,8 +132,22 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise { const m = readMetrics(run); const row = getServer(run.server); const server = row ? serverObservation(row) : null; - const timestamp = + let timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at; + let source = server?.status_source; + // Telemetry owns the displayed roster until it expires, but a newer + // successful RCON reply is already health evidence during the handoff. + const rconSeen = server?.rcon_status?.last_seen_at; + if ( + server?.health === 'online' && + server.status_source === 'telemetry' && + rconSeen && + Date.parse(rconSeen) > Date.parse(timestamp ?? '') && + Date.parse(rconSeen) <= Date.now() + ) { + timestamp = rconSeen; + source = 'rcon'; + } const sample = Date.parse(timestamp ?? ''); const freshSample = Number.isFinite(sample) && @@ -149,7 +163,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise { evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, - source: server?.status_source + source }); if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null }); diff --git a/src/lib/server/fleet/monitoring-snapshot.ts b/src/lib/server/fleet/monitoring-snapshot.ts index db66a93..e6a00f2 100644 --- a/src/lib/server/fleet/monitoring-snapshot.ts +++ b/src/lib/server/fleet/monitoring-snapshot.ts @@ -5,7 +5,7 @@ import type { Host } from '../../hosts.js'; import type { NativeInstance } from '../backends/native.js'; import type { ServerRow } from './servers.js'; import { parseGameEvent, type GameTelemetry } from '../../telemetry.js'; -import { ONLINE_MAX_MS } from '../../server-freshness.js'; +import { ONLINE_MAX_MS, STATUS_REFRESH_MS } from '../../server-freshness.js'; import { serverObservation } from './status.js'; import { recoverySettling } from '../provisioning/restarts.js'; @@ -172,6 +172,15 @@ export class MonitoringSnapshot { ? report : null; } + canSkipRcon(server: string): boolean { + const report = this.telemetryEvidence(server); + // Reserve a polling interval plus a response interval before expiry. + // Host status batches allow 25 seconds for agent delivery and replies. + return ( + report?.players.length === 0 && + Date.now() - Date.parse(report.receivedAt) < ONLINE_MAX_MS - 2 * STATUS_REFRESH_MS + ); + } observe = (server: ServerRow) => serverObservation(server, { provisioning: this.provisioning(server.name), diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index 0cc28c1..752179c 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -215,7 +215,7 @@ async function statusPass(): Promise { !pendingStatus.has(server.name) && !snapshot.provisioning(server.name) && !snapshot.settling(server.name) && - snapshot.telemetryEvidence(server.name)?.players.length !== 0 + !snapshot.canSkipRcon(server.name) ) .map((server) => server.name), snapshot @@ -332,7 +332,7 @@ async function refreshServerStatus( updates.last_error = null; updates.rcon_failures = 0; updates.last_reply_snippet = null; - } else if (unavailable || snapshot.telemetryEvidence(srv.name)?.players.length !== 0) { + } else if (unavailable || !snapshot.canSkipRcon(srv.name)) { // Occupied telemetry rosters still need RCON for player pings. flush(); const probeStartedAt = new Date().toISOString(); diff --git a/tests/unit/beta-evaluation.test.ts b/tests/unit/beta-evaluation.test.ts index 875204b..29d9c8f 100644 --- a/tests/unit/beta-evaluation.test.ts +++ b/tests/unit/beta-evaluation.test.ts @@ -142,6 +142,16 @@ describe('beta requirement evaluation', () => { expect(m.values.r0).toBe(20); }); + it('keeps health continuous through the freshness boundary but resets beyond it', () => { + const plan = planWith('healthy-minutes'); + const m = emptyMetrics(); + applySample(plan, m, 0, true); + applySample(plan, m, 90_000, true); + expect(m.values.r0).toBe(1.5); + applySample(plan, m, 180_001, true); + expect(m.values.r0).toBe(0); + }); + it('excludes monitoring gaps from healthy time and requires explicit manual sign-off', () => { const plan = planWith('healthy-minutes', 'failure-budget', 'manual'); const m = emptyMetrics(); diff --git a/tests/unit/host-status-batch.test.ts b/tests/unit/host-status-batch.test.ts index e1e2b68..bb4c5df 100644 --- a/tests/unit/host-status-batch.test.ts +++ b/tests/unit/host-status-batch.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { gameEvent, postTelemetry } from '../helpers/telemetry.js'; +import { defaultPlan, requirement } from '../../src/lib/beta-plan.js'; let fixture: Fixture; let at: number; @@ -108,6 +109,187 @@ async function setup(count = 5, hostCount = 1) { } describe('host status observations', () => { + async function emptyHandoff() { + const api = await setup(1); + const server = 'server-0-0'; + const { serverObservation } = await import('../../src/lib/server/fleet/status.js'); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { currentRun, readMetrics } = + await import('../../src/lib/server/fleet/beta-tests.js'); + const { plannedBetaTick } = + await import('../../src/lib/server/pipeline/beta-plan-runner.js'); + const { script, version } = createScript({ + releaseVersion: '1.0.0', + name: 'Handoff test', + targetRelPath: 'handoff.gsc', + content: Buffer.from('main() {}') + }); + const betaId = Number( + api.db + .prepare( + `INSERT INTO betas(script_id,script_version_id,trigger_kind,state,opened_at) + VALUES(?,?,'manual','ready-promote',?)` + ) + .run(script.id, version.id, new Date(at).toISOString()).lastInsertRowid + ); + api.db + .prepare( + `INSERT INTO beta_servers(beta_id,server,had_original,stage,restarted_at) + VALUES(?,?,0,'ready',?)` + ) + .run(betaId, server, new Date(at).toISOString()); + api.db + .prepare(`INSERT INTO beta_test_plans(beta_id,plan,outcome) VALUES(?,?,'passed')`) + .run( + betaId, + JSON.stringify({ + ...defaultPlan(), + requirements: [ + requirement('healthy-minutes', 'health'), + requirement('failure-budget', 'outages') + ] + }) + ); + api.db + .prepare( + `INSERT INTO beta_test_runs(id,beta_id,server,created_at,observing_at,metrics) + VALUES('handoff',?,?,?,?,?)` + ) + .run( + betaId, + server, + new Date(at).toISOString(), + new Date(at).toISOString(), + JSON.stringify({ + values: { health: 1 }, + manual: {}, + streaks: {}, + outages: 0, + lastSample: at, + wasHealthy: true + }) + ); + await postTelemetry(gameEvent('heartbeat', 15, 1, server)); + const occupied = api.fanoutStatus(); + await vi.waitFor(() => expect(api.operations()).toHaveLength(1)); + api.complete(); + await occupied; + vi.setSystemTime(at + 30_000); + await postTelemetry(gameEvent('heartbeat', 0, 0, server)); + await api.fanoutStatus(); + await plannedBetaTick(); + const emptyAt = Date.now(); + const observe = () => serverObservation(api.getServer(server)!); + const metrics = () => readMetrics(currentRun(betaId, server)!); + const expectReady = async () => { + await plannedBetaTick(); + expect(observe().health).toBe('online'); + expect(metrics()).toMatchObject({ wasHealthy: true, outages: 0 }); + expect(metrics().values.health).toBeGreaterThanOrEqual(1.5); + expect(api.db.prepare('SELECT state FROM betas WHERE id=?').get(betaId)).toEqual({ + state: 'ready-promote' + }); + expect( + api.db.prepare('SELECT outcome FROM beta_test_plans WHERE beta_id=?').get(betaId) + ).toEqual({ outcome: 'passed' }); + }; + const setAge = (age: number) => { + vi.setSystemTime(emptyAt + age); + const host = api.getHost(api.hostIds[0])!; + api.heartbeat(host.id, { + ...host.inventory, + instanceStates: host.inventory.instanceStates!.map((state) => ({ + ...state, + observedAt: new Date().toISOString() + })) + }); + }; + return { ...api, emptyAt, observe, metrics, expectReady, plannedBetaTick, setAge }; + } + + it.each([30_000, 59_999])( + 'preserves beta readiness when the empty handoff poll starts at %i ms and replies 25 seconds later', + async (pollAt) => { + const api = await emptyHandoff(); + api.setAge(pollAt - 30_000); + await api.fanoutStatus(); + expect(api.operations()).toHaveLength(1); + await api.expectReady(); + api.setAge(pollAt); + const pass = api.fanoutStatus(); + try { + await vi.waitFor(() => expect(api.operations()).toHaveLength(2)); + expect(api.operations()[0].kind).toBe('rcon.status-batch'); + api.setAge(pollAt + 24_999); + await api.expectReady(); + } finally { + api.setAge(pollAt + 25_000); + api.complete(); + await pass; + } + await api.expectReady(); + expect(api.metrics().lastSample).toBe(api.emptyAt + pollAt + 25_000); + expect(api.observe().status_source).toBe('telemetry'); + for (const age of [90_000, 90_001]) { + api.setAge(age); + await api.expectReady(); + } + expect(api.observe().status_source).toBe('rcon'); + expect(api.observe().telemetry?.receivedAt).toBe(new Date(api.emptyAt).toISOString()); + } + ); + + it.each(['pending', 'failed', 'skipped'] as const)( + 'expires empty telemetry on schedule with a %s RCON handoff', + async (result) => { + const api = await emptyHandoff(); + api.setAge(59_999); + const pass = api.fanoutStatus(); + try { + await vi.waitFor(() => expect(api.operations()).toHaveLength(2)); + if (result !== 'pending') { + api.setAge(84_999); + const operation = api.operations().find((op) => op.state === 'pending')!; + api.reportHostOperation( + operation.hostId, + operation.id, + result === 'failed' + ? { + state: 'failed', + error: 'agent did not respond' + } + : { + state: 'completed', + result: { + results: [ + { + instanceId: 'instance-0-0', + generation: 1, + reply: { ok: false, text: '', reason: 'deadline' } + } + ] + } + } + ); + await pass; + } + api.setAge(90_000); + await api.expectReady(); + api.setAge(90_001); + expect(api.observe().health).not.toBe('online'); + await api.plannedBetaTick(); + expect(api.metrics()).toMatchObject({ + wasHealthy: false, + outages: 1, + values: { health: 0 } + }); + } finally { + api.complete(); + await pass; + } + } + ); + it('batches occupied telemetry lobbies for pings while skipping empty lobbies', async () => { const api = await setup(2); await postTelemetry(gameEvent('heartbeat', 15, 1, 'server-0-0'));