josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve health during empty-server monitoring handoff (Refs #148)

Browse Source
This commit is contained in:
josh
2026-10-12 19:02:54 -04:00
parent 2f52dcb1ef
commit f91ec4d152
6 changed files with 224 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/beta-evaluation.ts
+4 -1
View File
@@ -1,6 +1,7 @@
import type { BetaPlan, RequirementProgress } from '../../beta-plan.js';
import type { GameEvent } from '../../telemetry.js';
import { MIN_RECORDED_ROUND } from '../../match-history.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
export interface MatchEvidence {
map: string;
@@ -84,7 +85,9 @@ export function applyGameEvent(
export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: boolean): void {
if (m.lastSample !== undefined && at <= m.lastSample) return;
const gap = m.lastSample === undefined ? Infinity : at - m.lastSample;
const contiguous = gap <= 60_000;
// Health evidence remains valid for the shared freshness window, including
// the polling and agent response delay when telemetry hands off to RCON.
const contiguous = gap <= ONLINE_MAX_MS;
const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
if (!healthy && m.wasHealthy === true) m.outages++;
for (const r of plan.requirements) {
src/lib/server/fleet/beta-tests.ts
+16 -2
View File
@@ -132,8 +132,22 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
const m = readMetrics(run);
const row = getServer(run.server);
const server = row ? serverObservation(row) : null;
const timestamp =
let timestamp =
server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
let source = server?.status_source;
// Telemetry owns the displayed roster until it expires, but a newer
// successful RCON reply is already health evidence during the handoff.
const rconSeen = server?.rcon_status?.last_seen_at;
if (
server?.health === 'online' &&
server.status_source === 'telemetry' &&
rconSeen &&
Date.parse(rconSeen) > Date.parse(timestamp ?? '') &&
Date.parse(rconSeen) <= Date.now()
) {
timestamp = rconSeen;
source = 'rcon';
}
const sample = Date.parse(timestamp ?? '');
const freshSample =
Number.isFinite(sample) &&
@@ -149,7 +163,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
evidence(run, `sample:${sample}`, 'status-sample', {
healthy,
checkedAt: timestamp,
source: server?.status_source
source
});
if (before !== m.wasHealthy)
evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
src/lib/server/fleet/monitoring-snapshot.ts
+10 -1
View File
@@ -5,7 +5,7 @@ import type { Host } from '../../hosts.js';
import type { NativeInstance } from '../backends/native.js';
import type { ServerRow } from './servers.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { ONLINE_MAX_MS, STATUS_REFRESH_MS } from '../../server-freshness.js';
import { serverObservation } from './status.js';
import { recoverySettling } from '../provisioning/restarts.js';
@@ -172,6 +172,15 @@ export class MonitoringSnapshot {
? report
: null;
}
canSkipRcon(server: string): boolean {
const report = this.telemetryEvidence(server);
// Reserve a polling interval plus a response interval before expiry.
// Host status batches allow 25 seconds for agent delivery and replies.
return (
report?.players.length === 0 &&
Date.now() - Date.parse(report.receivedAt) < ONLINE_MAX_MS - 2 * STATUS_REFRESH_MS
);
}
observe = (server: ServerRow) =>
serverObservation(server, {
provisioning: this.provisioning(server.name),
src/lib/server/pipeline/worker.ts
+2 -2
View File
@@ -215,7 +215,7 @@ async function statusPass(): Promise<void> {
!pendingStatus.has(server.name) &&
!snapshot.provisioning(server.name) &&
!snapshot.settling(server.name) &&
snapshot.telemetryEvidence(server.name)?.players.length !== 0
!snapshot.canSkipRcon(server.name)
)
.map((server) => server.name),
snapshot
@@ -332,7 +332,7 @@ async function refreshServerStatus(
updates.last_error = null;
updates.rcon_failures = 0;
updates.last_reply_snippet = null;
} else if (unavailable || snapshot.telemetryEvidence(srv.name)?.players.length !== 0) {
} else if (unavailable || !snapshot.canSkipRcon(srv.name)) {
// Occupied telemetry rosters still need RCON for player pings.
flush();
const probeStartedAt = new Date().toISOString();
tests/unit/beta-evaluation.test.ts
+10
View File
@@ -142,6 +142,16 @@ describe('beta requirement evaluation', () => {
expect(m.values.r0).toBe(20);
});
it('keeps health continuous through the freshness boundary but resets beyond it', () => {
const plan = planWith('healthy-minutes');
const m = emptyMetrics();
applySample(plan, m, 0, true);
applySample(plan, m, 90_000, true);
expect(m.values.r0).toBe(1.5);
applySample(plan, m, 180_001, true);
expect(m.values.r0).toBe(0);
});
it('excludes monitoring gaps from healthy time and requires explicit manual sign-off', () => {
const plan = planWith('healthy-minutes', 'failure-budget', 'manual');
const m = emptyMetrics();
tests/unit/host-status-batch.test.ts
+182
View File
@@ -1,6 +1,7 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { gameEvent, postTelemetry } from '../helpers/telemetry.js';
import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
let fixture: Fixture;
let at: number;
@@ -108,6 +109,187 @@ async function setup(count = 5, hostCount = 1) {
}
describe('host status observations', () => {
async function emptyHandoff() {
const api = await setup(1);
const server = 'server-0-0';
const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { currentRun, readMetrics } =
await import('../../src/lib/server/fleet/beta-tests.js');
const { plannedBetaTick } =
await import('../../src/lib/server/pipeline/beta-plan-runner.js');
const { script, version } = createScript({
releaseVersion: '1.0.0',
name: 'Handoff test',
targetRelPath: 'handoff.gsc',
content: Buffer.from('main() {}')
});
const betaId = Number(
api.db
.prepare(
`INSERT INTO betas(script_id,script_version_id,trigger_kind,state,opened_at)
VALUES(?,?,'manual','ready-promote',?)`
)
.run(script.id, version.id, new Date(at).toISOString()).lastInsertRowid
);
api.db
.prepare(
`INSERT INTO beta_servers(beta_id,server,had_original,stage,restarted_at)
VALUES(?,?,0,'ready',?)`
)
.run(betaId, server, new Date(at).toISOString());
api.db
.prepare(`INSERT INTO beta_test_plans(beta_id,plan,outcome) VALUES(?,?,'passed')`)
.run(
betaId,
JSON.stringify({
...defaultPlan(),
requirements: [
requirement('healthy-minutes', 'health'),
requirement('failure-budget', 'outages')
]
})
);
api.db
.prepare(
`INSERT INTO beta_test_runs(id,beta_id,server,created_at,observing_at,metrics)
VALUES('handoff',?,?,?,?,?)`
)
.run(
betaId,
server,
new Date(at).toISOString(),
new Date(at).toISOString(),
JSON.stringify({
values: { health: 1 },
manual: {},
streaks: {},
outages: 0,
lastSample: at,
wasHealthy: true
})
);
await postTelemetry(gameEvent('heartbeat', 15, 1, server));
const occupied = api.fanoutStatus();
await vi.waitFor(() => expect(api.operations()).toHaveLength(1));
api.complete();
await occupied;
vi.setSystemTime(at + 30_000);
await postTelemetry(gameEvent('heartbeat', 0, 0, server));
await api.fanoutStatus();
await plannedBetaTick();
const emptyAt = Date.now();
const observe = () => serverObservation(api.getServer(server)!);
const metrics = () => readMetrics(currentRun(betaId, server)!);
const expectReady = async () => {
await plannedBetaTick();
expect(observe().health).toBe('online');
expect(metrics()).toMatchObject({ wasHealthy: true, outages: 0 });
expect(metrics().values.health).toBeGreaterThanOrEqual(1.5);
expect(api.db.prepare('SELECT state FROM betas WHERE id=?').get(betaId)).toEqual({
state: 'ready-promote'
});
expect(
api.db.prepare('SELECT outcome FROM beta_test_plans WHERE beta_id=?').get(betaId)
).toEqual({ outcome: 'passed' });
};
const setAge = (age: number) => {
vi.setSystemTime(emptyAt + age);
const host = api.getHost(api.hostIds[0])!;
api.heartbeat(host.id, {
...host.inventory,
instanceStates: host.inventory.instanceStates!.map((state) => ({
...state,
observedAt: new Date().toISOString()
}))
});
};
return { ...api, emptyAt, observe, metrics, expectReady, plannedBetaTick, setAge };
}
it.each([30_000, 59_999])(
'preserves beta readiness when the empty handoff poll starts at %i ms and replies 25 seconds later',
async (pollAt) => {
const api = await emptyHandoff();
api.setAge(pollAt - 30_000);
await api.fanoutStatus();
expect(api.operations()).toHaveLength(1);
await api.expectReady();
api.setAge(pollAt);
const pass = api.fanoutStatus();
try {
await vi.waitFor(() => expect(api.operations()).toHaveLength(2));
expect(api.operations()[0].kind).toBe('rcon.status-batch');
api.setAge(pollAt + 24_999);
await api.expectReady();
} finally {
api.setAge(pollAt + 25_000);
api.complete();
await pass;
}
await api.expectReady();
expect(api.metrics().lastSample).toBe(api.emptyAt + pollAt + 25_000);
expect(api.observe().status_source).toBe('telemetry');
for (const age of [90_000, 90_001]) {
api.setAge(age);
await api.expectReady();
}
expect(api.observe().status_source).toBe('rcon');
expect(api.observe().telemetry?.receivedAt).toBe(new Date(api.emptyAt).toISOString());
}
);
it.each(['pending', 'failed', 'skipped'] as const)(
'expires empty telemetry on schedule with a %s RCON handoff',
async (result) => {
const api = await emptyHandoff();
api.setAge(59_999);
const pass = api.fanoutStatus();
try {
await vi.waitFor(() => expect(api.operations()).toHaveLength(2));
if (result !== 'pending') {
api.setAge(84_999);
const operation = api.operations().find((op) => op.state === 'pending')!;
api.reportHostOperation(
operation.hostId,
operation.id,
result === 'failed'
? {
state: 'failed',
error: 'agent did not respond'
}
: {
state: 'completed',
result: {
results: [
{
instanceId: 'instance-0-0',
generation: 1,
reply: { ok: false, text: '', reason: 'deadline' }
}
]
}
}
);
await pass;
}
api.setAge(90_000);
await api.expectReady();
api.setAge(90_001);
expect(api.observe().health).not.toBe('online');
await api.plannedBetaTick();
expect(api.metrics()).toMatchObject({
wasHealthy: false,
outages: 1,
values: { health: 0 }
});
} finally {
api.complete();
await pass;
}
}
);
it('batches occupied telemetry lobbies for pings while skipping empty lobbies', async () => {
const api = await setup(2);
await postTelemetry(gameEvent('heartbeat', 15, 1, 'server-0-0'));