Cover beta health continuity and readiness boundaries (Refs #149)

src/lib/server/fleet/beta-evaluation.ts

@@ -1,6 +1,7 @@
 import type { BetaPlan, RequirementProgress } from '../../beta-plan.js';
 import type { GameEvent } from '../../telemetry.js';
 import { MIN_RECORDED_ROUND } from '../../match-history.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
 
 export interface MatchEvidence {
     map: string;
@@ -84,7 +85,9 @@ export function applyGameEvent(
 export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: boolean): void {
     if (m.lastSample !== undefined && at <= m.lastSample) return;
     const gap = m.lastSample === undefined ? Infinity : at - m.lastSample;
-    const contiguous = gap <= 60_000;
+    // Health evidence remains valid for the shared freshness window, including
+    // the polling and agent response delay when telemetry hands off to RCON.
+    const contiguous = gap <= ONLINE_MAX_MS;
     const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
     if (!healthy && m.wasHealthy === true) m.outages++;
     for (const r of plan.requirements) {

src/lib/server/fleet/beta-tests.ts

@@ -132,8 +132,22 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
     const m = readMetrics(run);
     const row = getServer(run.server);
     const server = row ? serverObservation(row) : null;
-    const timestamp =
+    let timestamp =
         server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
+    let source = server?.status_source;
+    // Telemetry owns the displayed roster until it expires, but a newer
+    // successful RCON reply is already health evidence during the handoff.
+    const rconSeen = server?.rcon_status?.last_seen_at;
+    if (
+        server?.health === 'online' &&
+        server.status_source === 'telemetry' &&
+        rconSeen &&
+        Date.parse(rconSeen) > Date.parse(timestamp ?? '') &&
+        Date.parse(rconSeen) <= Date.now()
+    ) {
+        timestamp = rconSeen;
+        source = 'rcon';
+    }
     const sample = Date.parse(timestamp ?? '');
     const freshSample =
         Number.isFinite(sample) &&
@@ -149,7 +163,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
         evidence(run, `sample:${sample}`, 'status-sample', {
             healthy,
             checkedAt: timestamp,
-            source: server?.status_source
+            source
         });
     if (before !== m.wasHealthy)
         evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });

src/lib/server/fleet/monitoring-snapshot.ts

@@ -5,7 +5,7 @@ import type { Host } from '../../hosts.js';
 import type { NativeInstance } from '../backends/native.js';
 import type { ServerRow } from './servers.js';
 import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
-import { ONLINE_MAX_MS } from '../../server-freshness.js';
+import { ONLINE_MAX_MS, STATUS_REFRESH_MS } from '../../server-freshness.js';
 import { serverObservation } from './status.js';
 import { recoverySettling } from '../provisioning/restarts.js';
 
@@ -172,6 +172,15 @@ export class MonitoringSnapshot {
             ? report
             : null;
     }
+    canSkipRcon(server: string): boolean {
+        const report = this.telemetryEvidence(server);
+        // Reserve a polling interval plus a response interval before expiry.
+        // Host status batches allow 25 seconds for agent delivery and replies.
+        return (
+            report?.players.length === 0 &&
+            Date.now() - Date.parse(report.receivedAt) < ONLINE_MAX_MS - 2 * STATUS_REFRESH_MS
+        );
+    }
     observe = (server: ServerRow) =>
         serverObservation(server, {
             provisioning: this.provisioning(server.name),

src/lib/server/pipeline/worker.ts

@@ -215,7 +215,7 @@ async function statusPass(): Promise<void> {
                     !pendingStatus.has(server.name) &&
                     !snapshot.provisioning(server.name) &&
                     !snapshot.settling(server.name) &&
-                    snapshot.telemetryEvidence(server.name)?.players.length !== 0
+                    !snapshot.canSkipRcon(server.name)
             )
             .map((server) => server.name),
         snapshot
@@ -332,7 +332,7 @@ async function refreshServerStatus(
                 updates.last_error = null;
                 updates.rcon_failures = 0;
                 updates.last_reply_snippet = null;
-            } else if (unavailable || snapshot.telemetryEvidence(srv.name)?.players.length !== 0) {
+            } else if (unavailable || !snapshot.canSkipRcon(srv.name)) {
                 // Occupied telemetry rosters still need RCON for player pings.
                 flush();
                 const probeStartedAt = new Date().toISOString();

tests/unit/beta-evaluation.test.ts

@@ -142,6 +142,54 @@ describe('beta requirement evaluation', () => {
         expect(m.values.r0).toBe(20);
     });
 
+    it('keeps health continuous through the freshness boundary but resets beyond it', () => {
+        const plan = planWith('healthy-minutes');
+        const m = emptyMetrics();
+        applySample(plan, m, 0, true);
+        applySample(plan, m, 90_000, true);
+        expect(m.values.r0).toBe(1.5);
+        applySample(plan, m, 180_001, true);
+        expect(m.values.r0).toBe(0);
+    });
+
+    it.each([
+        { minutes: 2, gap: 60_005 },
+        { minutes: 2.5, gap: 60_048 },
+        { minutes: 2.5, gap: 90_000 }
+    ])(
+        'preserves a satisfied healthy-minute requirement after a $gap ms gap',
+        ({ minutes, gap }) => {
+            const plan = planWith('healthy-minutes', 'failure-budget');
+            const m = emptyMetrics();
+            const previous = minutes * 60_000;
+            for (let at = 0; at <= previous; at += 30_000) applySample(plan, m, at, true);
+            expect(m.values.r0).toBe(minutes);
+            expect(evaluateRequirements(plan, m, 0).every((r) => r.passed)).toBe(true);
+
+            applySample(plan, m, previous + gap, true);
+
+            expect(m.values.r0).toBeCloseTo(minutes + gap / 60_000);
+            expect(m.wasHealthy).toBe(true);
+            expect(m.outages).toBe(0);
+            expect(evaluateRequirements(plan, m, 0).every((r) => r.passed)).toBe(true);
+        }
+    );
+
+    it('resets a satisfied healthy-minute requirement on an unhealthy sample within the freshness window', () => {
+        const plan = planWith('healthy-minutes', 'failure-budget');
+        const m = emptyMetrics();
+        applySample(plan, m, 0, true);
+        applySample(plan, m, 60_000, true);
+        expect(evaluateRequirements(plan, m, 0)[0].passed).toBe(true);
+
+        applySample(plan, m, 90_000, false);
+
+        expect(m.values.r0).toBe(0);
+        expect(m.wasHealthy).toBe(false);
+        expect(m.outages).toBe(1);
+        expect(evaluateRequirements(plan, m, 0)[0].passed).toBe(false);
+    });
+
     it('excludes monitoring gaps from healthy time and requires explicit manual sign-off', () => {
         const plan = planWith('healthy-minutes', 'failure-budget', 'manual');
         const m = emptyMetrics();

tests/unit/beta-plans.test.ts

@@ -367,6 +367,54 @@ describe('durable beta test plans', () => {
         await expect(promoteBeta(id)).rejects.toThrow('Test plan has not passed');
     });
 
+    it.each(['expired', 'unhealthy', 'gap'] as const)(
+        'preserves healthy-minute readiness across polling variation until evidence is %s',
+        async (failure) => {
+            const id = await openPlan(defaultPlan());
+            const { getBeta, listBetaServers, promoteBeta } =
+                await import('../../src/lib/server/fleet/beta.js');
+            const { currentRun, readMetrics, testView } =
+                await import('../../src/lib/server/fleet/beta-tests.js');
+            await tick();
+            await tick();
+            await tick(30);
+            await tick(30);
+            const metrics = () => readMetrics(currentRun(id, names[0])!);
+            const expectReady = () => {
+                expect(getBeta(id)?.state).toBe('ready-promote');
+                expect(listBetaServers(id)[0].stage).toBe('ready');
+                expect(testView(id)?.outcome).toBe('passed');
+                expect(metrics()).toMatchObject({ wasHealthy: true, outages: 0 });
+            };
+            expectReady();
+            let minutes = metrics().values.health;
+            for (const seconds of [60.005, 60.048, 90]) {
+                await tick(seconds);
+                minutes += seconds / 60;
+                expect(metrics().values.health).toBeCloseTo(minutes);
+                expectReady();
+            }
+            if (failure === 'expired') {
+                const { betaTick } = await import('../helpers/provisioning.js');
+                // Re-reading the same sample cannot extend its original lifetime.
+                vi.setSystemTime(Date.now() + 90_000);
+                await betaTick();
+                expectReady();
+                expect(metrics().values.health).toBeCloseTo(minutes);
+                vi.setSystemTime(Date.now() + 1);
+                await betaTick();
+            } else {
+                await tick(failure === 'gap' ? 90.001 : 15, 0, failure === 'gap');
+            }
+            expect(metrics().values.health).toBe(0);
+            expect(metrics().outages).toBe(failure === 'gap' ? 0 : 1);
+            expect(getBeta(id)?.state).toBe('running');
+            expect(listBetaServers(id)[0].stage).toBe('observing');
+            expect(testView(id)?.outcome).toBe('active');
+            await expect(promoteBeta(id)).rejects.toThrow('Test plan has not passed');
+        }
+    );
+
     it('revokes readiness if health fails while awaiting manual promotion', async () => {
         const id = await openPlan({
             ...defaultPlan(),

tests/unit/host-status-batch.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 import { gameEvent, postTelemetry } from '../helpers/telemetry.js';
+import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
 
 let fixture: Fixture;
 let at: number;
@@ -108,6 +109,187 @@ async function setup(count = 5, hostCount = 1) {
 }
 
 describe('host status observations', () => {
+    async function emptyHandoff() {
+        const api = await setup(1);
+        const server = 'server-0-0';
+        const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
+        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
+        const { currentRun, readMetrics } =
+            await import('../../src/lib/server/fleet/beta-tests.js');
+        const { plannedBetaTick } =
+            await import('../../src/lib/server/pipeline/beta-plan-runner.js');
+        const { script, version } = createScript({
+            releaseVersion: '1.0.0',
+            name: 'Handoff test',
+            targetRelPath: 'handoff.gsc',
+            content: Buffer.from('main() {}')
+        });
+        const betaId = Number(
+            api.db
+                .prepare(
+                    `INSERT INTO betas(script_id,script_version_id,trigger_kind,state,opened_at)
+                VALUES(?,?,'manual','ready-promote',?)`
+                )
+                .run(script.id, version.id, new Date(at).toISOString()).lastInsertRowid
+        );
+        api.db
+            .prepare(
+                `INSERT INTO beta_servers(beta_id,server,had_original,stage,restarted_at)
+            VALUES(?,?,0,'ready',?)`
+            )
+            .run(betaId, server, new Date(at).toISOString());
+        api.db
+            .prepare(`INSERT INTO beta_test_plans(beta_id,plan,outcome) VALUES(?,?,'passed')`)
+            .run(
+                betaId,
+                JSON.stringify({
+                    ...defaultPlan(),
+                    requirements: [
+                        requirement('healthy-minutes', 'health'),
+                        requirement('failure-budget', 'outages')
+                    ]
+                })
+            );
+        api.db
+            .prepare(
+                `INSERT INTO beta_test_runs(id,beta_id,server,created_at,observing_at,metrics)
+            VALUES('handoff',?,?,?,?,?)`
+            )
+            .run(
+                betaId,
+                server,
+                new Date(at).toISOString(),
+                new Date(at).toISOString(),
+                JSON.stringify({
+                    values: { health: 1 },
+                    manual: {},
+                    streaks: {},
+                    outages: 0,
+                    lastSample: at,
+                    wasHealthy: true
+                })
+            );
+        await postTelemetry(gameEvent('heartbeat', 15, 1, server));
+        const occupied = api.fanoutStatus();
+        await vi.waitFor(() => expect(api.operations()).toHaveLength(1));
+        api.complete();
+        await occupied;
+        vi.setSystemTime(at + 30_000);
+        await postTelemetry(gameEvent('heartbeat', 0, 0, server));
+        await api.fanoutStatus();
+        await plannedBetaTick();
+        const emptyAt = Date.now();
+        const observe = () => serverObservation(api.getServer(server)!);
+        const metrics = () => readMetrics(currentRun(betaId, server)!);
+        const expectReady = async () => {
+            await plannedBetaTick();
+            expect(observe().health).toBe('online');
+            expect(metrics()).toMatchObject({ wasHealthy: true, outages: 0 });
+            expect(metrics().values.health).toBeGreaterThanOrEqual(1.5);
+            expect(api.db.prepare('SELECT state FROM betas WHERE id=?').get(betaId)).toEqual({
+                state: 'ready-promote'
+            });
+            expect(
+                api.db.prepare('SELECT outcome FROM beta_test_plans WHERE beta_id=?').get(betaId)
+            ).toEqual({ outcome: 'passed' });
+        };
+        const setAge = (age: number) => {
+            vi.setSystemTime(emptyAt + age);
+            const host = api.getHost(api.hostIds[0])!;
+            api.heartbeat(host.id, {
+                ...host.inventory,
+                instanceStates: host.inventory.instanceStates!.map((state) => ({
+                    ...state,
+                    observedAt: new Date().toISOString()
+                }))
+            });
+        };
+        return { ...api, emptyAt, observe, metrics, expectReady, plannedBetaTick, setAge };
+    }
+
+    it.each([30_000, 59_999])(
+        'preserves beta readiness when the empty handoff poll starts at %i ms and replies 25 seconds later',
+        async (pollAt) => {
+            const api = await emptyHandoff();
+            api.setAge(pollAt - 30_000);
+            await api.fanoutStatus();
+            expect(api.operations()).toHaveLength(1);
+            await api.expectReady();
+            api.setAge(pollAt);
+            const pass = api.fanoutStatus();
+            try {
+                await vi.waitFor(() => expect(api.operations()).toHaveLength(2));
+                expect(api.operations()[0].kind).toBe('rcon.status-batch');
+                api.setAge(pollAt + 24_999);
+                await api.expectReady();
+            } finally {
+                api.setAge(pollAt + 25_000);
+                api.complete();
+                await pass;
+            }
+            await api.expectReady();
+            expect(api.metrics().lastSample).toBe(api.emptyAt + pollAt + 25_000);
+            expect(api.observe().status_source).toBe('telemetry');
+            for (const age of [90_000, 90_001]) {
+                api.setAge(age);
+                await api.expectReady();
+            }
+            expect(api.observe().status_source).toBe('rcon');
+            expect(api.observe().telemetry?.receivedAt).toBe(new Date(api.emptyAt).toISOString());
+        }
+    );
+
+    it.each(['pending', 'failed', 'skipped'] as const)(
+        'expires empty telemetry on schedule with a %s RCON handoff',
+        async (result) => {
+            const api = await emptyHandoff();
+            api.setAge(59_999);
+            const pass = api.fanoutStatus();
+            try {
+                await vi.waitFor(() => expect(api.operations()).toHaveLength(2));
+                if (result !== 'pending') {
+                    api.setAge(84_999);
+                    const operation = api.operations().find((op) => op.state === 'pending')!;
+                    api.reportHostOperation(
+                        operation.hostId,
+                        operation.id,
+                        result === 'failed'
+                            ? {
+                                  state: 'failed',
+                                  error: 'agent did not respond'
+                              }
+                            : {
+                                  state: 'completed',
+                                  result: {
+                                      results: [
+                                          {
+                                              instanceId: 'instance-0-0',
+                                              generation: 1,
+                                              reply: { ok: false, text: '', reason: 'deadline' }
+                                          }
+                                      ]
+                                  }
+                              }
+                    );
+                    await pass;
+                }
+                api.setAge(90_000);
+                await api.expectReady();
+                api.setAge(90_001);
+                expect(api.observe().health).not.toBe('online');
+                await api.plannedBetaTick();
+                expect(api.metrics()).toMatchObject({
+                    wasHealthy: false,
+                    outages: 1,
+                    values: { health: 0 }
+                });
+            } finally {
+                api.complete();
+                await pass;
+            }
+        }
+    );
+
     it('batches occupied telemetry lobbies for pings while skipping empty lobbies', async () => {
         const api = await setup(2);
         await postTelemetry(gameEvent('heartbeat', 15, 1, 'server-0-0'));