Relax freshness limits and tolerate transient RCON failures

src/lib/server/fleet/beta-tests.ts

@@ -5,6 +5,7 @@ import { db } from '../db/index.js';
 import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
 import type { GameTelemetry } from '../../telemetry.js';
 import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
 
 export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
 export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
@@ -64,7 +65,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
     const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
     const sample = Date.parse(timestamp ?? '');
     const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
-        && Date.now() - sample >= 0 && Date.now() - sample <= 45_000;
+        && Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS;
     const healthy = freshSample && server?.health === 'online';
     const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
     const before = m.wasHealthy;

src/lib/server/fleet/health.ts

@@ -6,7 +6,7 @@ const STALE_MAX_MS = 120_000;
 /**
  * Classify server health from its last successful RCON timestamp.
  * `offline` if we've never seen it or the last successful reply is >120s old.
- * `stale` on an RCON failure or if the last reply is 45-120s old.
+ * `stale` on a blocking RCON failure or if the last reply is >90-120s old.
  * `online` if fresh, including the allowance for polling and delivery.
  */
 export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {

src/lib/server/fleet/metrics.ts

@@ -87,7 +87,7 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
     let localCell = { index: 0, date: '' };
     interface BatchCounts { known: number; active: number; idle: number; unknown: number; players: number }
     function addHeatmap(counts: BatchCounts, start: number, end: number): void {
-        // Samples are at most 45 seconds. Splitting at UTC minute boundaries
+        // Samples are bounded by the shared freshness window. Splitting at UTC minute boundaries
         // handles local hour/day boundaries, DST, and fractional-hour zones.
         // Cache each minute once across the complete fleet collection batches.
         for (let cursor = start; cursor < end;) {

src/lib/server/fleet/servers.ts

@@ -91,6 +91,7 @@ export interface ServerRow {
     last_roster: string | null;
     last_seen_at: string | null;
     last_error: string | null;
+    rcon_failures: number;
     last_reply_snippet: string | null;
     game_mode: string | null;
     map_slug: string | null;
@@ -200,6 +201,7 @@ export function setServerFields(name: string, fields: Partial<Record<keyof Serve
         'last_roster',
         'last_seen_at',
         'last_error',
+        'rcon_failures',
         'last_reply_snippet',
         'game_mode',
         'map_slug',

src/lib/server/fleet/status.ts

@@ -1,4 +1,4 @@
-import { selectServerObservation, type GameObservation } from '../../server-observation.js';
+import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js';
 import { classifyHealth } from './health.js';
 import { isProvisioning } from './provisioning.js';
 import { restartSettling } from '../provisioning/restarts.js';
@@ -44,11 +44,12 @@ export function serverObservation(server: ServerRow, evidence?: Evidence) {
     let roster = null;
     try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
     const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
-        last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
+        last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error,
+        rcon_failures: server.rcon_failures, roster };
     const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
     const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
     const ready = !provisioning && (!state || state === 'running' || state === 'unknown');
-    const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
+    const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, rconHealthError(rcon), state);
     return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
         status_source: null as 'telemetry' | 'rcon' | null,
         telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,