Relax freshness limits and tolerate transient RCON failures

src/lib/server-freshness.ts

@@ -1,6 +1,6 @@
-// Allow a polling cycle plus time for queued RCON requests and delivery.
+// Allow missed polling cycles and queued requests across all status evidence.
 export const STATUS_REFRESH_MS = 30_000;
-export const ONLINE_MAX_MS = STATUS_REFRESH_MS + 15_000;
+export const ONLINE_MAX_MS = 90_000;
 // The UI clock ticks once per second and can trail a newly arrived report.
 const CLOCK_TOLERANCE_MS = 1_000;
 

src/lib/server-observation.ts

@@ -14,6 +14,8 @@ export interface GameObservation {
     last_gametype: string | null;
     last_seen_at: string | null;
     last_error: string | null;
+    /** Consecutive failed monitoring checks; absent on older snapshots. */
+    rcon_failures?: number;
     roster: StatusPlayer[] | null;
 }
 export interface ServerObservation extends GameObservation {
@@ -32,6 +34,12 @@ export interface ServerObservation extends GameObservation {
     game_ended_at?: string | null;
 }
 
+/** Keep a first transient monitoring failure in diagnostics without demoting health. */
+export function rconHealthError(status: Pick<GameObservation, 'last_error' | 'rcon_failures'>): string | null {
+    const reason = status.last_error?.split(':')[0];
+    return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error') ? null : status.last_error;
+}
+
 export function freshTelemetry(report: GameTelemetry | undefined, now: number, after?: string | null): report is GameTelemetry {
     if (!report || !freshStatusTimestamp(report.receivedAt, now)) return false;
     if (after && (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))) return false;
@@ -42,7 +50,7 @@ export function freshTelemetry(report: GameTelemetry | undefined, now: number, a
 export function selectServerObservation<T extends ServerObservation>(server: T, report: GameTelemetry | undefined, now: number): T {
     if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
     const raw = server.rcon_status ?? server;
-    const result: T = { ...server, ...raw, telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
+    const result: T = { ...server, ...raw, last_error: rconHealthError(raw), telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
     const containerUnknown = server.container_state === 'unknown';
     if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) {
         return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null };
@@ -69,11 +77,12 @@ export function selectServerObservation<T extends ServerObservation>(server: T,
             game_ended_at: players === 0 ? null : timing.gameEndedAt };
     }
     const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
-    const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !raw.last_error;
+    const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !result.last_error;
     // A current authenticated RCON reply can establish game liveness when
     // inspection is unavailable, without claiming the container was inspected.
     const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh;
     const age = now - Date.parse(raw.last_seen_at ?? '');
-    return { ...result, health: fresh && !containerExpired ? 'online'
-        : after && age >= 0 && age <= 120_000 ? 'stale' : 'offline' };
+    const health = fresh && !containerExpired ? 'online'
+        : after && age >= 0 && age <= 120_000 ? 'stale' : 'offline';
+    return { ...result, health, last_error: health === 'online' ? result.last_error : raw.last_error };
 }

src/lib/server/db/index.ts

@@ -35,6 +35,7 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
     { name: 'last_roster', ddl: 'ALTER TABLE servers ADD COLUMN last_roster TEXT' },
     { name: 'last_seen_at', ddl: 'ALTER TABLE servers ADD COLUMN last_seen_at TEXT' },
     { name: 'last_error', ddl: 'ALTER TABLE servers ADD COLUMN last_error TEXT' },
+    { name: 'rcon_failures', ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0' },
     { name: 'last_reply_snippet', ddl: 'ALTER TABLE servers ADD COLUMN last_reply_snippet TEXT' },
     // Script-scope columns — static per server, populated via server-edit UI.
     { name: 'game', ddl: 'ALTER TABLE servers ADD COLUMN game TEXT' },

src/lib/server/db/schema.sql

@@ -80,6 +80,7 @@ CREATE TABLE IF NOT EXISTS servers (
     last_roster      TEXT,                                -- JSON array of RconPlayer
     last_seen_at     TEXT,                                -- ISO ts of last successful RCON reply
     last_error       TEXT,                                -- last RCON failure reason (cleared on success)
+    rcon_failures    INTEGER NOT NULL DEFAULT 0,          -- consecutive monitoring failures, capped at two
     last_reply_snippet TEXT,                              -- hex+ASCII of last bad-reply UDP payload
     -- Provisioning fields — set when Electric Cherry creates the deployment.
     game_mode        TEXT,                                -- 'zombie' | 'mp' | 'sp' (currently zombie only)

src/lib/server/fleet/beta-tests.ts

@@ -5,6 +5,7 @@ import { db } from '../db/index.js';
 import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
 import type { GameTelemetry } from '../../telemetry.js';
 import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
 
 export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
 export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
@@ -64,7 +65,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
     const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
     const sample = Date.parse(timestamp ?? '');
     const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
-        && Date.now() - sample >= 0 && Date.now() - sample <= 45_000;
+        && Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS;
     const healthy = freshSample && server?.health === 'online';
     const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
     const before = m.wasHealthy;

src/lib/server/fleet/health.ts

@@ -6,7 +6,7 @@ const STALE_MAX_MS = 120_000;
 /**
  * Classify server health from its last successful RCON timestamp.
  * `offline` if we've never seen it or the last successful reply is >120s old.
- * `stale` on an RCON failure or if the last reply is 45-120s old.
+ * `stale` on a blocking RCON failure or if the last reply is >90-120s old.
  * `online` if fresh, including the allowance for polling and delivery.
  */
 export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {

src/lib/server/fleet/metrics.ts

@@ -87,7 +87,7 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
     let localCell = { index: 0, date: '' };
     interface BatchCounts { known: number; active: number; idle: number; unknown: number; players: number }
     function addHeatmap(counts: BatchCounts, start: number, end: number): void {
-        // Samples are at most 45 seconds. Splitting at UTC minute boundaries
+        // Samples are bounded by the shared freshness window. Splitting at UTC minute boundaries
         // handles local hour/day boundaries, DST, and fractional-hour zones.
         // Cache each minute once across the complete fleet collection batches.
         for (let cursor = start; cursor < end;) {

src/lib/server/fleet/servers.ts

@@ -91,6 +91,7 @@ export interface ServerRow {
     last_roster: string | null;
     last_seen_at: string | null;
     last_error: string | null;
+    rcon_failures: number;
     last_reply_snippet: string | null;
     game_mode: string | null;
     map_slug: string | null;
@@ -200,6 +201,7 @@ export function setServerFields(name: string, fields: Partial<Record<keyof Serve
         'last_roster',
         'last_seen_at',
         'last_error',
+        'rcon_failures',
         'last_reply_snippet',
         'game_mode',
         'map_slug',

src/lib/server/fleet/status.ts

@@ -1,4 +1,4 @@
-import { selectServerObservation, type GameObservation } from '../../server-observation.js';
+import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js';
 import { classifyHealth } from './health.js';
 import { isProvisioning } from './provisioning.js';
 import { restartSettling } from '../provisioning/restarts.js';
@@ -44,11 +44,12 @@ export function serverObservation(server: ServerRow, evidence?: Evidence) {
     let roster = null;
     try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
     const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
-        last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
+        last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error,
+        rcon_failures: server.rcon_failures, roster };
     const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
     const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
     const ready = !provisioning && (!state || state === 'running' || state === 'unknown');
-    const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
+    const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, rconHealthError(rcon), state);
     return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
         status_source: null as 'telemetry' | 'rcon' | null,
         telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,

src/lib/server/pipeline/worker.ts

@@ -177,6 +177,7 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
             const unavailable = updates.container_state === 'unknown';
             if (!container.ok && !unavailable) {
                 updates.last_error = null;
+                updates.rcon_failures = 0;
                 updates.last_reply_snippet = null;
             } else if (unavailable || !telemetryEvidence(srv.name)) {
                 flush();
@@ -192,7 +193,8 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
                 });
                 if (!probe) return null;
                 const result = probe.result;
-                if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) {
+                const previous = getServer(srv.name);
+                if (unchanged() && previous && (previous.last_checked_at ?? '') <= probe.observedAt) {
                     gameObservation = probe;
                     const now = probe.observedAt;
                     updates.last_checked_at = now;
@@ -205,13 +207,23 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
                         updates.last_roster = JSON.stringify(result.status.roster);
                         updates.last_seen_at = now;
                         updates.last_error = null;
-                    } else updates.last_error = encodeError(result.reason, result.detail);
+                        updates.rcon_failures = 0;
+                    } else {
+                        // Only distinct accepted checks in the current lifecycle count.
+                        // Skipped/superseded checks return above without changing the streak.
+                        const consecutive = previous.last_error && (!epoch || (previous.last_checked_at ?? '') > epoch);
+                        const failures = consecutive ? previous.rcon_failures || 1 : 0;
+                        updates.rcon_failures = consecutive && previous.last_checked_at === now
+                            ? failures : Math.min(2, failures + 1);
+                        updates.last_error = encodeError(result.reason, result.detail);
+                    }
                 }
             }
             if (!settling && unchanged()) publishObservation(inspected, gameObservation);
         }
     } else {
         updates.last_error = null;
+        updates.rcon_failures = 0;
         updates.last_reply_snippet = null;
     }
     flush();

src/lib/server/provisioning/host-drain.ts

@@ -104,7 +104,7 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance
             db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
             db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
             db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
-                last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,active_since=NULL,inactive_since=NULL,
+                last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL,
                 game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
             db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
             db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);

src/lib/server/provisioning/restarts.ts

@@ -107,7 +107,7 @@ export async function restartInRollout(server: string, options: { operationId: s
         });
     })();
     requestsInFlight.set(server, operationId);
-    db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
+    db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, rcon_failures = 0, last_reply_snippet = NULL,
         game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
     try {
         const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);

src/routes/api/servers/[name]/probe-rcon/+server.ts

@@ -38,11 +38,14 @@ export const POST: RequestHandler = async ({ params }) => {
         setServerField(name, 'last_seen_at', now);
         setServerField(name, 'last_checked_at', now);
         setServerField(name, 'last_error', null);
+        setServerField(name, 'rcon_failures', 0);
         setServerField(name, 'last_reply_snippet', result.rawReply);
     } else {
         const encoded = `${result.reason}:${result.detail}`;
         setServerField(name, 'last_checked_at', now);
         setServerField(name, 'last_error', encoded);
+        // An explicitly requested diagnostic failure takes effect immediately.
+        setServerField(name, 'rcon_failures', 2);
         setServerField(name, 'last_reply_snippet', result.rawReply ?? null);
     }
 

src/routes/metrics/+page.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
     import { onMount, untrack } from 'svelte';
+    import { ONLINE_MAX_MS } from '$lib/server-freshness.js';
     import { page } from '$app/state';
     import { afterNavigate, beforeNavigate, goto } from '$app/navigation';
     import { timeZone } from '$ui/stores/time-zone.js';
@@ -169,7 +170,7 @@
     <details class="methodology">
         <summary>How these estimates work</summary>
         <p>Time with players is occupied server time divided by known server time. Known time includes occupied and idle observations; unknown status and collection gaps are excluded from that percentage.</p>
-        <p>Each periodic observation covers at most 45 seconds. Longer collection gaps are excluded. History is retained for 31 days. Player-hours include all players reported by the server and are estimates, not individual session records.</p>
+        <p>Each periodic observation covers at most {ONLINE_MAX_MS / 1000} seconds. Longer collection gaps are excluded. History is retained for 31 days. Player-hours include all players reported by the server and are estimates, not individual session records.</p>
         <p>Timeline buckets are aligned to UTC, with labels shown in {report.scope.timeZone.replaceAll('_', ' ')}. Daily buckets are 24-hour periods, not local calendar days. The first and latest buckets can be partial. Percentage bars divide occupied, idle, and unknown time by captured time; the headline occupancy percentage uses known time only.</p>
         <p>Historical server rows are grouped by name and may include multiple registrations using that name. Map groups preserve the recorded game and map label.</p>
     </details>