From b1b3e8b958091914fb68523398ac2897ad4664b0 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 5 Oct 2026 16:36:23 -0400 Subject: [PATCH] Relax freshness limits and tolerate transient RCON failures --- docs/maintenance.md | 11 ++- docs/metrics.md | 2 +- docs/provisioning.md | 2 +- src/lib/server-freshness.ts | 4 +- src/lib/server-observation.ts | 17 +++- src/lib/server/db/index.ts | 1 + src/lib/server/db/schema.sql | 1 + src/lib/server/fleet/beta-tests.ts | 3 +- src/lib/server/fleet/health.ts | 2 +- src/lib/server/fleet/metrics.ts | 2 +- src/lib/server/fleet/servers.ts | 2 + src/lib/server/fleet/status.ts | 7 +- src/lib/server/pipeline/worker.ts | 16 +++- src/lib/server/provisioning/host-drain.ts | 2 +- src/lib/server/provisioning/restarts.ts | 2 +- .../api/servers/[name]/probe-rcon/+server.ts | 3 + src/routes/metrics/+page.svelte | 3 +- tests/unit/beta-plans.test.ts | 10 +-- tests/unit/fleet-overview.test.ts | 4 +- tests/unit/health-observations.test.ts | 4 +- tests/unit/host-status-batch.test.ts | 5 +- tests/unit/metrics.test.ts | 24 +++--- tests/unit/provisioning.test.ts | 2 +- tests/unit/server-freshness.test.ts | 4 +- tests/unit/server-observation.test.ts | 36 +++++++-- tests/unit/server-status.test.ts | 79 ++++++++++++++++++- tests/unit/ui-server-clock.test.ts | 26 +++++- tests/unit/watchdog.test.ts | 5 +- 28 files changed, 221 insertions(+), 58 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index 83a06c1..fdc4e82 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -102,9 +102,18 @@ reachable. Finished scheduled and legacy maintenance activity is retained for ## Health recovery and configuration drift +Fleet keeps successful game and running-container reports fresh for 90 seconds. +Two consecutive failed monitoring checks mark RCON health stale; a first timeout +or send failure remains visible in diagnostics without changing the health badge. +Configuration and invalid-reply errors, including authentication failures, take +effect immediately, as do failed manual probes. Successful RCON checks reset the +failure count; skipped checks do not count. Without fresh telemetry, RCON evidence +older than 120 seconds is offline. Explicit stopped/unhealthy container states +and lifecycle changes still take precedence. + The status monitor collects container and game health every 30 seconds. Once per minute, the watchdog evaluates that completed evidence without issuing its own -routine health probes. Samples expire after 45 seconds; missing or stale evidence +routine health probes. Samples expire after 90 seconds; missing or stale evidence leaves health unknown and cannot justify recovery. Samples retain their original timestamps and instance identity, and each failure sample counts at most once. A monitoring gap or changed instance resets the sustained failure window. After diff --git a/docs/metrics.md b/docs/metrics.md index 6daf54d..18f7957 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -78,7 +78,7 @@ a change; a retained map may no longer have a hosted lobby. - Player-hours integrate the reported player count over observed time. Bots are included if the server counts them as players. - All lobbies full measures map wall time when every tracked lobby in that game/map group has a known capacity and is full. Missing capacity or status makes that interval unknown. This is a capacity-pressure indicator, not a count of rejected joins or unmet demand. -Observations are recorded by the existing status worker. Each remains valid until the next collection, up to the 45-second freshness limit. Longer manager outages are excluded. Charts split time at UTC hour/day boundaries; partial periods remain partial. Map variants use the configured variant when it matches the reported map. +Observations are recorded by the existing status worker. Each remains valid until the next collection, up to the 90-second freshness limit. Longer manager outages are excluded. Charts split time at UTC hour/day boundaries; partial periods remain partial. Map variants use the configured variant when it matches the reported map. Capacity comes from `sv_maxclients` / `sv_maxClients` in the status response. When unavailable, the UI displays unknown rather than assuming a four-player limit. Capacity coverage is the map time for which every tracked lobby reported both occupancy and capacity. diff --git a/docs/provisioning.md b/docs/provisioning.md index 48318f4..35ad60d 100644 --- a/docs/provisioning.md +++ b/docs/provisioning.md @@ -57,7 +57,7 @@ unreachable Komodo API is not evidence that a stop succeeded. Disruptive operations require a fresh zero-player result from RCON or accepted telemetry, unless Komodo positively reports a stopped container. Fresh occupied telemetry vetoes a conflicting empty RCON reading. Reports must be received within -45 seconds; recovery verification also excludes reports from before the submitted +90 seconds; recovery verification also excludes reports from before the submitted operation. Telemetry can verify the game's activity and selected map after Komodo reports healthy. Runtime actions wait for conflicting campaign, beta, maintenance, or provisioning work. Duplicate pending requests for diff --git a/src/lib/server-freshness.ts b/src/lib/server-freshness.ts index ec01e72..8ee9a79 100644 --- a/src/lib/server-freshness.ts +++ b/src/lib/server-freshness.ts @@ -1,6 +1,6 @@ -// Allow a polling cycle plus time for queued RCON requests and delivery. +// Allow missed polling cycles and queued requests across all status evidence. export const STATUS_REFRESH_MS = 30_000; -export const ONLINE_MAX_MS = STATUS_REFRESH_MS + 15_000; +export const ONLINE_MAX_MS = 90_000; // The UI clock ticks once per second and can trail a newly arrived report. const CLOCK_TOLERANCE_MS = 1_000; diff --git a/src/lib/server-observation.ts b/src/lib/server-observation.ts index f8bfa5f..ce8ed39 100644 --- a/src/lib/server-observation.ts +++ b/src/lib/server-observation.ts @@ -14,6 +14,8 @@ export interface GameObservation { last_gametype: string | null; last_seen_at: string | null; last_error: string | null; + /** Consecutive failed monitoring checks; absent on older snapshots. */ + rcon_failures?: number; roster: StatusPlayer[] | null; } export interface ServerObservation extends GameObservation { @@ -32,6 +34,12 @@ export interface ServerObservation extends GameObservation { game_ended_at?: string | null; } +/** Keep a first transient monitoring failure in diagnostics without demoting health. */ +export function rconHealthError(status: Pick): string | null { + const reason = status.last_error?.split(':')[0]; + return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error') ? null : status.last_error; +} + export function freshTelemetry(report: GameTelemetry | undefined, now: number, after?: string | null): report is GameTelemetry { if (!report || !freshStatusTimestamp(report.receivedAt, now)) return false; if (after && (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))) return false; @@ -42,7 +50,7 @@ export function freshTelemetry(report: GameTelemetry | undefined, now: number, a export function selectServerObservation(server: T, report: GameTelemetry | undefined, now: number): T { if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry; const raw = server.rcon_status ?? server; - const result: T = { ...server, ...raw, telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null }; + const result: T = { ...server, ...raw, last_error: rconHealthError(raw), telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null }; const containerUnknown = server.container_state === 'unknown'; if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) { return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null }; @@ -69,11 +77,12 @@ export function selectServerObservation(server: T, game_ended_at: players === 0 ? null : timing.gameEndedAt }; } const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after); - const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !raw.last_error; + const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !result.last_error; // A current authenticated RCON reply can establish game liveness when // inspection is unavailable, without claiming the container was inspected. const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh; const age = now - Date.parse(raw.last_seen_at ?? ''); - return { ...result, health: fresh && !containerExpired ? 'online' - : after && age >= 0 && age <= 120_000 ? 'stale' : 'offline' }; + const health = fresh && !containerExpired ? 'online' + : after && age >= 0 && age <= 120_000 ? 'stale' : 'offline'; + return { ...result, health, last_error: health === 'online' ? result.last_error : raw.last_error }; } diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index cdeb472..826cc6f 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -35,6 +35,7 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [ { name: 'last_roster', ddl: 'ALTER TABLE servers ADD COLUMN last_roster TEXT' }, { name: 'last_seen_at', ddl: 'ALTER TABLE servers ADD COLUMN last_seen_at TEXT' }, { name: 'last_error', ddl: 'ALTER TABLE servers ADD COLUMN last_error TEXT' }, + { name: 'rcon_failures', ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0' }, { name: 'last_reply_snippet', ddl: 'ALTER TABLE servers ADD COLUMN last_reply_snippet TEXT' }, // Script-scope columns — static per server, populated via server-edit UI. { name: 'game', ddl: 'ALTER TABLE servers ADD COLUMN game TEXT' }, diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 5f5b83c..b9017c0 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -80,6 +80,7 @@ CREATE TABLE IF NOT EXISTS servers ( last_roster TEXT, -- JSON array of RconPlayer last_seen_at TEXT, -- ISO ts of last successful RCON reply last_error TEXT, -- last RCON failure reason (cleared on success) + rcon_failures INTEGER NOT NULL DEFAULT 0, -- consecutive monitoring failures, capped at two last_reply_snippet TEXT, -- hex+ASCII of last bad-reply UDP payload -- Provisioning fields — set when Electric Cherry creates the deployment. game_mode TEXT, -- 'zombie' | 'mp' | 'sp' (currently zombie only) diff --git a/src/lib/server/fleet/beta-tests.ts b/src/lib/server/fleet/beta-tests.ts index 6c29d8e..6fb8939 100644 --- a/src/lib/server/fleet/beta-tests.ts +++ b/src/lib/server/fleet/beta-tests.ts @@ -5,6 +5,7 @@ import { db } from '../db/index.js'; import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js'; import type { GameTelemetry } from '../../telemetry.js'; import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js'; +import { ONLINE_MAX_MS } from '../../server-freshness.js'; export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string } export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null } @@ -64,7 +65,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise { const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at; const sample = Date.parse(timestamp ?? ''); const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at) - && Date.now() - sample >= 0 && Date.now() - sample <= 45_000; + && Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS; const healthy = freshSample && server?.health === 'online'; const at = freshSample && server?.container_state === 'running' ? sample : Date.now(); const before = m.wasHealthy; diff --git a/src/lib/server/fleet/health.ts b/src/lib/server/fleet/health.ts index 9a1dba9..3c69f9a 100644 --- a/src/lib/server/fleet/health.ts +++ b/src/lib/server/fleet/health.ts @@ -6,7 +6,7 @@ const STALE_MAX_MS = 120_000; /** * Classify server health from its last successful RCON timestamp. * `offline` if we've never seen it or the last successful reply is >120s old. - * `stale` on an RCON failure or if the last reply is 45-120s old. + * `stale` on a blocking RCON failure or if the last reply is >90-120s old. * `online` if fresh, including the allowance for polling and delivery. */ export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health { diff --git a/src/lib/server/fleet/metrics.ts b/src/lib/server/fleet/metrics.ts index a6312d4..9ace2d1 100644 --- a/src/lib/server/fleet/metrics.ts +++ b/src/lib/server/fleet/metrics.ts @@ -87,7 +87,7 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()) let localCell = { index: 0, date: '' }; interface BatchCounts { known: number; active: number; idle: number; unknown: number; players: number } function addHeatmap(counts: BatchCounts, start: number, end: number): void { - // Samples are at most 45 seconds. Splitting at UTC minute boundaries + // Samples are bounded by the shared freshness window. Splitting at UTC minute boundaries // handles local hour/day boundaries, DST, and fractional-hour zones. // Cache each minute once across the complete fleet collection batches. for (let cursor = start; cursor < end;) { diff --git a/src/lib/server/fleet/servers.ts b/src/lib/server/fleet/servers.ts index 8d7ba29..05101a7 100644 --- a/src/lib/server/fleet/servers.ts +++ b/src/lib/server/fleet/servers.ts @@ -91,6 +91,7 @@ export interface ServerRow { last_roster: string | null; last_seen_at: string | null; last_error: string | null; + rcon_failures: number; last_reply_snippet: string | null; game_mode: string | null; map_slug: string | null; @@ -200,6 +201,7 @@ export function setServerFields(name: string, fields: Partial ReturnTyp const unavailable = updates.container_state === 'unknown'; if (!container.ok && !unavailable) { updates.last_error = null; + updates.rcon_failures = 0; updates.last_reply_snippet = null; } else if (unavailable || !telemetryEvidence(srv.name)) { flush(); @@ -192,7 +193,8 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp }); if (!probe) return null; const result = probe.result; - if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) { + const previous = getServer(srv.name); + if (unchanged() && previous && (previous.last_checked_at ?? '') <= probe.observedAt) { gameObservation = probe; const now = probe.observedAt; updates.last_checked_at = now; @@ -205,13 +207,23 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp updates.last_roster = JSON.stringify(result.status.roster); updates.last_seen_at = now; updates.last_error = null; - } else updates.last_error = encodeError(result.reason, result.detail); + updates.rcon_failures = 0; + } else { + // Only distinct accepted checks in the current lifecycle count. + // Skipped/superseded checks return above without changing the streak. + const consecutive = previous.last_error && (!epoch || (previous.last_checked_at ?? '') > epoch); + const failures = consecutive ? previous.rcon_failures || 1 : 0; + updates.rcon_failures = consecutive && previous.last_checked_at === now + ? failures : Math.min(2, failures + 1); + updates.last_error = encodeError(result.reason, result.detail); + } } } if (!settling && unchanged()) publishObservation(inspected, gameObservation); } } else { updates.last_error = null; + updates.rcon_failures = 0; updates.last_reply_snippet = null; } flush(); diff --git a/src/lib/server/provisioning/host-drain.ts b/src/lib/server/provisioning/host-drain.ts index 440e763..cae8d67 100644 --- a/src/lib/server/provisioning/host-drain.ts +++ b/src/lib/server/provisioning/host-drain.ts @@ -104,7 +104,7 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id); db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id); db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL, - last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,active_since=NULL,inactive_since=NULL, + last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL, game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server); db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id); db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id); diff --git a/src/lib/server/provisioning/restarts.ts b/src/lib/server/provisioning/restarts.ts index 5aacb1d..0b4d563 100644 --- a/src/lib/server/provisioning/restarts.ts +++ b/src/lib/server/provisioning/restarts.ts @@ -107,7 +107,7 @@ export async function restartInRollout(server: string, options: { operationId: s }); })(); requestsInFlight.set(server, operationId); - db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL, + db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, rcon_failures = 0, last_reply_snippet = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server); try { const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server); diff --git a/src/routes/api/servers/[name]/probe-rcon/+server.ts b/src/routes/api/servers/[name]/probe-rcon/+server.ts index 505b011..53401f2 100644 --- a/src/routes/api/servers/[name]/probe-rcon/+server.ts +++ b/src/routes/api/servers/[name]/probe-rcon/+server.ts @@ -38,11 +38,14 @@ export const POST: RequestHandler = async ({ params }) => { setServerField(name, 'last_seen_at', now); setServerField(name, 'last_checked_at', now); setServerField(name, 'last_error', null); + setServerField(name, 'rcon_failures', 0); setServerField(name, 'last_reply_snippet', result.rawReply); } else { const encoded = `${result.reason}:${result.detail}`; setServerField(name, 'last_checked_at', now); setServerField(name, 'last_error', encoded); + // An explicitly requested diagnostic failure takes effect immediately. + setServerField(name, 'rcon_failures', 2); setServerField(name, 'last_reply_snippet', result.rawReply ?? null); } diff --git a/src/routes/metrics/+page.svelte b/src/routes/metrics/+page.svelte index 3f7876f..f482e9c 100644 --- a/src/routes/metrics/+page.svelte +++ b/src/routes/metrics/+page.svelte @@ -1,5 +1,6 @@