Relax freshness limits and tolerate transient RCON failures

docs/maintenance.md

@@ -102,9 +102,18 @@ reachable. Finished scheduled and legacy maintenance activity is retained for
 
 ## Health recovery and configuration drift
 
+Fleet keeps successful game and running-container reports fresh for 90 seconds.
+Two consecutive failed monitoring checks mark RCON health stale; a first timeout
+or send failure remains visible in diagnostics without changing the health badge.
+Configuration and invalid-reply errors, including authentication failures, take
+effect immediately, as do failed manual probes. Successful RCON checks reset the
+failure count; skipped checks do not count. Without fresh telemetry, RCON evidence
+older than 120 seconds is offline. Explicit stopped/unhealthy container states
+and lifecycle changes still take precedence.
+
 The status monitor collects container and game health every 30 seconds. Once per
 minute, the watchdog evaluates that completed evidence without issuing its own
-routine health probes. Samples expire after 45 seconds; missing or stale evidence
+routine health probes. Samples expire after 90 seconds; missing or stale evidence
 leaves health unknown and cannot justify recovery. Samples retain their original
 timestamps and instance identity, and each failure sample counts at most once.
 A monitoring gap or changed instance resets the sustained failure window. After

docs/metrics.md

@@ -78,7 +78,7 @@ a change; a retained map may no longer have a hosted lobby.
 - Player-hours integrate the reported player count over observed time. Bots are included if the server counts them as players.
 - All lobbies full measures map wall time when every tracked lobby in that game/map group has a known capacity and is full. Missing capacity or status makes that interval unknown. This is a capacity-pressure indicator, not a count of rejected joins or unmet demand.
 
-Observations are recorded by the existing status worker. Each remains valid until the next collection, up to the 45-second freshness limit. Longer manager outages are excluded. Charts split time at UTC hour/day boundaries; partial periods remain partial. Map variants use the configured variant when it matches the reported map.
+Observations are recorded by the existing status worker. Each remains valid until the next collection, up to the 90-second freshness limit. Longer manager outages are excluded. Charts split time at UTC hour/day boundaries; partial periods remain partial. Map variants use the configured variant when it matches the reported map.
 
 Capacity comes from `sv_maxclients` / `sv_maxClients` in the status response. When unavailable, the UI displays unknown rather than assuming a four-player limit. Capacity coverage is the map time for which every tracked lobby reported both occupancy and capacity.
 

docs/provisioning.md

@@ -57,7 +57,7 @@ unreachable Komodo API is not evidence that a stop succeeded.
 Disruptive operations require a fresh zero-player result from RCON or accepted
 telemetry, unless Komodo positively reports a stopped container. Fresh occupied
 telemetry vetoes a conflicting empty RCON reading. Reports must be received within
-45 seconds; recovery verification also excludes reports from before the submitted
+90 seconds; recovery verification also excludes reports from before the submitted
 operation. Telemetry can verify the game's activity and selected map after Komodo
 reports healthy. Runtime actions wait for conflicting
 campaign, beta, maintenance, or provisioning work. Duplicate pending requests for

src/lib/server-freshness.ts

@@ -1,6 +1,6 @@
-// Allow a polling cycle plus time for queued RCON requests and delivery.
+// Allow missed polling cycles and queued requests across all status evidence.
 export const STATUS_REFRESH_MS = 30_000;
-export const ONLINE_MAX_MS = STATUS_REFRESH_MS + 15_000;
+export const ONLINE_MAX_MS = 90_000;
 // The UI clock ticks once per second and can trail a newly arrived report.
 const CLOCK_TOLERANCE_MS = 1_000;
 

src/lib/server-observation.ts

@@ -14,6 +14,8 @@ export interface GameObservation {
     last_gametype: string | null;
     last_seen_at: string | null;
     last_error: string | null;
+    /** Consecutive failed monitoring checks; absent on older snapshots. */
+    rcon_failures?: number;
     roster: StatusPlayer[] | null;
 }
 export interface ServerObservation extends GameObservation {
@@ -32,6 +34,12 @@ export interface ServerObservation extends GameObservation {
     game_ended_at?: string | null;
 }
 
+/** Keep a first transient monitoring failure in diagnostics without demoting health. */
+export function rconHealthError(status: Pick<GameObservation, 'last_error' | 'rcon_failures'>): string | null {
+    const reason = status.last_error?.split(':')[0];
+    return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error') ? null : status.last_error;
+}
+
 export function freshTelemetry(report: GameTelemetry | undefined, now: number, after?: string | null): report is GameTelemetry {
     if (!report || !freshStatusTimestamp(report.receivedAt, now)) return false;
     if (after && (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))) return false;
@@ -42,7 +50,7 @@ export function freshTelemetry(report: GameTelemetry | undefined, now: number, a
 export function selectServerObservation<T extends ServerObservation>(server: T, report: GameTelemetry | undefined, now: number): T {
     if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
     const raw = server.rcon_status ?? server;
-    const result: T = { ...server, ...raw, telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
+    const result: T = { ...server, ...raw, last_error: rconHealthError(raw), telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
     const containerUnknown = server.container_state === 'unknown';
     if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) {
         return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null };
@@ -69,11 +77,12 @@ export function selectServerObservation<T extends ServerObservation>(server: T,
             game_ended_at: players === 0 ? null : timing.gameEndedAt };
     }
     const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
-    const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !raw.last_error;
+    const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !result.last_error;
     // A current authenticated RCON reply can establish game liveness when
     // inspection is unavailable, without claiming the container was inspected.
     const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh;
     const age = now - Date.parse(raw.last_seen_at ?? '');
-    return { ...result, health: fresh && !containerExpired ? 'online'
+    const health = fresh && !containerExpired ? 'online'
-        : after && age >= 0 && age <= 120_000 ? 'stale' : 'offline' };
+        : after && age >= 0 && age <= 120_000 ? 'stale' : 'offline';
+    return { ...result, health, last_error: health === 'online' ? result.last_error : raw.last_error };
 }

src/lib/server/db/index.ts

@@ -35,6 +35,7 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
     { name: 'last_roster', ddl: 'ALTER TABLE servers ADD COLUMN last_roster TEXT' },
     { name: 'last_seen_at', ddl: 'ALTER TABLE servers ADD COLUMN last_seen_at TEXT' },
     { name: 'last_error', ddl: 'ALTER TABLE servers ADD COLUMN last_error TEXT' },
+    { name: 'rcon_failures', ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0' },
     { name: 'last_reply_snippet', ddl: 'ALTER TABLE servers ADD COLUMN last_reply_snippet TEXT' },
     // Script-scope columns — static per server, populated via server-edit UI.
     { name: 'game', ddl: 'ALTER TABLE servers ADD COLUMN game TEXT' },

src/lib/server/db/schema.sql

@@ -80,6 +80,7 @@ CREATE TABLE IF NOT EXISTS servers (
     last_roster      TEXT,                                -- JSON array of RconPlayer
     last_seen_at     TEXT,                                -- ISO ts of last successful RCON reply
     last_error       TEXT,                                -- last RCON failure reason (cleared on success)
+    rcon_failures    INTEGER NOT NULL DEFAULT 0,          -- consecutive monitoring failures, capped at two
     last_reply_snippet TEXT,                              -- hex+ASCII of last bad-reply UDP payload
     -- Provisioning fields — set when Electric Cherry creates the deployment.
     game_mode        TEXT,                                -- 'zombie' | 'mp' | 'sp' (currently zombie only)

src/lib/server/fleet/beta-tests.ts

@@ -5,6 +5,7 @@ import { db } from '../db/index.js';
 import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
 import type { GameTelemetry } from '../../telemetry.js';
 import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
 
 export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
 export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
@@ -64,7 +65,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
     const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
     const sample = Date.parse(timestamp ?? '');
     const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
-        && Date.now() - sample >= 0 && Date.now() - sample <= 45_000;
+        && Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS;
     const healthy = freshSample && server?.health === 'online';
     const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
     const before = m.wasHealthy;

src/lib/server/fleet/health.ts

@@ -6,7 +6,7 @@ const STALE_MAX_MS = 120_000;
 /**
  * Classify server health from its last successful RCON timestamp.
  * `offline` if we've never seen it or the last successful reply is >120s old.
- * `stale` on an RCON failure or if the last reply is 45-120s old.
+ * `stale` on a blocking RCON failure or if the last reply is >90-120s old.
  * `online` if fresh, including the allowance for polling and delivery.
  */
 export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {

src/lib/server/fleet/metrics.ts

@@ -87,7 +87,7 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
     let localCell = { index: 0, date: '' };
     interface BatchCounts { known: number; active: number; idle: number; unknown: number; players: number }
     function addHeatmap(counts: BatchCounts, start: number, end: number): void {
-        // Samples are at most 45 seconds. Splitting at UTC minute boundaries
+        // Samples are bounded by the shared freshness window. Splitting at UTC minute boundaries
         // handles local hour/day boundaries, DST, and fractional-hour zones.
         // Cache each minute once across the complete fleet collection batches.
         for (let cursor = start; cursor < end;) {

src/lib/server/fleet/servers.ts

@@ -91,6 +91,7 @@ export interface ServerRow {
     last_roster: string | null;
     last_seen_at: string | null;
     last_error: string | null;
+    rcon_failures: number;
     last_reply_snippet: string | null;
     game_mode: string | null;
     map_slug: string | null;
@@ -200,6 +201,7 @@ export function setServerFields(name: string, fields: Partial<Record<keyof Serve
         'last_roster',
         'last_seen_at',
         'last_error',
+        'rcon_failures',
         'last_reply_snippet',
         'game_mode',
         'map_slug',

src/lib/server/fleet/status.ts

@@ -1,4 +1,4 @@
-import { selectServerObservation, type GameObservation } from '../../server-observation.js';
+import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js';
 import { classifyHealth } from './health.js';
 import { isProvisioning } from './provisioning.js';
 import { restartSettling } from '../provisioning/restarts.js';
@@ -44,11 +44,12 @@ export function serverObservation(server: ServerRow, evidence?: Evidence) {
     let roster = null;
     try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
     const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
-        last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
+        last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error,
+        rcon_failures: server.rcon_failures, roster };
     const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
     const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
     const ready = !provisioning && (!state || state === 'running' || state === 'unknown');
-    const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
+    const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, rconHealthError(rcon), state);
     return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
         status_source: null as 'telemetry' | 'rcon' | null,
         telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,

src/lib/server/pipeline/worker.ts

@@ -177,6 +177,7 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
             const unavailable = updates.container_state === 'unknown';
             if (!container.ok && !unavailable) {
                 updates.last_error = null;
+                updates.rcon_failures = 0;
                 updates.last_reply_snippet = null;
             } else if (unavailable || !telemetryEvidence(srv.name)) {
                 flush();
@@ -192,7 +193,8 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
                 });
                 if (!probe) return null;
                 const result = probe.result;
-                if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) {
+                const previous = getServer(srv.name);
+                if (unchanged() && previous && (previous.last_checked_at ?? '') <= probe.observedAt) {
                     gameObservation = probe;
                     const now = probe.observedAt;
                     updates.last_checked_at = now;
@@ -205,13 +207,23 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
                         updates.last_roster = JSON.stringify(result.status.roster);
                         updates.last_seen_at = now;
                         updates.last_error = null;
-                    } else updates.last_error = encodeError(result.reason, result.detail);
+                        updates.rcon_failures = 0;
+                    } else {
+                        // Only distinct accepted checks in the current lifecycle count.
+                        // Skipped/superseded checks return above without changing the streak.
+                        const consecutive = previous.last_error && (!epoch || (previous.last_checked_at ?? '') > epoch);
+                        const failures = consecutive ? previous.rcon_failures || 1 : 0;
+                        updates.rcon_failures = consecutive && previous.last_checked_at === now
+                            ? failures : Math.min(2, failures + 1);
+                        updates.last_error = encodeError(result.reason, result.detail);
+                    }
                 }
             }
             if (!settling && unchanged()) publishObservation(inspected, gameObservation);
         }
     } else {
         updates.last_error = null;
+        updates.rcon_failures = 0;
         updates.last_reply_snippet = null;
     }
     flush();

src/lib/server/provisioning/host-drain.ts

@@ -104,7 +104,7 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance
             db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
             db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
             db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
-                last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,active_since=NULL,inactive_since=NULL,
+                last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL,
                 game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
             db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
             db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);

src/lib/server/provisioning/restarts.ts

@@ -107,7 +107,7 @@ export async function restartInRollout(server: string, options: { operationId: s
         });
     })();
     requestsInFlight.set(server, operationId);
-    db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
+    db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, rcon_failures = 0, last_reply_snippet = NULL,
         game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
     try {
         const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);

src/routes/api/servers/[name]/probe-rcon/+server.ts

@@ -38,11 +38,14 @@ export const POST: RequestHandler = async ({ params }) => {
         setServerField(name, 'last_seen_at', now);
         setServerField(name, 'last_checked_at', now);
         setServerField(name, 'last_error', null);
+        setServerField(name, 'rcon_failures', 0);
         setServerField(name, 'last_reply_snippet', result.rawReply);
     } else {
         const encoded = `${result.reason}:${result.detail}`;
         setServerField(name, 'last_checked_at', now);
         setServerField(name, 'last_error', encoded);
+        // An explicitly requested diagnostic failure takes effect immediately.
+        setServerField(name, 'rcon_failures', 2);
         setServerField(name, 'last_reply_snippet', result.rawReply ?? null);
     }
 

src/routes/metrics/+page.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
     import { onMount, untrack } from 'svelte';
+    import { ONLINE_MAX_MS } from '$lib/server-freshness.js';
     import { page } from '$app/state';
     import { afterNavigate, beforeNavigate, goto } from '$app/navigation';
     import { timeZone } from '$ui/stores/time-zone.js';
@@ -169,7 +170,7 @@
     <details class="methodology">
         <summary>How these estimates work</summary>
         <p>Time with players is occupied server time divided by known server time. Known time includes occupied and idle observations; unknown status and collection gaps are excluded from that percentage.</p>
-        <p>Each periodic observation covers at most 45 seconds. Longer collection gaps are excluded. History is retained for 31 days. Player-hours include all players reported by the server and are estimates, not individual session records.</p>
+        <p>Each periodic observation covers at most {ONLINE_MAX_MS / 1000} seconds. Longer collection gaps are excluded. History is retained for 31 days. Player-hours include all players reported by the server and are estimates, not individual session records.</p>
         <p>Timeline buckets are aligned to UTC, with labels shown in {report.scope.timeZone.replaceAll('_', ' ')}. Daily buckets are 24-hour periods, not local calendar days. The first and latest buckets can be partial. Percentage bars divide occupied, idle, and unknown time by captured time; the headline occupancy percentage uses known time only.</p>
         <p>Historical server rows are grouped by name and may include multiple registrations using that name. Map groups preserve the recorded game and map label.</p>
     </details>

tests/unit/beta-plans.test.ts

@@ -96,7 +96,7 @@ describe('durable beta test plans', () => {
     it('uses script events instead of logs and keeps match progress across app restarts', async () => {
         await postTelemetry(gameEvent('game_end'));
         const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches')] });
-        await tick(46); await tick();
+        await tick(91); await tick();
         const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
         await appendFile(join(fixture.t5, names[0], 'logs/games_sp.log'), 'InitGame:\n 0:00 J;new;1;Josh\nExitLevel: executed\n');
         await tick();
@@ -113,7 +113,7 @@ describe('durable beta test plans', () => {
     it('automatically credits ordinary rounds only to active deployed beta servers', async () => {
         const id = await openPlan({ ...defaultPlan(), waveSize: 1, requirements: [requirement('round', 'round')] }, names);
         await postTelemetry(gameEvent('round_change'));
-        await tick(46); await tick();
+        await tick(91); await tick();
         const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
         expect(testView(id)?.servers[0].progress[0].current).toBe(0);
         await postTelemetry(gameEvent('round_change', 99, 4, names[1]));
@@ -135,7 +135,7 @@ describe('durable beta test plans', () => {
         const first = currentRun(id, names[0])!;
         expect(testView(id)?.servers[0].progress[0].current).toBe(0);
         await postTelemetry(gameEvent('game_start'));
-        await tick(46); await tick();
+        await tick(91); await tick();
         expect(currentRun(id, names[0])?.id).not.toBe(first.id);
         expect(currentRun(id, names[0])?.phase).toBe(1);
         expect(testView(id)?.servers[0].progress[0].current).toBe(0);
@@ -162,7 +162,7 @@ describe('durable beta test plans', () => {
         await tick(15, 0, false);
         // Fresh occupied telemetry remains healthy despite an RCON timeout.
         expect(getBeta(id)?.state).toBe('ready-promote');
-        await tick(31, 0, false);
+        await tick(61, 0, false);
         expect(getBeta(id)?.state).toBe('running');
         await expect(promoteBeta(id)).rejects.toThrow();
     });
@@ -236,7 +236,7 @@ describe('durable beta test plans', () => {
         expect(retry.status).toBe(200);
         await postTelemetry(gameEvent('round_change'));
         expect(testView(id)?.servers).toHaveLength(0);
-        await tick(46); await tick();
+        await tick(91); await tick();
         expect(testView(id)?.servers[0].progress[0].current).toBe(0);
         await postTelemetry(gameEvent('round_change'));
         expect(testView(id)?.servers[0].progress[0].current).toBe(25);

tests/unit/fleet-overview.test.ts

@@ -31,14 +31,14 @@ describe('Fleet discovery and match freshness', () => {
         expect(fleetIdentity({ ...server, game_mode: 'mp' }).launch).toBeNull();
     });
     it('does not classify stale, missing, or starting occupancy as an empty lobby', () => {
-        for (const change of [{ last_seen_at: iso(-46_000) }, { last_players: null }, { health: 'starting' as const }, { health: 'offline' as const }]) {
+        for (const change of [{ last_seen_at: iso(-91_000) }, { last_players: null }, { health: 'starting' as const }, { health: 'offline' as const }]) {
             expect(matchesFleet({ ...server, last_players: 0, ...change }, '', 'empty', '', [], [], now)).toBe(false);
         }
         expect(matchesFleet({ ...server, last_players: 0 }, '', 'empty', '', [], [], now)).toBe(true);
         expect(matchesFleet(server, '', 'occupied', 't5', [], [], now)).toBe(false);
     });
     it('includes stale health and failed operations in attention without flagging normal startup', () => {
-        expect(matchesFleet({ ...server, last_seen_at: iso(-46_000) }, '', 'attention', '', [], [], now)).toBe(true);
+        expect(matchesFleet({ ...server, last_seen_at: iso(-91_000) }, '', 'attention', '', [], [], now)).toBe(true);
         expect(matchesFleet(server, '', 'attention', '', [], [{ ...operation, state: 'attention' }], now)).toBe(true);
         expect(matchesFleet({ ...server, health: 'starting' }, '', 'attention', '', [], [], now)).toBe(false);
     });

tests/unit/health-observations.test.ts

@@ -61,7 +61,7 @@ describe('shared monitoring observations', () => {
         expect(api.monitoringStatus(server)?.game?.observedAt).toBe(new Date(now).toISOString());
         expect(api.inspect).toHaveBeenCalledTimes(1);
         expect(api.rcon).toHaveBeenCalledTimes(1);
-        vi.setSystemTime(now + 45_001);
+        vi.setSystemTime(now + 90_001);
         expect(api.monitoringStatus(server)).toBeNull();
         vi.setSystemTime(now - 1);
         expect(api.monitoringStatus(server)).toBeNull();
@@ -108,7 +108,7 @@ describe('shared monitoring observations', () => {
     it('excludes old game evidence even when the container sample is fresh', async () => {
         const api = await setup();
         const game = await api.rconForMonitoring(server);
-        vi.setSystemTime(now + 46_000);
+        vi.setSystemTime(now + 91_000);
         const publish = api.monitoringStatusWriter(server);
         publish(await api.inspectForMonitoring(server), game);
         expect(api.monitoringStatus(server)?.container.result).toEqual(healthy);

tests/unit/host-status-batch.test.ts

@@ -123,6 +123,7 @@ describe('host status observations', () => {
         const first = api.fanoutStatus();
         await vi.waitFor(() => expect(api.operations()).toHaveLength(1));
         api.complete(); await first;
+        api.db.prepare("UPDATE servers SET rcon_failures=1,last_error='timeout:first failure'").run();
         const previous = api.getServer('server-0-0')!;
         const previousGame = api.monitoringStatus('server-0-0')?.game;
         vi.setSystemTime(at + 31_000);
@@ -138,9 +139,11 @@ describe('host status observations', () => {
         } finally { api.complete(); await pass; }
         expect(api.getServer('server-0-0')).toMatchObject({ last_checked_at: previous.last_checked_at,
             last_seen_at: previous.last_seen_at, last_error: previous.last_error, last_players: previous.last_players,
-            last_reply_snippet: previous.last_reply_snippet });
+            last_reply_snippet: previous.last_reply_snippet, rcon_failures: 1 });
         expect(api.monitoringStatus('server-0-0')?.game).toEqual(previousGame);
         expect(api.getServer('server-0-1')?.last_error).toContain('timeout:');
+        expect(api.getServer('server-0-1')?.rcon_failures).toBe(2);
+        expect(api.getServer('server-0-2')?.rcon_failures).toBe(0);
         expect(Date.parse(api.getServer('server-0-2')!.last_seen_at!)).toBeGreaterThanOrEqual(at + 31_000);
     });
 

tests/unit/metrics.test.ts

@@ -17,7 +17,7 @@ describe('historical metrics', () => {
         const result = readMetrics(1, at + 3600_000);
         expect(result.summary.active * 3600).toBeCloseTo(30);
         expect(result.summary.idle * 3600).toBeCloseTo(30);
-        expect(result.summary.unknown * 3600).toBeCloseTo(45);
+        expect(result.summary.unknown * 3600).toBeCloseTo(90);
         expect(result.summary.playerHours * 3600).toBeCloseTo(60);
         expect(result.points.reduce((sum, p) => sum + p.observed, 0)).toBeCloseTo(result.summary.observed);
     });
@@ -43,7 +43,7 @@ describe('historical metrics', () => {
         let result = readMetrics(1, midnight + 15_000);
         expect(result.points.filter(p => p.active > 0).map(p => p.active * 3600)).toEqual([15, 15]);
         result = readMetrics(1, midnight + 86400_000);
-        expect(result.summary.active * 3600).toBeCloseTo(30);
+        expect(result.summary.active * 3600).toBeCloseTo(75);
         expect(result.points.reduce((sum, p) => sum + p.active, 0)).toBeCloseTo(result.summary.active);
     });
 
@@ -84,7 +84,7 @@ describe('historical metrics', () => {
         expect(initial.summary).toEqual({ observed: 0, active: 0, idle: 0, unknown: 0, playerHours: 0 });
         const later = readMetrics(1, at + 120_000);
         expect(later.summary.observed).toBe(0);
-        expect(later.summary.unknown * 3600).toBeCloseTo(45);
+        expect(later.summary.unknown * 3600).toBeCloseTo(90);
         expect(later.maps[0].capacityObserved).toBe(0);
     });
 
@@ -112,10 +112,10 @@ describe('historical metrics', () => {
         // Removing a lobby ends its previous observation at the next complete collection.
         recordMetrics([{ ...one, map: 'Farm', players: 0 }], at + 30_000);
         // A collection outage is excluded; an identically named map in another game remains separate.
-        recordMetrics([{ ...one, game: 't5', players: 2, capacity: null }], at + 120_000);
+        recordMetrics([{ ...one, game: 't5', players: 2, capacity: null }], at + 150_000);
-        const result = readMetrics(1, at + 150_000);
+        const result = readMetrics(1, at + 180_000);
         expect(result.summary.active * 3600).toBeCloseTo(90);
-        expect(result.summary.idle * 3600).toBeCloseTo(45);
+        expect(result.summary.idle * 3600).toBeCloseTo(90);
         expect(result.summary.playerHours * 3600).toBeCloseTo(300);
         expect(result.servers.find(server => server.server === 'two')!.observed * 3600).toBeCloseTo(30);
         const town = result.maps.find(map => map.game === 't6' && map.map === 'Town')!;
@@ -244,7 +244,7 @@ describe('historical metrics', () => {
         expect(result.previous.until).toBe(result.since);
         expect(result.previous.summary.active * 3600).toBeCloseTo(15);
         expect(result.summary.active * 3600).toBeCloseTo(15);
-        expect(result.summary.idle * 3600).toBeCloseTo(45);
+        expect(result.summary.idle * 3600).toBeCloseTo(90);
         expect(result.previous).toMatchObject({ firstSampleAt: since - 15_000, lastSampleAt: since - 15_000, partialHistory: true });
     });
 
@@ -258,10 +258,10 @@ describe('historical metrics', () => {
         recordMetrics([sample], now - 30_000);
         const result = readMetrics(30, now);
         expect(result.previous).toMatchObject({ since: now - 60 * day, until: now - 30 * day, partialHistory: true });
-        expect(result.previous.summary.active * 3600).toBeCloseTo(45);
+        expect(result.previous.summary.active * 3600).toBeCloseTo(90);
         expect(result.previous.firstSampleAt).toBe(now - 31 * day + 30_000);
         expect(result.summary.active * 3600).toBeCloseTo(30);
-        expect(result.summary.idle * 3600).toBeCloseTo(45);
+        expect(result.summary.idle * 3600).toBeCloseTo(90);
     });
 
     it('marks the retained start bound separately from gaps inside the previous window', async () => {
@@ -273,7 +273,7 @@ describe('historical metrics', () => {
         recordMetrics([sample], now - 30_000);
         const result = readMetrics(1, now);
         expect(result.previous.partialHistory).toBe(false);
-        expect(result.previous.summary.unknown * 3600).toBeCloseTo(30);
+        expect(result.previous.summary.unknown * 3600).toBeCloseTo(75);
         expect(result.previous.summary.observed).toBe(0);
         expect(result.previous.firstSampleAt).toBe(priorStart - 15_000);
         // Having the left edge says nothing about the uncaptured time after it.
@@ -326,8 +326,8 @@ describe('historical metrics', () => {
         recordMetrics([{ ...sample, players: 0 }], at + 3600_000);
         const result = readMetrics(parseMetricsQuery(new URLSearchParams({ days: '1', timeZone: 'America/New_York' })), at + 3630_000);
         const cell = result.heatmap.cells.find(cell => cell.weekday === 6 && cell.hour === 1)!;
-        expect(cell.observed * 3600).toBeCloseTo(75);
+        expect(cell.observed * 3600).toBeCloseTo(120);
-        expect(cell.active * 3600).toBeCloseTo(45);
+        expect(cell.active * 3600).toBeCloseTo(90);
         expect(cell.idle * 3600).toBeCloseTo(30);
         expect(cell.distinctDates).toBe(1);
         expect(result.heatmap.cells.filter(cell => cell.observed > 0)).toHaveLength(1);

tests/unit/provisioning.test.ts

@@ -368,7 +368,7 @@ describe('Watchdog provisioning and telemetry evidence', () => {
         const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
         const { db } = await import('../../src/lib/server/db/index.js');
         vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
-        for (const offset of [-45_001, 1]) {
+        for (const offset of [-90_001, 1]) {
             await report(0, new Date(Date.now() + offset).toISOString());
             expect(telemetryEvidence(server)).toBeNull();
             expect(await probePlayerCount(server)).toBeNull();

tests/unit/server-freshness.test.ts

@@ -9,10 +9,10 @@ it('keeps healthy reports online across polling and delivery delays, then expire
     vi.useFakeTimers();
     const reportAt = Date.parse('2026-09-18T12:00:00Z');
     const timestamp = new Date(reportAt).toISOString();
-    for (const elapsed of [-999, 0, STATUS_REFRESH_MS, STATUS_REFRESH_MS + 4000, 45_000, 45_001]) {
+    for (const elapsed of [-999, 0, STATUS_REFRESH_MS, STATUS_REFRESH_MS + 4000, 45_000, 60_000, 90_000, 90_001, 120_000]) {
         const now = reportAt + elapsed;
         vi.setSystemTime(now);
-        const expected = elapsed <= 45_000;
+        const expected = elapsed <= 90_000;
         const health = classifyHealth(timestamp, null, 'running');
         expect(health).toBe(expected ? 'online' : 'stale');
         expect(freshServer({ health, last_seen_at: timestamp, last_error: null } as FleetServer, now)).toBe(expected);

tests/unit/server-observation.test.ts

@@ -12,6 +12,32 @@ function server(changes: Partial<ServerObservation> = {}): ServerObservation {
 }
 
 describe('shared server observation', () => {
+    it('tolerates only the first transient failure while retaining diagnostics and expiring evidence', () => {
+        for (const reason of ['timeout', 'send-error']) {
+            const raw = { ...server(), last_error: `${reason}:no reply`, rcon_failures: 1 };
+            const selected = selectServerObservation(server({ rcon_status: raw }), undefined, now);
+            expect(selected).toMatchObject({ health: 'online', last_error: null, rcon_status: raw });
+            expect(selectServerObservation(selected, undefined, now + 90_000).health).toBe('online');
+            expect(selectServerObservation(selected, undefined, now + 90_001).health).toBe('stale');
+            expect(selectServerObservation(selected, undefined, now + 120_001).health).toBe('offline');
+            expect(selectServerObservation(server({ rcon_status: { ...raw, rcon_failures: 2 } }), undefined, now))
+                .toMatchObject({ health: 'stale', last_error: raw.last_error });
+        }
+        for (const reason of ['no-config', 'no-port', 'bad-reply', 'no-players']) {
+            expect(selectServerObservation(server({ last_error: `${reason}:failed`, rcon_failures: 1 }), undefined, now).health).toBe('stale');
+        }
+        expect(selectServerObservation(server({ last_error: 'timeout:legacy failure' }), undefined, now).health).toBe('stale');
+    });
+
+    it('accepts telemetry and container evidence through 90 seconds, then expires each independently', () => {
+        expect(selectServerObservation(server(), report(), now + 90_000)).toMatchObject({ health: 'online', status_source: 'telemetry' });
+        expect(selectServerObservation(server(), report(90_001), now + 90_001).health).toBe('stale');
+        const current = server({ container_checked_at: at(90_001) });
+        expect(selectServerObservation(current, report(), now + 90_001).health).toBe('stale');
+        expect(freshTelemetry(report(-90_000), now)).toBe(true);
+        expect(freshTelemetry(report(-90_001), now)).toBe(false);
+    });
+
     it('selects the telemetry count and roster together and keeps raw RCON evidence', () => {
         const raw = server({ last_error: 'timeout' });
         const selected = selectServerObservation(server({ rcon_status: raw }), report(), now);
@@ -25,8 +51,8 @@ describe('shared server observation', () => {
         const original = server();
         const selected = selectServerObservation(server({ rcon_status: original, telemetry: report(0, 2) }), report(-1000), now);
         expect(selected.last_players).toBe(2);
-        const fallback = selectServerObservation({ ...selected, container_checked_at: at(46_000),
+        const fallback = selectServerObservation({ ...selected, container_checked_at: at(91_000),
-            rcon_status: { ...original, last_seen_at: at(40_000) } }, undefined, now + 46_000);
+            rcon_status: { ...original, last_seen_at: at(40_000) } }, undefined, now + 91_000);
         expect(fallback).toMatchObject({ status_source: 'rcon', health: 'online', last_players: 4, last_seen_at: at(40_000) });
     });
 
@@ -40,19 +66,19 @@ describe('shared server observation', () => {
     it('uses fresh RCON with unknown container state, but does not substitute telemetry or extend freshness', () => {
         const raw = server({ container_state: 'unknown', container_checked_at: at(-60_000) });
         expect(selectServerObservation(raw, report(), now)).toMatchObject({ health: 'online', last_players: 4, status_source: 'rcon' });
-        expect(selectServerObservation(raw, report(46_000), now + 46_000).health).toBe('stale');
+        expect(selectServerObservation(raw, report(91_000), now + 91_000).health).toBe('stale');
         expect(selectServerObservation(raw, report(121_000), now + 121_000).health).toBe('offline');
         expect(selectServerObservation({ ...raw, telemetry_after: at() }, report(), now).health).toBe('offline');
         expect(selectServerObservation({ ...raw, last_seen_at: null }, report(), now).health).toBe('offline');
     });
 
     it('requires fresh container evidence and reports after the latest lifecycle submission', () => {
-        expect(selectServerObservation(server({ container_checked_at: at(-46_000) }), report(), now).health).not.toBe('online');
+        expect(selectServerObservation(server({ container_checked_at: at(-91_000) }), report(), now).health).not.toBe('online');
         expect(selectServerObservation(server({ health: 'provisioning' }), report(), now).health).toBe('provisioning');
         expect(selectServerObservation(server({ telemetry_after: at() }), report(-1), now).health).toBe('offline');
         expect(freshTelemetry(report(), now, 'invalid')).toBe(false);
         expect(freshTelemetry(report(2000), now)).toBe(false);
-        expect(freshTelemetry(report(-46_000), now)).toBe(false);
+        expect(freshTelemetry(report(-91_000), now)).toBe(false);
         expect(freshTelemetry({ ...report(), players: [{}] } as never, now)).toBe(false);
     });
 

tests/unit/server-status.test.ts

@@ -30,6 +30,77 @@ async function setup() {
 }
 
 describe('database fleet and container readiness', () => {
+    it('requires two distinct failed monitoring checks, preserves diagnostics, and resets on success', async () => {
+        const { inspect, rcon, fanoutStatus, getServer } = await setup();
+        const { GET } = await import('../../src/routes/api/servers/+server.js');
+        const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
+        const snapshot = async () => (await (await GET()).json()).servers[0];
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const at = Date.parse('2026-10-05T12:00:00Z');
+        vi.setSystemTime(at);
+        inspect.mockResolvedValue({ ok: true, state: 'running' });
+        const success = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
+            status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } };
+        rcon.mockResolvedValue(success);
+        await fanoutStatus();
+        const events: unknown[] = [];
+        const unsubscribe = subscribe(event => events.push(event));
+        try {
+            rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
+            vi.setSystemTime(at + 30_000);
+            await fanoutStatus();
+            expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:no reply' });
+            expect(await snapshot()).toMatchObject({ health: 'online', last_error: null,
+                rcon_status: { last_error: 'timeout:no reply', rcon_failures: 1 } });
+            expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
+                health: 'online', lastError: null, rconStatus: expect.objectContaining({ rcon_failures: 1, last_error: 'timeout:no reply' })
+            })] }));
+            const { closeDb } = await import('../../src/lib/server/db/index.js');
+            closeDb();
+            expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } });
+            // Reprocessing a sample with the same timestamp cannot count it twice.
+            await fanoutStatus();
+            expect(getServer('managed')?.rcon_failures).toBe(1);
+            vi.setSystemTime(at + 60_000);
+            await fanoutStatus();
+            expect(await snapshot()).toMatchObject({ health: 'stale', last_error: 'timeout:no reply', rcon_status: { rcon_failures: 2 } });
+            rcon.mockResolvedValue(success);
+            vi.setSystemTime(at + 90_000);
+            await fanoutStatus();
+            expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: null });
+            rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'send-error', detail: 'unreachable' });
+            vi.setSystemTime(at + 120_000);
+            await fanoutStatus();
+            expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } });
+            vi.setSystemTime(at + 180_001);
+            expect(await snapshot()).toMatchObject({ health: 'stale' });
+            vi.setSystemTime(at + 210_001);
+            expect(await snapshot()).toMatchObject({ health: 'offline' });
+        } finally { unsubscribe(); }
+    });
+
+    it('adds the failure counter to an existing database without suppressing legacy errors', async () => {
+        const { db, getServer } = await setup();
+        const { closeDb } = await import('../../src/lib/server/db/index.js');
+        db.exec('ALTER TABLE servers DROP COLUMN rcon_failures');
+        closeDb();
+        expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: 'timeout:old process' });
+    });
+
+    it('does not carry a failure streak across a new lifecycle submission', async () => {
+        const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const at = Date.parse('2026-10-05T12:00:00Z');
+        vi.setSystemTime(at);
+        db.prepare("UPDATE servers SET rcon_failures=1,last_checked_at=?").run(new Date(at - 30_000).toISOString());
+        db.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
+            VALUES('new-start','managed','restart','done','done',?,?,'test')`).run(new Date(at - 10_000).toISOString(), new Date(at - 10_000).toISOString());
+        inspect.mockResolvedValue({ ok: true, state: 'running' });
+        rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'new process' });
+        await fanoutStatus();
+        expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:new process' });
+    });
+
     it('shares routine observations without renewing their timestamps and keeps manual probes fresh', async () => {
         const { inspect, rcon, fanoutStatus, getServer } = await setup();
         const { inspectForMonitoring, rconForMonitoring } = await import('../../src/lib/server/fleet/health-observations.js');
@@ -193,7 +264,7 @@ describe('database fleet and container readiness', () => {
         const pending = fanoutStatus();
         try {
             await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
-            vi.setSystemTime(now + 46_000);
+            vi.setSystemTime(now + 91_000);
         } finally {
             release({ ok: true, state: 'running' });
             await pending;
@@ -444,7 +515,7 @@ describe('database fleet and container readiness', () => {
         const { GET } = await import('../../src/routes/api/servers/+server.js');
         inspect.mockResolvedValue({ ok: true, state: 'running' });
         await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
-        db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 46_000).toISOString());
+        db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 91_000).toISOString());
         rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
             status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
         await fanoutStatus();
@@ -515,6 +586,10 @@ describe('database fleet and container readiness', () => {
         rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
         vi.setSystemTime(Date.now() + 30_000);
         await fanoutStatus();
+        expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null,
+            rcon_status: { rcon_failures: 1, last_error: 'timeout:no reply' } });
+        vi.setSystemTime(Date.now() + 30_000);
+        await fanoutStatus();
         expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' });
     });
 

tests/unit/ui-server-clock.test.ts

@@ -287,7 +287,7 @@ describe('Fleet report clock', () => {
         expect(get(fleetServers)[0]).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry' });
         expect(get(fleetServers)[0].roster).toHaveLength(3);
         expect(matchesFleet(get(fleetServers)[0], 'Player 3', 'occupied', '', [], [], get(uiNow))).toBe(true);
-        await vi.advanceTimersByTimeAsync(46_000);
+        await vi.advanceTimersByTimeAsync(91_000);
         expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
         expect(matchesFleet(get(fleetServers)[0], '', 'occupied', '', [], [], get(uiNow))).toBe(false);
     });
@@ -368,15 +368,33 @@ describe('Fleet report clock', () => {
     it('keeps advancing through wall-clock changes and expires a report if updates stop', async () => {
         const { fleetServers, uiNow } = await setup();
         vi.setSystemTime(epoch - 120_000);
-        await vi.advanceTimersByTimeAsync(45_000);
+        await vi.advanceTimersByTimeAsync(90_000);
-        expect(get(uiNow)).toBe(epoch + 45_000);
+        expect(get(uiNow)).toBe(epoch + 90_000);
         expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true);
         await vi.advanceTimersByTimeAsync(1000);
-        expect(get(uiNow)).toBe(epoch + 46_000);
+        expect(get(uiNow)).toBe(epoch + 91_000);
         expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
         expect(matchesFleet(get(fleetServers)[0], '', 'attention', '', [], [], get(uiNow))).toBe(true);
     });
 
+    it('applies the same RCON failure grace to HTTP snapshots and live updates', async () => {
+        const raw = { ...server(), last_error: 'timeout:no reply', rcon_failures: 1 };
+        snapshot.servers = [server(epoch, { rcon_status: raw })];
+        const { fleetServers, uiNow } = await setup();
+        expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true);
+        expect(get(fleetServers)[0].rcon_status?.last_error).toBe('timeout:no reply');
+        await vi.advanceTimersByTimeAsync(30_000);
+        emit(epoch + 30_000, [status(epoch, { health: 'stale', lastError: 'timeout:no reply',
+            rconStatus: { ...raw, rcon_failures: 2 } })]);
+        expect(get(fleetServers)[0].health).toBe('stale');
+        expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
+        await vi.advanceTimersByTimeAsync(30_000);
+        emit(epoch + 60_000, [status(epoch + 60_000, {
+            rconStatus: { ...raw, last_seen_at: new Date(epoch + 60_000).toISOString(), last_error: null, rcon_failures: 0 }
+        })]);
+        expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true);
+    });
+
     it('preserves actual RCON failures and stopped-container health', async () => {
         const { fleetServers, uiNow } = await setup();
         await vi.advanceTimersByTimeAsync(1000);

tests/unit/watchdog.test.ts

@@ -114,7 +114,7 @@ describe('Watchdog detection and ownership', () => {
         await watchdog.watchdogTick();
         expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at });
         expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(1);
-        advance(60_000);
+        advance(90_001);
         await pollStatus();
         await watchdog.watchdogTick();
         expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
@@ -168,6 +168,7 @@ describe('Watchdog detection and ownership', () => {
         expect(incidents()).toEqual([]);
         expect(rcon.rconStatusResult).not.toHaveBeenCalled();
         vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
+        advance(30_001);
         await pollStatus();
         advance(30_000);
         database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
@@ -310,7 +311,7 @@ describe('Watchdog detection and ownership', () => {
         await watchdog.watchdogTick(true);
         expect(incidents()).toEqual([]);
         expect(rcon.rconStatusResult).not.toHaveBeenCalled();
-        advance(60_000);
+        advance(90_001);
         await watchdog.watchdogTick(true);
         expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 1 });
         saveTelemetry({ ...gameEvent('heartbeat', 18, 4, server), receivedAt: new Date().toISOString() });