Relax freshness limits and tolerate transient RCON failures
This commit is contained in:
+10
-1
@@ -102,9 +102,18 @@ reachable. Finished scheduled and legacy maintenance activity is retained for
|
||||
|
||||
## Health recovery and configuration drift
|
||||
|
||||
Fleet keeps successful game and running-container reports fresh for 90 seconds.
|
||||
Two consecutive failed monitoring checks mark RCON health stale; a first timeout
|
||||
or send failure remains visible in diagnostics without changing the health badge.
|
||||
Configuration and invalid-reply errors, including authentication failures, take
|
||||
effect immediately, as do failed manual probes. Successful RCON checks reset the
|
||||
failure count; skipped checks do not count. Without fresh telemetry, RCON evidence
|
||||
older than 120 seconds is offline. Explicit stopped/unhealthy container states
|
||||
and lifecycle changes still take precedence.
|
||||
|
||||
The status monitor collects container and game health every 30 seconds. Once per
|
||||
minute, the watchdog evaluates that completed evidence without issuing its own
|
||||
routine health probes. Samples expire after 45 seconds; missing or stale evidence
|
||||
routine health probes. Samples expire after 90 seconds; missing or stale evidence
|
||||
leaves health unknown and cannot justify recovery. Samples retain their original
|
||||
timestamps and instance identity, and each failure sample counts at most once.
|
||||
A monitoring gap or changed instance resets the sustained failure window. After
|
||||
|
||||
+1
-1
@@ -78,7 +78,7 @@ a change; a retained map may no longer have a hosted lobby.
|
||||
- Player-hours integrate the reported player count over observed time. Bots are included if the server counts them as players.
|
||||
- All lobbies full measures map wall time when every tracked lobby in that game/map group has a known capacity and is full. Missing capacity or status makes that interval unknown. This is a capacity-pressure indicator, not a count of rejected joins or unmet demand.
|
||||
|
||||
Observations are recorded by the existing status worker. Each remains valid until the next collection, up to the 45-second freshness limit. Longer manager outages are excluded. Charts split time at UTC hour/day boundaries; partial periods remain partial. Map variants use the configured variant when it matches the reported map.
|
||||
Observations are recorded by the existing status worker. Each remains valid until the next collection, up to the 90-second freshness limit. Longer manager outages are excluded. Charts split time at UTC hour/day boundaries; partial periods remain partial. Map variants use the configured variant when it matches the reported map.
|
||||
|
||||
Capacity comes from `sv_maxclients` / `sv_maxClients` in the status response. When unavailable, the UI displays unknown rather than assuming a four-player limit. Capacity coverage is the map time for which every tracked lobby reported both occupancy and capacity.
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ unreachable Komodo API is not evidence that a stop succeeded.
|
||||
Disruptive operations require a fresh zero-player result from RCON or accepted
|
||||
telemetry, unless Komodo positively reports a stopped container. Fresh occupied
|
||||
telemetry vetoes a conflicting empty RCON reading. Reports must be received within
|
||||
45 seconds; recovery verification also excludes reports from before the submitted
|
||||
90 seconds; recovery verification also excludes reports from before the submitted
|
||||
operation. Telemetry can verify the game's activity and selected map after Komodo
|
||||
reports healthy. Runtime actions wait for conflicting
|
||||
campaign, beta, maintenance, or provisioning work. Duplicate pending requests for
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Allow a polling cycle plus time for queued RCON requests and delivery.
|
||||
// Allow missed polling cycles and queued requests across all status evidence.
|
||||
export const STATUS_REFRESH_MS = 30_000;
|
||||
export const ONLINE_MAX_MS = STATUS_REFRESH_MS + 15_000;
|
||||
export const ONLINE_MAX_MS = 90_000;
|
||||
// The UI clock ticks once per second and can trail a newly arrived report.
|
||||
const CLOCK_TOLERANCE_MS = 1_000;
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ export interface GameObservation {
|
||||
last_gametype: string | null;
|
||||
last_seen_at: string | null;
|
||||
last_error: string | null;
|
||||
/** Consecutive failed monitoring checks; absent on older snapshots. */
|
||||
rcon_failures?: number;
|
||||
roster: StatusPlayer[] | null;
|
||||
}
|
||||
export interface ServerObservation extends GameObservation {
|
||||
@@ -32,6 +34,12 @@ export interface ServerObservation extends GameObservation {
|
||||
game_ended_at?: string | null;
|
||||
}
|
||||
|
||||
/** Keep a first transient monitoring failure in diagnostics without demoting health. */
|
||||
export function rconHealthError(status: Pick<GameObservation, 'last_error' | 'rcon_failures'>): string | null {
|
||||
const reason = status.last_error?.split(':')[0];
|
||||
return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error') ? null : status.last_error;
|
||||
}
|
||||
|
||||
export function freshTelemetry(report: GameTelemetry | undefined, now: number, after?: string | null): report is GameTelemetry {
|
||||
if (!report || !freshStatusTimestamp(report.receivedAt, now)) return false;
|
||||
if (after && (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))) return false;
|
||||
@@ -42,7 +50,7 @@ export function freshTelemetry(report: GameTelemetry | undefined, now: number, a
|
||||
export function selectServerObservation<T extends ServerObservation>(server: T, report: GameTelemetry | undefined, now: number): T {
|
||||
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
|
||||
const raw = server.rcon_status ?? server;
|
||||
const result: T = { ...server, ...raw, telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
|
||||
const result: T = { ...server, ...raw, last_error: rconHealthError(raw), telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
|
||||
const containerUnknown = server.container_state === 'unknown';
|
||||
if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) {
|
||||
return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null };
|
||||
@@ -69,11 +77,12 @@ export function selectServerObservation<T extends ServerObservation>(server: T,
|
||||
game_ended_at: players === 0 ? null : timing.gameEndedAt };
|
||||
}
|
||||
const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
|
||||
const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !raw.last_error;
|
||||
const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !result.last_error;
|
||||
// A current authenticated RCON reply can establish game liveness when
|
||||
// inspection is unavailable, without claiming the container was inspected.
|
||||
const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh;
|
||||
const age = now - Date.parse(raw.last_seen_at ?? '');
|
||||
return { ...result, health: fresh && !containerExpired ? 'online'
|
||||
: after && age >= 0 && age <= 120_000 ? 'stale' : 'offline' };
|
||||
const health = fresh && !containerExpired ? 'online'
|
||||
: after && age >= 0 && age <= 120_000 ? 'stale' : 'offline';
|
||||
return { ...result, health, last_error: health === 'online' ? result.last_error : raw.last_error };
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
|
||||
{ name: 'last_roster', ddl: 'ALTER TABLE servers ADD COLUMN last_roster TEXT' },
|
||||
{ name: 'last_seen_at', ddl: 'ALTER TABLE servers ADD COLUMN last_seen_at TEXT' },
|
||||
{ name: 'last_error', ddl: 'ALTER TABLE servers ADD COLUMN last_error TEXT' },
|
||||
{ name: 'rcon_failures', ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0' },
|
||||
{ name: 'last_reply_snippet', ddl: 'ALTER TABLE servers ADD COLUMN last_reply_snippet TEXT' },
|
||||
// Script-scope columns — static per server, populated via server-edit UI.
|
||||
{ name: 'game', ddl: 'ALTER TABLE servers ADD COLUMN game TEXT' },
|
||||
|
||||
@@ -80,6 +80,7 @@ CREATE TABLE IF NOT EXISTS servers (
|
||||
last_roster TEXT, -- JSON array of RconPlayer
|
||||
last_seen_at TEXT, -- ISO ts of last successful RCON reply
|
||||
last_error TEXT, -- last RCON failure reason (cleared on success)
|
||||
rcon_failures INTEGER NOT NULL DEFAULT 0, -- consecutive monitoring failures, capped at two
|
||||
last_reply_snippet TEXT, -- hex+ASCII of last bad-reply UDP payload
|
||||
-- Provisioning fields — set when Electric Cherry creates the deployment.
|
||||
game_mode TEXT, -- 'zombie' | 'mp' | 'sp' (currently zombie only)
|
||||
|
||||
@@ -5,6 +5,7 @@ import { db } from '../db/index.js';
|
||||
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
|
||||
import type { GameTelemetry } from '../../telemetry.js';
|
||||
import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
|
||||
import { ONLINE_MAX_MS } from '../../server-freshness.js';
|
||||
|
||||
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
|
||||
export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
|
||||
@@ -64,7 +65,7 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
|
||||
const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
|
||||
const sample = Date.parse(timestamp ?? '');
|
||||
const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at)
|
||||
&& Date.now() - sample >= 0 && Date.now() - sample <= 45_000;
|
||||
&& Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS;
|
||||
const healthy = freshSample && server?.health === 'online';
|
||||
const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
|
||||
const before = m.wasHealthy;
|
||||
|
||||
@@ -6,7 +6,7 @@ const STALE_MAX_MS = 120_000;
|
||||
/**
|
||||
* Classify server health from its last successful RCON timestamp.
|
||||
* `offline` if we've never seen it or the last successful reply is >120s old.
|
||||
* `stale` on an RCON failure or if the last reply is 45-120s old.
|
||||
* `stale` on a blocking RCON failure or if the last reply is >90-120s old.
|
||||
* `online` if fresh, including the allowance for polling and delivery.
|
||||
*/
|
||||
export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
|
||||
|
||||
@@ -87,7 +87,7 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now())
|
||||
let localCell = { index: 0, date: '' };
|
||||
interface BatchCounts { known: number; active: number; idle: number; unknown: number; players: number }
|
||||
function addHeatmap(counts: BatchCounts, start: number, end: number): void {
|
||||
// Samples are at most 45 seconds. Splitting at UTC minute boundaries
|
||||
// Samples are bounded by the shared freshness window. Splitting at UTC minute boundaries
|
||||
// handles local hour/day boundaries, DST, and fractional-hour zones.
|
||||
// Cache each minute once across the complete fleet collection batches.
|
||||
for (let cursor = start; cursor < end;) {
|
||||
|
||||
@@ -91,6 +91,7 @@ export interface ServerRow {
|
||||
last_roster: string | null;
|
||||
last_seen_at: string | null;
|
||||
last_error: string | null;
|
||||
rcon_failures: number;
|
||||
last_reply_snippet: string | null;
|
||||
game_mode: string | null;
|
||||
map_slug: string | null;
|
||||
@@ -200,6 +201,7 @@ export function setServerFields(name: string, fields: Partial<Record<keyof Serve
|
||||
'last_roster',
|
||||
'last_seen_at',
|
||||
'last_error',
|
||||
'rcon_failures',
|
||||
'last_reply_snippet',
|
||||
'game_mode',
|
||||
'map_slug',
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { selectServerObservation, type GameObservation } from '../../server-observation.js';
|
||||
import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js';
|
||||
import { classifyHealth } from './health.js';
|
||||
import { isProvisioning } from './provisioning.js';
|
||||
import { restartSettling } from '../provisioning/restarts.js';
|
||||
@@ -44,11 +44,12 @@ export function serverObservation(server: ServerRow, evidence?: Evidence) {
|
||||
let roster = null;
|
||||
try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
|
||||
const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
|
||||
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
|
||||
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error,
|
||||
rcon_failures: server.rcon_failures, roster };
|
||||
const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
|
||||
const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
|
||||
const ready = !provisioning && (!state || state === 'running' || state === 'unknown');
|
||||
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
|
||||
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, rconHealthError(rcon), state);
|
||||
return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
|
||||
status_source: null as 'telemetry' | 'rcon' | null,
|
||||
telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,
|
||||
|
||||
@@ -177,6 +177,7 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
|
||||
const unavailable = updates.container_state === 'unknown';
|
||||
if (!container.ok && !unavailable) {
|
||||
updates.last_error = null;
|
||||
updates.rcon_failures = 0;
|
||||
updates.last_reply_snippet = null;
|
||||
} else if (unavailable || !telemetryEvidence(srv.name)) {
|
||||
flush();
|
||||
@@ -192,7 +193,8 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
|
||||
});
|
||||
if (!probe) return null;
|
||||
const result = probe.result;
|
||||
if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) {
|
||||
const previous = getServer(srv.name);
|
||||
if (unchanged() && previous && (previous.last_checked_at ?? '') <= probe.observedAt) {
|
||||
gameObservation = probe;
|
||||
const now = probe.observedAt;
|
||||
updates.last_checked_at = now;
|
||||
@@ -205,13 +207,23 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
|
||||
updates.last_roster = JSON.stringify(result.status.roster);
|
||||
updates.last_seen_at = now;
|
||||
updates.last_error = null;
|
||||
} else updates.last_error = encodeError(result.reason, result.detail);
|
||||
updates.rcon_failures = 0;
|
||||
} else {
|
||||
// Only distinct accepted checks in the current lifecycle count.
|
||||
// Skipped/superseded checks return above without changing the streak.
|
||||
const consecutive = previous.last_error && (!epoch || (previous.last_checked_at ?? '') > epoch);
|
||||
const failures = consecutive ? previous.rcon_failures || 1 : 0;
|
||||
updates.rcon_failures = consecutive && previous.last_checked_at === now
|
||||
? failures : Math.min(2, failures + 1);
|
||||
updates.last_error = encodeError(result.reason, result.detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!settling && unchanged()) publishObservation(inspected, gameObservation);
|
||||
}
|
||||
} else {
|
||||
updates.last_error = null;
|
||||
updates.rcon_failures = 0;
|
||||
updates.last_reply_snippet = null;
|
||||
}
|
||||
flush();
|
||||
|
||||
@@ -104,7 +104,7 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance
|
||||
db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
|
||||
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
|
||||
db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
|
||||
last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,active_since=NULL,inactive_since=NULL,
|
||||
last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL,
|
||||
game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
|
||||
db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
|
||||
|
||||
@@ -107,7 +107,7 @@ export async function restartInRollout(server: string, options: { operationId: s
|
||||
});
|
||||
})();
|
||||
requestsInFlight.set(server, operationId);
|
||||
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
|
||||
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, rcon_failures = 0, last_reply_snippet = NULL,
|
||||
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
|
||||
try {
|
||||
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
|
||||
|
||||
@@ -38,11 +38,14 @@ export const POST: RequestHandler = async ({ params }) => {
|
||||
setServerField(name, 'last_seen_at', now);
|
||||
setServerField(name, 'last_checked_at', now);
|
||||
setServerField(name, 'last_error', null);
|
||||
setServerField(name, 'rcon_failures', 0);
|
||||
setServerField(name, 'last_reply_snippet', result.rawReply);
|
||||
} else {
|
||||
const encoded = `${result.reason}:${result.detail}`;
|
||||
setServerField(name, 'last_checked_at', now);
|
||||
setServerField(name, 'last_error', encoded);
|
||||
// An explicitly requested diagnostic failure takes effect immediately.
|
||||
setServerField(name, 'rcon_failures', 2);
|
||||
setServerField(name, 'last_reply_snippet', result.rawReply ?? null);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<script lang="ts">
|
||||
import { onMount, untrack } from 'svelte';
|
||||
import { ONLINE_MAX_MS } from '$lib/server-freshness.js';
|
||||
import { page } from '$app/state';
|
||||
import { afterNavigate, beforeNavigate, goto } from '$app/navigation';
|
||||
import { timeZone } from '$ui/stores/time-zone.js';
|
||||
@@ -169,7 +170,7 @@
|
||||
<details class="methodology">
|
||||
<summary>How these estimates work</summary>
|
||||
<p>Time with players is occupied server time divided by known server time. Known time includes occupied and idle observations; unknown status and collection gaps are excluded from that percentage.</p>
|
||||
<p>Each periodic observation covers at most 45 seconds. Longer collection gaps are excluded. History is retained for 31 days. Player-hours include all players reported by the server and are estimates, not individual session records.</p>
|
||||
<p>Each periodic observation covers at most {ONLINE_MAX_MS / 1000} seconds. Longer collection gaps are excluded. History is retained for 31 days. Player-hours include all players reported by the server and are estimates, not individual session records.</p>
|
||||
<p>Timeline buckets are aligned to UTC, with labels shown in {report.scope.timeZone.replaceAll('_', ' ')}. Daily buckets are 24-hour periods, not local calendar days. The first and latest buckets can be partial. Percentage bars divide occupied, idle, and unknown time by captured time; the headline occupancy percentage uses known time only.</p>
|
||||
<p>Historical server rows are grouped by name and may include multiple registrations using that name. Map groups preserve the recorded game and map label.</p>
|
||||
</details>
|
||||
|
||||
@@ -96,7 +96,7 @@ describe('durable beta test plans', () => {
|
||||
it('uses script events instead of logs and keeps match progress across app restarts', async () => {
|
||||
await postTelemetry(gameEvent('game_end'));
|
||||
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches')] });
|
||||
await tick(46); await tick();
|
||||
await tick(91); await tick();
|
||||
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
await appendFile(join(fixture.t5, names[0], 'logs/games_sp.log'), 'InitGame:\n 0:00 J;new;1;Josh\nExitLevel: executed\n');
|
||||
await tick();
|
||||
@@ -113,7 +113,7 @@ describe('durable beta test plans', () => {
|
||||
it('automatically credits ordinary rounds only to active deployed beta servers', async () => {
|
||||
const id = await openPlan({ ...defaultPlan(), waveSize: 1, requirements: [requirement('round', 'round')] }, names);
|
||||
await postTelemetry(gameEvent('round_change'));
|
||||
await tick(46); await tick();
|
||||
await tick(91); await tick();
|
||||
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
await postTelemetry(gameEvent('round_change', 99, 4, names[1]));
|
||||
@@ -135,7 +135,7 @@ describe('durable beta test plans', () => {
|
||||
const first = currentRun(id, names[0])!;
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
await postTelemetry(gameEvent('game_start'));
|
||||
await tick(46); await tick();
|
||||
await tick(91); await tick();
|
||||
expect(currentRun(id, names[0])?.id).not.toBe(first.id);
|
||||
expect(currentRun(id, names[0])?.phase).toBe(1);
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
@@ -162,7 +162,7 @@ describe('durable beta test plans', () => {
|
||||
await tick(15, 0, false);
|
||||
// Fresh occupied telemetry remains healthy despite an RCON timeout.
|
||||
expect(getBeta(id)?.state).toBe('ready-promote');
|
||||
await tick(31, 0, false);
|
||||
await tick(61, 0, false);
|
||||
expect(getBeta(id)?.state).toBe('running');
|
||||
await expect(promoteBeta(id)).rejects.toThrow();
|
||||
});
|
||||
@@ -236,7 +236,7 @@ describe('durable beta test plans', () => {
|
||||
expect(retry.status).toBe(200);
|
||||
await postTelemetry(gameEvent('round_change'));
|
||||
expect(testView(id)?.servers).toHaveLength(0);
|
||||
await tick(46); await tick();
|
||||
await tick(91); await tick();
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
await postTelemetry(gameEvent('round_change'));
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(25);
|
||||
|
||||
@@ -31,14 +31,14 @@ describe('Fleet discovery and match freshness', () => {
|
||||
expect(fleetIdentity({ ...server, game_mode: 'mp' }).launch).toBeNull();
|
||||
});
|
||||
it('does not classify stale, missing, or starting occupancy as an empty lobby', () => {
|
||||
for (const change of [{ last_seen_at: iso(-46_000) }, { last_players: null }, { health: 'starting' as const }, { health: 'offline' as const }]) {
|
||||
for (const change of [{ last_seen_at: iso(-91_000) }, { last_players: null }, { health: 'starting' as const }, { health: 'offline' as const }]) {
|
||||
expect(matchesFleet({ ...server, last_players: 0, ...change }, '', 'empty', '', [], [], now)).toBe(false);
|
||||
}
|
||||
expect(matchesFleet({ ...server, last_players: 0 }, '', 'empty', '', [], [], now)).toBe(true);
|
||||
expect(matchesFleet(server, '', 'occupied', 't5', [], [], now)).toBe(false);
|
||||
});
|
||||
it('includes stale health and failed operations in attention without flagging normal startup', () => {
|
||||
expect(matchesFleet({ ...server, last_seen_at: iso(-46_000) }, '', 'attention', '', [], [], now)).toBe(true);
|
||||
expect(matchesFleet({ ...server, last_seen_at: iso(-91_000) }, '', 'attention', '', [], [], now)).toBe(true);
|
||||
expect(matchesFleet(server, '', 'attention', '', [], [{ ...operation, state: 'attention' }], now)).toBe(true);
|
||||
expect(matchesFleet({ ...server, health: 'starting' }, '', 'attention', '', [], [], now)).toBe(false);
|
||||
});
|
||||
|
||||
@@ -61,7 +61,7 @@ describe('shared monitoring observations', () => {
|
||||
expect(api.monitoringStatus(server)?.game?.observedAt).toBe(new Date(now).toISOString());
|
||||
expect(api.inspect).toHaveBeenCalledTimes(1);
|
||||
expect(api.rcon).toHaveBeenCalledTimes(1);
|
||||
vi.setSystemTime(now + 45_001);
|
||||
vi.setSystemTime(now + 90_001);
|
||||
expect(api.monitoringStatus(server)).toBeNull();
|
||||
vi.setSystemTime(now - 1);
|
||||
expect(api.monitoringStatus(server)).toBeNull();
|
||||
@@ -108,7 +108,7 @@ describe('shared monitoring observations', () => {
|
||||
it('excludes old game evidence even when the container sample is fresh', async () => {
|
||||
const api = await setup();
|
||||
const game = await api.rconForMonitoring(server);
|
||||
vi.setSystemTime(now + 46_000);
|
||||
vi.setSystemTime(now + 91_000);
|
||||
const publish = api.monitoringStatusWriter(server);
|
||||
publish(await api.inspectForMonitoring(server), game);
|
||||
expect(api.monitoringStatus(server)?.container.result).toEqual(healthy);
|
||||
|
||||
@@ -123,6 +123,7 @@ describe('host status observations', () => {
|
||||
const first = api.fanoutStatus();
|
||||
await vi.waitFor(() => expect(api.operations()).toHaveLength(1));
|
||||
api.complete(); await first;
|
||||
api.db.prepare("UPDATE servers SET rcon_failures=1,last_error='timeout:first failure'").run();
|
||||
const previous = api.getServer('server-0-0')!;
|
||||
const previousGame = api.monitoringStatus('server-0-0')?.game;
|
||||
vi.setSystemTime(at + 31_000);
|
||||
@@ -138,9 +139,11 @@ describe('host status observations', () => {
|
||||
} finally { api.complete(); await pass; }
|
||||
expect(api.getServer('server-0-0')).toMatchObject({ last_checked_at: previous.last_checked_at,
|
||||
last_seen_at: previous.last_seen_at, last_error: previous.last_error, last_players: previous.last_players,
|
||||
last_reply_snippet: previous.last_reply_snippet });
|
||||
last_reply_snippet: previous.last_reply_snippet, rcon_failures: 1 });
|
||||
expect(api.monitoringStatus('server-0-0')?.game).toEqual(previousGame);
|
||||
expect(api.getServer('server-0-1')?.last_error).toContain('timeout:');
|
||||
expect(api.getServer('server-0-1')?.rcon_failures).toBe(2);
|
||||
expect(api.getServer('server-0-2')?.rcon_failures).toBe(0);
|
||||
expect(Date.parse(api.getServer('server-0-2')!.last_seen_at!)).toBeGreaterThanOrEqual(at + 31_000);
|
||||
});
|
||||
|
||||
|
||||
+12
-12
@@ -17,7 +17,7 @@ describe('historical metrics', () => {
|
||||
const result = readMetrics(1, at + 3600_000);
|
||||
expect(result.summary.active * 3600).toBeCloseTo(30);
|
||||
expect(result.summary.idle * 3600).toBeCloseTo(30);
|
||||
expect(result.summary.unknown * 3600).toBeCloseTo(45);
|
||||
expect(result.summary.unknown * 3600).toBeCloseTo(90);
|
||||
expect(result.summary.playerHours * 3600).toBeCloseTo(60);
|
||||
expect(result.points.reduce((sum, p) => sum + p.observed, 0)).toBeCloseTo(result.summary.observed);
|
||||
});
|
||||
@@ -43,7 +43,7 @@ describe('historical metrics', () => {
|
||||
let result = readMetrics(1, midnight + 15_000);
|
||||
expect(result.points.filter(p => p.active > 0).map(p => p.active * 3600)).toEqual([15, 15]);
|
||||
result = readMetrics(1, midnight + 86400_000);
|
||||
expect(result.summary.active * 3600).toBeCloseTo(30);
|
||||
expect(result.summary.active * 3600).toBeCloseTo(75);
|
||||
expect(result.points.reduce((sum, p) => sum + p.active, 0)).toBeCloseTo(result.summary.active);
|
||||
});
|
||||
|
||||
@@ -84,7 +84,7 @@ describe('historical metrics', () => {
|
||||
expect(initial.summary).toEqual({ observed: 0, active: 0, idle: 0, unknown: 0, playerHours: 0 });
|
||||
const later = readMetrics(1, at + 120_000);
|
||||
expect(later.summary.observed).toBe(0);
|
||||
expect(later.summary.unknown * 3600).toBeCloseTo(45);
|
||||
expect(later.summary.unknown * 3600).toBeCloseTo(90);
|
||||
expect(later.maps[0].capacityObserved).toBe(0);
|
||||
});
|
||||
|
||||
@@ -112,10 +112,10 @@ describe('historical metrics', () => {
|
||||
// Removing a lobby ends its previous observation at the next complete collection.
|
||||
recordMetrics([{ ...one, map: 'Farm', players: 0 }], at + 30_000);
|
||||
// A collection outage is excluded; an identically named map in another game remains separate.
|
||||
recordMetrics([{ ...one, game: 't5', players: 2, capacity: null }], at + 120_000);
|
||||
const result = readMetrics(1, at + 150_000);
|
||||
recordMetrics([{ ...one, game: 't5', players: 2, capacity: null }], at + 150_000);
|
||||
const result = readMetrics(1, at + 180_000);
|
||||
expect(result.summary.active * 3600).toBeCloseTo(90);
|
||||
expect(result.summary.idle * 3600).toBeCloseTo(45);
|
||||
expect(result.summary.idle * 3600).toBeCloseTo(90);
|
||||
expect(result.summary.playerHours * 3600).toBeCloseTo(300);
|
||||
expect(result.servers.find(server => server.server === 'two')!.observed * 3600).toBeCloseTo(30);
|
||||
const town = result.maps.find(map => map.game === 't6' && map.map === 'Town')!;
|
||||
@@ -244,7 +244,7 @@ describe('historical metrics', () => {
|
||||
expect(result.previous.until).toBe(result.since);
|
||||
expect(result.previous.summary.active * 3600).toBeCloseTo(15);
|
||||
expect(result.summary.active * 3600).toBeCloseTo(15);
|
||||
expect(result.summary.idle * 3600).toBeCloseTo(45);
|
||||
expect(result.summary.idle * 3600).toBeCloseTo(90);
|
||||
expect(result.previous).toMatchObject({ firstSampleAt: since - 15_000, lastSampleAt: since - 15_000, partialHistory: true });
|
||||
});
|
||||
|
||||
@@ -258,10 +258,10 @@ describe('historical metrics', () => {
|
||||
recordMetrics([sample], now - 30_000);
|
||||
const result = readMetrics(30, now);
|
||||
expect(result.previous).toMatchObject({ since: now - 60 * day, until: now - 30 * day, partialHistory: true });
|
||||
expect(result.previous.summary.active * 3600).toBeCloseTo(45);
|
||||
expect(result.previous.summary.active * 3600).toBeCloseTo(90);
|
||||
expect(result.previous.firstSampleAt).toBe(now - 31 * day + 30_000);
|
||||
expect(result.summary.active * 3600).toBeCloseTo(30);
|
||||
expect(result.summary.idle * 3600).toBeCloseTo(45);
|
||||
expect(result.summary.idle * 3600).toBeCloseTo(90);
|
||||
});
|
||||
|
||||
it('marks the retained start bound separately from gaps inside the previous window', async () => {
|
||||
@@ -273,7 +273,7 @@ describe('historical metrics', () => {
|
||||
recordMetrics([sample], now - 30_000);
|
||||
const result = readMetrics(1, now);
|
||||
expect(result.previous.partialHistory).toBe(false);
|
||||
expect(result.previous.summary.unknown * 3600).toBeCloseTo(30);
|
||||
expect(result.previous.summary.unknown * 3600).toBeCloseTo(75);
|
||||
expect(result.previous.summary.observed).toBe(0);
|
||||
expect(result.previous.firstSampleAt).toBe(priorStart - 15_000);
|
||||
// Having the left edge says nothing about the uncaptured time after it.
|
||||
@@ -326,8 +326,8 @@ describe('historical metrics', () => {
|
||||
recordMetrics([{ ...sample, players: 0 }], at + 3600_000);
|
||||
const result = readMetrics(parseMetricsQuery(new URLSearchParams({ days: '1', timeZone: 'America/New_York' })), at + 3630_000);
|
||||
const cell = result.heatmap.cells.find(cell => cell.weekday === 6 && cell.hour === 1)!;
|
||||
expect(cell.observed * 3600).toBeCloseTo(75);
|
||||
expect(cell.active * 3600).toBeCloseTo(45);
|
||||
expect(cell.observed * 3600).toBeCloseTo(120);
|
||||
expect(cell.active * 3600).toBeCloseTo(90);
|
||||
expect(cell.idle * 3600).toBeCloseTo(30);
|
||||
expect(cell.distinctDates).toBe(1);
|
||||
expect(result.heatmap.cells.filter(cell => cell.observed > 0)).toHaveLength(1);
|
||||
|
||||
@@ -368,7 +368,7 @@ describe('Watchdog provisioning and telemetry evidence', () => {
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
for (const offset of [-45_001, 1]) {
|
||||
for (const offset of [-90_001, 1]) {
|
||||
await report(0, new Date(Date.now() + offset).toISOString());
|
||||
expect(telemetryEvidence(server)).toBeNull();
|
||||
expect(await probePlayerCount(server)).toBeNull();
|
||||
|
||||
@@ -9,10 +9,10 @@ it('keeps healthy reports online across polling and delivery delays, then expire
|
||||
vi.useFakeTimers();
|
||||
const reportAt = Date.parse('2026-09-18T12:00:00Z');
|
||||
const timestamp = new Date(reportAt).toISOString();
|
||||
for (const elapsed of [-999, 0, STATUS_REFRESH_MS, STATUS_REFRESH_MS + 4000, 45_000, 45_001]) {
|
||||
for (const elapsed of [-999, 0, STATUS_REFRESH_MS, STATUS_REFRESH_MS + 4000, 45_000, 60_000, 90_000, 90_001, 120_000]) {
|
||||
const now = reportAt + elapsed;
|
||||
vi.setSystemTime(now);
|
||||
const expected = elapsed <= 45_000;
|
||||
const expected = elapsed <= 90_000;
|
||||
const health = classifyHealth(timestamp, null, 'running');
|
||||
expect(health).toBe(expected ? 'online' : 'stale');
|
||||
expect(freshServer({ health, last_seen_at: timestamp, last_error: null } as FleetServer, now)).toBe(expected);
|
||||
|
||||
@@ -12,6 +12,32 @@ function server(changes: Partial<ServerObservation> = {}): ServerObservation {
|
||||
}
|
||||
|
||||
describe('shared server observation', () => {
|
||||
it('tolerates only the first transient failure while retaining diagnostics and expiring evidence', () => {
|
||||
for (const reason of ['timeout', 'send-error']) {
|
||||
const raw = { ...server(), last_error: `${reason}:no reply`, rcon_failures: 1 };
|
||||
const selected = selectServerObservation(server({ rcon_status: raw }), undefined, now);
|
||||
expect(selected).toMatchObject({ health: 'online', last_error: null, rcon_status: raw });
|
||||
expect(selectServerObservation(selected, undefined, now + 90_000).health).toBe('online');
|
||||
expect(selectServerObservation(selected, undefined, now + 90_001).health).toBe('stale');
|
||||
expect(selectServerObservation(selected, undefined, now + 120_001).health).toBe('offline');
|
||||
expect(selectServerObservation(server({ rcon_status: { ...raw, rcon_failures: 2 } }), undefined, now))
|
||||
.toMatchObject({ health: 'stale', last_error: raw.last_error });
|
||||
}
|
||||
for (const reason of ['no-config', 'no-port', 'bad-reply', 'no-players']) {
|
||||
expect(selectServerObservation(server({ last_error: `${reason}:failed`, rcon_failures: 1 }), undefined, now).health).toBe('stale');
|
||||
}
|
||||
expect(selectServerObservation(server({ last_error: 'timeout:legacy failure' }), undefined, now).health).toBe('stale');
|
||||
});
|
||||
|
||||
it('accepts telemetry and container evidence through 90 seconds, then expires each independently', () => {
|
||||
expect(selectServerObservation(server(), report(), now + 90_000)).toMatchObject({ health: 'online', status_source: 'telemetry' });
|
||||
expect(selectServerObservation(server(), report(90_001), now + 90_001).health).toBe('stale');
|
||||
const current = server({ container_checked_at: at(90_001) });
|
||||
expect(selectServerObservation(current, report(), now + 90_001).health).toBe('stale');
|
||||
expect(freshTelemetry(report(-90_000), now)).toBe(true);
|
||||
expect(freshTelemetry(report(-90_001), now)).toBe(false);
|
||||
});
|
||||
|
||||
it('selects the telemetry count and roster together and keeps raw RCON evidence', () => {
|
||||
const raw = server({ last_error: 'timeout' });
|
||||
const selected = selectServerObservation(server({ rcon_status: raw }), report(), now);
|
||||
@@ -25,8 +51,8 @@ describe('shared server observation', () => {
|
||||
const original = server();
|
||||
const selected = selectServerObservation(server({ rcon_status: original, telemetry: report(0, 2) }), report(-1000), now);
|
||||
expect(selected.last_players).toBe(2);
|
||||
const fallback = selectServerObservation({ ...selected, container_checked_at: at(46_000),
|
||||
rcon_status: { ...original, last_seen_at: at(40_000) } }, undefined, now + 46_000);
|
||||
const fallback = selectServerObservation({ ...selected, container_checked_at: at(91_000),
|
||||
rcon_status: { ...original, last_seen_at: at(40_000) } }, undefined, now + 91_000);
|
||||
expect(fallback).toMatchObject({ status_source: 'rcon', health: 'online', last_players: 4, last_seen_at: at(40_000) });
|
||||
});
|
||||
|
||||
@@ -40,19 +66,19 @@ describe('shared server observation', () => {
|
||||
it('uses fresh RCON with unknown container state, but does not substitute telemetry or extend freshness', () => {
|
||||
const raw = server({ container_state: 'unknown', container_checked_at: at(-60_000) });
|
||||
expect(selectServerObservation(raw, report(), now)).toMatchObject({ health: 'online', last_players: 4, status_source: 'rcon' });
|
||||
expect(selectServerObservation(raw, report(46_000), now + 46_000).health).toBe('stale');
|
||||
expect(selectServerObservation(raw, report(91_000), now + 91_000).health).toBe('stale');
|
||||
expect(selectServerObservation(raw, report(121_000), now + 121_000).health).toBe('offline');
|
||||
expect(selectServerObservation({ ...raw, telemetry_after: at() }, report(), now).health).toBe('offline');
|
||||
expect(selectServerObservation({ ...raw, last_seen_at: null }, report(), now).health).toBe('offline');
|
||||
});
|
||||
|
||||
it('requires fresh container evidence and reports after the latest lifecycle submission', () => {
|
||||
expect(selectServerObservation(server({ container_checked_at: at(-46_000) }), report(), now).health).not.toBe('online');
|
||||
expect(selectServerObservation(server({ container_checked_at: at(-91_000) }), report(), now).health).not.toBe('online');
|
||||
expect(selectServerObservation(server({ health: 'provisioning' }), report(), now).health).toBe('provisioning');
|
||||
expect(selectServerObservation(server({ telemetry_after: at() }), report(-1), now).health).toBe('offline');
|
||||
expect(freshTelemetry(report(), now, 'invalid')).toBe(false);
|
||||
expect(freshTelemetry(report(2000), now)).toBe(false);
|
||||
expect(freshTelemetry(report(-46_000), now)).toBe(false);
|
||||
expect(freshTelemetry(report(-91_000), now)).toBe(false);
|
||||
expect(freshTelemetry({ ...report(), players: [{}] } as never, now)).toBe(false);
|
||||
});
|
||||
|
||||
|
||||
@@ -30,6 +30,77 @@ async function setup() {
|
||||
}
|
||||
|
||||
describe('database fleet and container readiness', () => {
|
||||
it('requires two distinct failed monitoring checks, preserves diagnostics, and resets on success', async () => {
|
||||
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
||||
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
||||
const snapshot = async () => (await (await GET()).json()).servers[0];
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
const at = Date.parse('2026-10-05T12:00:00Z');
|
||||
vi.setSystemTime(at);
|
||||
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
||||
const success = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
||||
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } };
|
||||
rcon.mockResolvedValue(success);
|
||||
await fanoutStatus();
|
||||
const events: unknown[] = [];
|
||||
const unsubscribe = subscribe(event => events.push(event));
|
||||
try {
|
||||
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
|
||||
vi.setSystemTime(at + 30_000);
|
||||
await fanoutStatus();
|
||||
expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:no reply' });
|
||||
expect(await snapshot()).toMatchObject({ health: 'online', last_error: null,
|
||||
rcon_status: { last_error: 'timeout:no reply', rcon_failures: 1 } });
|
||||
expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
|
||||
health: 'online', lastError: null, rconStatus: expect.objectContaining({ rcon_failures: 1, last_error: 'timeout:no reply' })
|
||||
})] }));
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } });
|
||||
// Reprocessing a sample with the same timestamp cannot count it twice.
|
||||
await fanoutStatus();
|
||||
expect(getServer('managed')?.rcon_failures).toBe(1);
|
||||
vi.setSystemTime(at + 60_000);
|
||||
await fanoutStatus();
|
||||
expect(await snapshot()).toMatchObject({ health: 'stale', last_error: 'timeout:no reply', rcon_status: { rcon_failures: 2 } });
|
||||
rcon.mockResolvedValue(success);
|
||||
vi.setSystemTime(at + 90_000);
|
||||
await fanoutStatus();
|
||||
expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: null });
|
||||
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'send-error', detail: 'unreachable' });
|
||||
vi.setSystemTime(at + 120_000);
|
||||
await fanoutStatus();
|
||||
expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } });
|
||||
vi.setSystemTime(at + 180_001);
|
||||
expect(await snapshot()).toMatchObject({ health: 'stale' });
|
||||
vi.setSystemTime(at + 210_001);
|
||||
expect(await snapshot()).toMatchObject({ health: 'offline' });
|
||||
} finally { unsubscribe(); }
|
||||
});
|
||||
|
||||
it('adds the failure counter to an existing database without suppressing legacy errors', async () => {
|
||||
const { db, getServer } = await setup();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
db.exec('ALTER TABLE servers DROP COLUMN rcon_failures');
|
||||
closeDb();
|
||||
expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: 'timeout:old process' });
|
||||
});
|
||||
|
||||
it('does not carry a failure streak across a new lifecycle submission', async () => {
|
||||
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
const at = Date.parse('2026-10-05T12:00:00Z');
|
||||
vi.setSystemTime(at);
|
||||
db.prepare("UPDATE servers SET rcon_failures=1,last_checked_at=?").run(new Date(at - 30_000).toISOString());
|
||||
db.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
|
||||
VALUES('new-start','managed','restart','done','done',?,?,'test')`).run(new Date(at - 10_000).toISOString(), new Date(at - 10_000).toISOString());
|
||||
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
||||
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'new process' });
|
||||
await fanoutStatus();
|
||||
expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:new process' });
|
||||
});
|
||||
|
||||
it('shares routine observations without renewing their timestamps and keeps manual probes fresh', async () => {
|
||||
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
const { inspectForMonitoring, rconForMonitoring } = await import('../../src/lib/server/fleet/health-observations.js');
|
||||
@@ -193,7 +264,7 @@ describe('database fleet and container readiness', () => {
|
||||
const pending = fanoutStatus();
|
||||
try {
|
||||
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
|
||||
vi.setSystemTime(now + 46_000);
|
||||
vi.setSystemTime(now + 91_000);
|
||||
} finally {
|
||||
release({ ok: true, state: 'running' });
|
||||
await pending;
|
||||
@@ -444,7 +515,7 @@ describe('database fleet and container readiness', () => {
|
||||
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
||||
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
||||
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
||||
db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 46_000).toISOString());
|
||||
db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 91_000).toISOString());
|
||||
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
||||
await fanoutStatus();
|
||||
@@ -515,6 +586,10 @@ describe('database fleet and container readiness', () => {
|
||||
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
|
||||
vi.setSystemTime(Date.now() + 30_000);
|
||||
await fanoutStatus();
|
||||
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null,
|
||||
rcon_status: { rcon_failures: 1, last_error: 'timeout:no reply' } });
|
||||
vi.setSystemTime(Date.now() + 30_000);
|
||||
await fanoutStatus();
|
||||
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' });
|
||||
});
|
||||
|
||||
|
||||
@@ -287,7 +287,7 @@ describe('Fleet report clock', () => {
|
||||
expect(get(fleetServers)[0]).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry' });
|
||||
expect(get(fleetServers)[0].roster).toHaveLength(3);
|
||||
expect(matchesFleet(get(fleetServers)[0], 'Player 3', 'occupied', '', [], [], get(uiNow))).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(46_000);
|
||||
await vi.advanceTimersByTimeAsync(91_000);
|
||||
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
|
||||
expect(matchesFleet(get(fleetServers)[0], '', 'occupied', '', [], [], get(uiNow))).toBe(false);
|
||||
});
|
||||
@@ -368,15 +368,33 @@ describe('Fleet report clock', () => {
|
||||
it('keeps advancing through wall-clock changes and expires a report if updates stop', async () => {
|
||||
const { fleetServers, uiNow } = await setup();
|
||||
vi.setSystemTime(epoch - 120_000);
|
||||
await vi.advanceTimersByTimeAsync(45_000);
|
||||
expect(get(uiNow)).toBe(epoch + 45_000);
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(get(uiNow)).toBe(epoch + 90_000);
|
||||
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(get(uiNow)).toBe(epoch + 46_000);
|
||||
expect(get(uiNow)).toBe(epoch + 91_000);
|
||||
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
|
||||
expect(matchesFleet(get(fleetServers)[0], '', 'attention', '', [], [], get(uiNow))).toBe(true);
|
||||
});
|
||||
|
||||
it('applies the same RCON failure grace to HTTP snapshots and live updates', async () => {
|
||||
const raw = { ...server(), last_error: 'timeout:no reply', rcon_failures: 1 };
|
||||
snapshot.servers = [server(epoch, { rcon_status: raw })];
|
||||
const { fleetServers, uiNow } = await setup();
|
||||
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true);
|
||||
expect(get(fleetServers)[0].rcon_status?.last_error).toBe('timeout:no reply');
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
emit(epoch + 30_000, [status(epoch, { health: 'stale', lastError: 'timeout:no reply',
|
||||
rconStatus: { ...raw, rcon_failures: 2 } })]);
|
||||
expect(get(fleetServers)[0].health).toBe('stale');
|
||||
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
emit(epoch + 60_000, [status(epoch + 60_000, {
|
||||
rconStatus: { ...raw, last_seen_at: new Date(epoch + 60_000).toISOString(), last_error: null, rcon_failures: 0 }
|
||||
})]);
|
||||
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true);
|
||||
});
|
||||
|
||||
it('preserves actual RCON failures and stopped-container health', async () => {
|
||||
const { fleetServers, uiNow } = await setup();
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
|
||||
@@ -114,7 +114,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
await watchdog.watchdogTick();
|
||||
expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at });
|
||||
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(1);
|
||||
advance(60_000);
|
||||
advance(90_001);
|
||||
await pollStatus();
|
||||
await watchdog.watchdogTick();
|
||||
expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
|
||||
@@ -168,6 +168,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
||||
advance(30_001);
|
||||
await pollStatus();
|
||||
advance(30_000);
|
||||
database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
|
||||
@@ -310,7 +311,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
||||
advance(60_000);
|
||||
advance(90_001);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 1 });
|
||||
saveTelemetry({ ...gameEvent('heartbeat', 18, 4, server), receivedAt: new Date().toISOString() });
|
||||
|
||||
Reference in New Issue
Block a user