From 1b42244858fa7bb6add31cd4c02bc9f4cfbf4c87 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Tue, 6 Oct 2026 21:13:02 -0400 Subject: [PATCH] Supplement telemetry rosters with fresh RCON pings --- docs/beta-test-plans.md | 9 +++-- src/lib/server-observation.ts | 18 +++++++-- src/lib/server/pipeline/worker.ts | 5 ++- tests/unit/host-status-batch.test.ts | 14 +++++++ tests/unit/server-observation.test.ts | 55 +++++++++++++++++++++++++++ tests/unit/server-status.test.ts | 38 +++++++++++++++--- 6 files changed, 125 insertions(+), 14 deletions(-) diff --git a/docs/beta-test-plans.md b/docs/beta-test-plans.md index dabdcee..3006b5d 100644 --- a/docs/beta-test-plans.md +++ b/docs/beta-test-plans.md @@ -37,9 +37,12 @@ Every event also accepts an optional per-player `ping`: the game server's curren latency measurement in milliseconds, as a nonnegative JSON integer (for example, `"ping": 42`). Send `null` or omit the field when unavailable; do not substitute zero for a missing measurement. Older reporters without this field remain valid. -Fleet displays ping from the same telemetry snapshot as the player roster and -scores, with `—` for unknown values. Ping is retained and streamed with the latest -report; it does not contribute to leaderboard statistics. +Fleet prefers telemetry ping and fills missing readings from fresh RCON status, +matching players by GUID. Occupied lobbies continue RCON polling every 30 seconds. +RCON supplements only ping; telemetry still supplies roster membership and scores. +RCON pings expire after 90 seconds and must match the current map and server +lifecycle. Missing or ambiguous matches display `—`. Telemetry ping is retained +and streamed with the latest report; it does not contribute to leaderboard statistics. ### Game start diff --git a/src/lib/server-observation.ts b/src/lib/server-observation.ts index b680f1e..8a72df8 100644 --- a/src/lib/server-observation.ts +++ b/src/lib/server-observation.ts @@ -50,15 +50,27 @@ export function freshTelemetry(report: GameTelemetry | undefined, now: number, a export function selectServerObservation(server: T, report: GameTelemetry | undefined, now: number): T { if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry; const raw = server.rcon_status ?? server; - const result: T = { ...server, ...raw, last_error: rconHealthError(raw), telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null }; + const result: T = { ...server, ...raw, rcon_status: raw, last_error: rconHealthError(raw), telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null }; const containerUnknown = server.container_state === 'unknown'; if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) { return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null }; } const checked = server.container_checked_at ?? server.last_checked_at ?? raw.last_seen_at; const containerFresh = server.container_state === 'running' && freshStatusTimestamp(checked ?? null, now); + const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after); + const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !result.last_error; if (containerFresh && report?.server === server.name && freshTelemetry(report, now, server.telemetry_after)) { const players = report.players.length; + // Telemetry owns roster membership. Supplement missing pings only with + // unambiguous RCON identities from the current map and server lifecycle. + const pings = new Map(); + if (fresh && raw.last_map === report.map) { + for (const player of raw.roster ?? []) { + if (!player.guid) continue; + const ping = player.ping; + pings.set(player.guid, pings.has(player.guid) || ping === null || !Number.isSafeInteger(ping) || ping < 0 ? null : ping); + } + } // New reports carry persisted anchors through both SSE and HTTP. Keep // the fallback for telemetry saved before server timing was included. const timing = report.timing?.server ?? { @@ -70,14 +82,12 @@ export function selectServerObservation(server: T, return { ...result, health: 'online', status_source: 'telemetry', last_players: players, last_map: report.map, last_seen_at: report.receivedAt, last_error: null, roster: report.players.map(player => ({ num: null, score: 'score' in player ? player.score : null, - ping: player.ping ?? null, guid: player.guid, name: player.name })), + ping: player.ping ?? pings.get(player.guid) ?? null, guid: player.guid, name: player.name })), active_since: players > 0 ? timing.activeSince : null, inactive_since: players === 0 ? timing.inactiveSince : null, game_started_at: players === 0 ? null : timing.gameStartedAt, game_ended_at: players === 0 ? null : timing.gameEndedAt }; } - const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after); - const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !result.last_error; // A current authenticated RCON reply can establish game liveness when // inspection is unavailable, without claiming the container was inspected. const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh; diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index 4f01bcb..dcc127a 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -138,7 +138,7 @@ async function statusPass(): Promise { const servers = snapshot.servers(); if (!servers.length) return; const probes = prepareMonitoringRcon(servers.filter(server => !pendingStatus.has(server.name) - && !snapshot.provisioning(server.name) && !snapshot.settling(server.name) && !snapshot.telemetryEvidence(server.name)).map(server => server.name), snapshot); + && !snapshot.provisioning(server.name) && !snapshot.settling(server.name) && snapshot.telemetryEvidence(server.name)?.players.length !== 0).map(server => server.name), snapshot); const results = await Promise.all(servers.map(server => queueStatus(server, snapshot, probes.get(server.name)))); const capacities = new Map(servers.map((server, i) => [server.name, results[i]])); try { @@ -197,7 +197,8 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, updates.last_error = null; updates.rcon_failures = 0; updates.last_reply_snippet = null; - } else if (unavailable || !snapshot.telemetryEvidence(srv.name)) { + } else if (unavailable || snapshot.telemetryEvidence(srv.name)?.players.length !== 0) { + // Occupied telemetry rosters still need RCON for player pings. flush(); const probeStartedAt = new Date().toISOString(); const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name, false, snapshot)).catch(err => { diff --git a/tests/unit/host-status-batch.test.ts b/tests/unit/host-status-batch.test.ts index 0093d65..a50e3f5 100644 --- a/tests/unit/host-status-batch.test.ts +++ b/tests/unit/host-status-batch.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { gameEvent, postTelemetry } from '../helpers/telemetry.js'; let fixture: Fixture; let at: number; @@ -45,6 +46,19 @@ async function setup(count = 5, hostCount = 1) { } describe('host status observations', () => { + it('batches occupied telemetry lobbies for pings while skipping empty lobbies', async () => { + const api = await setup(2); + await postTelemetry(gameEvent('heartbeat', 15, 1, 'server-0-0')); + await postTelemetry(gameEvent('heartbeat', 15, 0, 'server-0-1')); + const pass = api.fanoutStatus(); + try { + await vi.waitFor(() => expect(api.operations()).toHaveLength(1)); + expect(api.operations()[0]).toMatchObject({ kind: 'rcon.status-batch', + payload: { instances: [{ instanceId: 'instance-0-0', generation: 1 }] } }); + } finally { api.complete(); await pass; } + expect(api.getServer('server-0-1')?.last_seen_at).toBeNull(); + }); + it('bounds monitoring reads below five per server across thirty instances', async () => { const api = await setup(10, 3); const hosts = await import('../../src/lib/server/hosts/index.js'); diff --git a/tests/unit/server-observation.test.ts b/tests/unit/server-observation.test.ts index 3f05c7f..c242ada 100644 --- a/tests/unit/server-observation.test.ts +++ b/tests/unit/server-observation.test.ts @@ -47,6 +47,61 @@ describe('shared server observation', () => { expect(selected.rcon_status).toBe(raw); }); + it('supplements telemetry members with GUID-matched pings without changing scores or names', () => { + const telemetry = report(); + const raw = server({ roster: [ + { ...telemetry.players[1], num: 2, score: 10, ping: 0, name: 'Old name' }, + { ...telemetry.players[0], num: 3, score: 20, ping: 999 }, + { ...telemetry.players[2], num: 4, score: 30, ping: 45, guid: 'different-player' } + ] }); + const selected = selectServerObservation(server({ rcon_status: raw }), telemetry, now); + expect(selected.roster).toEqual(telemetry.players.map((player, i) => ({ + name: player.name, guid: player.guid, num: null, score: 5000, ping: [999, 0, null][i] + }))); + expect(selected.last_players).toBe(3); + expect(selected.rcon_status).toBe(raw); + // A new telemetry report must not renew the RCON sample's age. + const expired = selectServerObservation({ ...selected, container_checked_at: at(90_001) }, report(90_001), now + 90_001); + expect(expired.health).toBe('online'); + expect(expired.roster?.map(player => player.ping)).toEqual([null, null, null]); + }); + + it('prefers telemetry ping, including zero, and fills absent or null readings from RCON', () => { + const telemetry = report(); + telemetry.players[0].ping = 0; + telemetry.players[1].ping = null; + const raw = server({ roster: telemetry.players.map(player => ({ ...player, num: 0, score: 10, ping: 42 })) }); + expect(selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.map(player => player.ping)).toEqual([0, 42, 42]); + }); + + it.each([ + { last_seen_at: at(-90_001) }, { last_seen_at: at(2000) }, { last_seen_at: null }, + { last_error: 'timeout', rcon_failures: 2 }, { last_map: 'other-map' } + ])('does not supplement pings from unusable RCON evidence: %j', changes => { + const telemetry = report(); + const raw = server({ ...changes, roster: [{ ...telemetry.players[0], num: 0, score: 100, ping: 42 }] }); + expect(selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.[0].ping).toBeNull(); + }); + + it('requires RCON samples after the latest lifecycle and tolerates one transient failure', () => { + const telemetry = report(); + const raw = server({ last_seen_at: at(-90_000), roster: [{ ...telemetry.players[0], num: 0, score: 100, ping: 42 }] }); + expect(selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.[0].ping).toBe(42); + expect(selectServerObservation(server({ rcon_status: raw, telemetry_after: at(-90_000) }), telemetry, now).roster?.[0].ping).toBeNull(); + expect(selectServerObservation(server({ rcon_status: { ...raw, last_error: 'timeout', rcon_failures: 1 } }), telemetry, now).roster?.[0].ping).toBe(42); + }); + + it.each(['duplicate', 'missing-guid', 'null', 'negative', 'invalid'])('rejects %s ping matches', mode => { + const telemetry = report(); + const player = { ...telemetry.players[0], num: 0, score: 100, ping: 42 as number | null, guid: telemetry.players[0].guid as string | null }; + if (mode === 'missing-guid') player.guid = null; + if (mode === 'null') player.ping = null; + if (mode === 'negative') player.ping = -1; + if (mode === 'invalid') player.ping = NaN; + const raw = server({ roster: mode === 'duplicate' ? [player, { ...player, ping: 50 }] : [player] }); + expect(selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.[0].ping).toBeNull(); + }); + it('uses the latest HTTP or streamed telemetry and falls back without extending freshness', () => { const original = server(); const selected = selectServerObservation(server({ rcon_status: original, telemetry: report(0, 2) }), report(-1000), now); diff --git a/tests/unit/server-status.test.ts b/tests/unit/server-status.test.ts index 39ea2ea..9c4698d 100644 --- a/tests/unit/server-status.test.ts +++ b/tests/unit/server-status.test.ts @@ -372,6 +372,8 @@ describe('database fleet and container readiness', () => { vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(startedAt); inspect.mockResolvedValue({ ok: true, state: 'running' }); + rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', + status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } }); expect((await postTelemetry(gameEvent('game_start', 0, 1, 'managed'))).status).toBe(200); expect(getServer('managed')).toMatchObject({ active_since: at(0), inactive_since: null, game_started_at: at(0) }); @@ -384,7 +386,7 @@ describe('database fleet and container readiness', () => { const events: unknown[] = []; const unsubscribe = subscribe(event => events.push(event)); try { await fanoutStatus(); } finally { unsubscribe(); } - expect(rcon).not.toHaveBeenCalled(); + expect(rcon).toHaveBeenCalledOnce(); expect((await (await GET()).json()).servers[0]).toMatchObject({ active_since: at(0), game_started_at: at(0), last_seen_at: at(19) }); @@ -488,7 +490,7 @@ describe('database fleet and container readiness', () => { expect(snapshot.serverNow).toBeLessThanOrEqual(Date.now()); }); - it('uses telemetry consistently in REST, status events, and metrics without polling RCON', async () => { + it('supplements telemetry with RCON pings consistently in REST and status events', async () => { const { db, inspect, rcon, fanoutStatus } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); @@ -496,23 +498,49 @@ describe('database fleet and container readiness', () => { inspect.mockResolvedValue({ ok: true, state: 'running' }); const telemetry = gameEvent('heartbeat', 15, 3, 'managed'); telemetry.players[0].ping = 42; + rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', + status: { players: 4, map: 'zombie_theater', gametype: 'zom', roster: [ + { num: 0, guid: telemetry.players[0].guid, name: 'Old name', score: 50, ping: 100 }, + { num: 1, guid: telemetry.players[1].guid, name: 'Another old name', score: 100, ping: 30 } + ] } }); await postTelemetry(telemetry); const events: unknown[] = []; const unsubscribe = subscribe(event => events.push(event)); try { await fanoutStatus(); } finally { unsubscribe(); } - expect(rcon).not.toHaveBeenCalled(); + expect(rcon).toHaveBeenCalledOnce(); const server = (await (await GET()).json()).servers[0]; expect(server).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry', last_error: null, - rcon_status: { last_players: 4, last_error: 'timeout:old process' } }); + rcon_status: { last_players: 4, last_error: null } }); expect(server.roster).toHaveLength(3); expect(server.roster[0]).toMatchObject({ ping: 42, score: 5000 }); - expect(server.roster[1].ping).toBeNull(); + expect(server.roster[1]).toMatchObject({ ping: 30, score: 5000, name: 'Player 2' }); + expect(server.roster[2].ping).toBeNull(); expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ health: 'online', players: 3, roster: server.roster, lastSeenAt: server.last_seen_at })] })); expect(db.prepare("SELECT players FROM metric_samples WHERE server = 'managed'").get()).toEqual({ players: 3 }); }); + it('keeps occupied telemetry online when supplementary RCON fails', async () => { + const { inspect, rcon, fanoutStatus } = await setup(); + const { GET } = await import('../../src/routes/api/servers/+server.js'); + inspect.mockResolvedValue({ ok: true, state: 'running' }); + rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); + await postTelemetry(gameEvent('heartbeat', 15, 1, 'managed')); + await fanoutStatus(); + expect(rcon).toHaveBeenCalledOnce(); + expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', status_source: 'telemetry', + last_players: 1, last_error: null, roster: [{ name: 'Player 1', score: 5000, ping: null }] }); + }); + + it('skips supplementary RCON for empty telemetry rosters', async () => { + const { inspect, rcon, fanoutStatus } = await setup(); + inspect.mockResolvedValue({ ok: true, state: 'running' }); + await postTelemetry(gameEvent('heartbeat', 15, 0, 'managed')); + await fanoutStatus(); + expect(rcon).not.toHaveBeenCalled(); + }); + it('falls back to RCON when telemetry expires and does not renew old reports', async () => { const { db, inspect, rcon, fanoutStatus } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js');