fix: honor Komodo health and prefer fresh telemetry across fleet status

tests/unit/beta-plans.test.ts

@@ -155,6 +155,9 @@ describe('durable beta test plans', () => {
         const { getBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
         expect(getBeta(id)?.state).toBe('ready-promote');
         await tick(15, 0, false);
+        // Fresh occupied telemetry remains healthy despite an RCON timeout.
+        expect(getBeta(id)?.state).toBe('ready-promote');
+        await tick(31, 0, false);
         expect(getBeta(id)?.state).toBe('running');
         await expect(promoteBeta(id)).rejects.toThrow();
     });

tests/unit/komodo.test.ts

@@ -2,6 +2,21 @@ import { describe, expect, it } from 'vitest';
 import { extractUdpPort, extractContainerState } from '../../src/lib/server/integrations/komodo.js';
 
 describe('container readiness', () => {
+    it.each([
+        ['Up 35 seconds (health: starting)', 'starting'],
+        ['Up about a minute (healthy)', 'running'],
+        ['Up 5 minutes (unhealthy)', 'unhealthy'],
+        ['Up 2 hours', 'running'],
+        ['Exited (1) 5 seconds ago', 'exited']
+    ])('reads nested Komodo container status: %s', (status, expected) => {
+        expect(extractContainerState({ state: 'Running', container: { state: 'running', status } })).toBe(expected);
+    });
+    it('reads lowercase inspect health and does not trust missing containers or configuration', () => {
+        expect(extractContainerState({ state: { status: 'running', running: true, health: { status: 'starting' } } })).toBe('starting');
+        expect(extractContainerState({ state: 'Running', container: null })).toBe('unknown');
+        expect(extractContainerState({ config: { state: 'running' }, status: 'ok' })).toBeNull();
+        expect(extractContainerState({ state: 'Exited', container: { status: 'Up 1 minute (healthy)' } })).toBe('exited');
+    });
     it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => {
         expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health);
         expect(extractContainerState({ state: 'running', health })).toBe(health);

tests/unit/server-observation.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, it } from 'vitest';
+import { freshTelemetry, selectServerObservation, type ServerObservation } from '../../src/lib/server-observation.js';
+import { gameEvent } from '../helpers/telemetry.js';
+
+const now = Date.parse('2026-09-27T12:00:00Z');
+const at = (offset = 0) => new Date(now + offset).toISOString();
+const report = (offset = 0, players = 3) => ({ ...gameEvent('heartbeat', 15, players, 'managed'), receivedAt: at(offset) });
+function server(changes: Partial<ServerObservation> = {}): ServerObservation {
+    return { name: 'managed', health: 'online', container_state: 'running', container_checked_at: at(),
+        last_players: 4, last_map: 'zombie_theater', last_gametype: 'zom', last_seen_at: at(),
+        last_error: null, roster: [], ...changes };
+}
+
+describe('shared server observation', () => {
+    it('selects the telemetry count and roster together and keeps raw RCON evidence', () => {
+        const raw = server({ last_error: 'timeout' });
+        const selected = selectServerObservation(server({ rcon_status: raw }), report(), now);
+        expect(selected).toMatchObject({ health: 'online', status_source: 'telemetry', last_players: 3, last_error: null });
+        expect(selected.roster).toHaveLength(3);
+        expect(selected.roster?.[0]).toMatchObject({ ping: null, score: 5000 });
+        expect(selected.rcon_status).toBe(raw);
+    });
+
+    it('uses the latest HTTP or streamed telemetry and falls back without extending freshness', () => {
+        const original = server();
+        const selected = selectServerObservation(server({ rcon_status: original, telemetry: report(0, 2) }), report(-1000), now);
+        expect(selected.last_players).toBe(2);
+        const fallback = selectServerObservation({ ...selected, container_checked_at: at(46_000),
+            rcon_status: { ...original, last_seen_at: at(40_000) } }, undefined, now + 46_000);
+        expect(fallback).toMatchObject({ status_source: 'rcon', health: 'online', last_players: 4, last_seen_at: at(40_000) });
+    });
+
+    it.each(['starting', 'restarting', 'unhealthy', 'exited', 'unknown'])('telemetry cannot override container state %s', state => {
+        const selected = selectServerObservation(server({ health: state === 'starting' || state === 'restarting' ? 'starting' : 'offline', container_state: state }), report(), now);
+        expect(selected.health).not.toBe('online');
+        expect(selected.last_players).toBeNull();
+        expect(selected.roster).toBeNull();
+    });
+
+    it('requires fresh container evidence and reports after the latest lifecycle submission', () => {
+        expect(selectServerObservation(server({ container_checked_at: at(-46_000) }), report(), now).health).not.toBe('online');
+        expect(selectServerObservation(server({ health: 'provisioning' }), report(), now).health).toBe('provisioning');
+        expect(selectServerObservation(server({ telemetry_after: at() }), report(-1), now).health).toBe('offline');
+        expect(freshTelemetry(report(), now, 'invalid')).toBe(false);
+        expect(freshTelemetry(report(2000), now)).toBe(false);
+        expect(freshTelemetry(report(-46_000), now)).toBe(false);
+        expect(freshTelemetry({ ...report(), players: [{}] } as never, now)).toBe(false);
+    });
+
+    it('uses game-start rosters without inventing scores and clears match times on an empty report', () => {
+        const start = { ...gameEvent('game_start', 0, 2, 'managed'), receivedAt: at() };
+        const active = selectServerObservation(server(), start, now);
+        expect(active.roster?.[0]).toMatchObject({ score: null, ping: null });
+        expect(active.game_started_at).toBe(at());
+        expect(selectServerObservation({ ...active, game_ended_at: at() }, report(1, 0), now + 1))
+            .toMatchObject({ last_players: 0, roster: [], game_started_at: null, game_ended_at: null });
+    });
+});

tests/unit/server-status.test.ts

@@ -76,6 +76,9 @@ describe('database fleet and container readiness', () => {
         await postTelemetry(gameEvent('game_end', 10, 1, 'managed'));
         rcon.mockResolvedValue(reply(0));
         await fanoutStatus();
+        expect(getServer('managed')?.game_ended_at).not.toBeNull();
+        await postTelemetry(gameEvent('heartbeat', 0, 0, 'managed'));
+        await fanoutStatus();
         expect(getServer('managed')).toMatchObject({ game_started_at: null, game_ended_at: null });
         rcon.mockResolvedValue(reply(1));
         await fanoutStatus();
@@ -118,6 +121,41 @@ describe('database fleet and container readiness', () => {
         expect(snapshot.serverNow).toBeLessThanOrEqual(Date.now());
     });
 
+    it('uses telemetry consistently in REST, status events, and metrics without polling RCON', async () => {
+        const { db, inspect, rcon, fanoutStatus } = await setup();
+        const { GET } = await import('../../src/routes/api/servers/+server.js');
+        const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
+        db.prepare("UPDATE servers SET last_players = 4, last_seen_at = ? WHERE name = 'managed'").run(new Date().toISOString());
+        inspect.mockResolvedValue({ ok: true, state: 'running' });
+        await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
+        const events: unknown[] = [];
+        const unsubscribe = subscribe(event => events.push(event));
+        try { await fanoutStatus(); } finally { unsubscribe(); }
+        expect(rcon).not.toHaveBeenCalled();
+        const server = (await (await GET()).json()).servers[0];
+        expect(server).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry', last_error: null,
+            rcon_status: { last_players: 4, last_error: 'timeout:old process' } });
+        expect(server.roster).toHaveLength(3);
+        expect(server.roster[0]).toMatchObject({ ping: null, score: 5000 });
+        expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
+            health: 'online', players: 3, roster: server.roster, lastSeenAt: server.last_seen_at
+        })] }));
+        expect(db.prepare("SELECT players FROM metric_samples WHERE server = 'managed'").get()).toEqual({ players: 3 });
+    });
+
+    it('falls back to RCON when telemetry expires and does not renew old reports', async () => {
+        const { db, inspect, rcon, fanoutStatus } = await setup();
+        const { GET } = await import('../../src/routes/api/servers/+server.js');
+        inspect.mockResolvedValue({ ok: true, state: 'running' });
+        await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
+        db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 46_000).toISOString());
+        rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
+            status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        await fanoutStatus();
+        expect(rcon).toHaveBeenCalledTimes(1);
+        expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_players: 0, status_source: 'rcon' });
+    });
+
     it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => {
         const { db } = await setup();
         const { allServers, listServers } = await import('../../src/lib/server/fleet/servers.js');
@@ -151,8 +189,11 @@ describe('database fleet and container readiness', () => {
     it.each(['unhealthy', 'exited', undefined])('does not poll RCON with container state %s', async (state) => {
         const { inspect, rcon, fanoutStatus } = await setup();
         inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' });
+        await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
         await fanoutStatus();
         expect(rcon).not.toHaveBeenCalled();
+        const { GET } = await import('../../src/routes/api/servers/+server.js');
+        expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'offline', last_players: null });
     });
 
     it('waits for an EC restart request to settle before trusting the old healthy container', async () => {

tests/unit/ui-server-clock.test.ts

@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { get } from 'svelte/store';
 import { freshServer, matchesFleet, type FleetServer } from '../../src/lib/fleet-overview.js';
 import type { ServerStatus } from '../../src/lib/ui/stores/servers.js';
+import { gameEvent } from '../helpers/telemetry.js';
 
 vi.mock('$app/environment', () => ({ browser: true }));
 
@@ -65,6 +66,20 @@ async function setup() {
 }
 
 describe('Fleet report clock', () => {
+    it('updates count and roster immediately from telemetry and expires them together', async () => {
+        snapshot.servers = [server(epoch, { container_state: 'running', container_checked_at: new Date(epoch).toISOString(), last_players: 4 })];
+        const { fleetServers, uiNow } = await setup();
+        await vi.advanceTimersByTimeAsync(1000);
+        source.dispatchEvent(new MessageEvent('server-telemetry', { data: JSON.stringify({ serverNow: epoch + 1000,
+            entries: [{ ...gameEvent('heartbeat', 15, 3), receivedAt: new Date(epoch + 1000).toISOString() }] }) }));
+        expect(get(fleetServers)[0]).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry' });
+        expect(get(fleetServers)[0].roster).toHaveLength(3);
+        expect(matchesFleet(get(fleetServers)[0], 'Player 3', 'occupied', '', [], [], get(uiNow))).toBe(true);
+        await vi.advanceTimersByTimeAsync(46_000);
+        expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
+        expect(matchesFleet(get(fleetServers)[0], '', 'occupied', '', [], [], get(uiNow))).toBe(false);
+    });
+
     it('applies game lifecycle events immediately and clears them after an empty report', async () => {
         const { fleetServers } = await setup();
         const report = (event: string, time: number) => source.dispatchEvent(new MessageEvent('server-telemetry', {