Use status monitor evidence for routine watchdog health checks

tests/unit/health-observations.test.ts

@@ -48,6 +48,74 @@ function deferred<T>() {
 }
 
 describe('shared monitoring observations', () => {
+    it('retains failures for watchdog consumption without caching them as reusable probes', async () => {
+        const api = await setup();
+        api.rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'No reply' });
+        const publish = api.monitoringStatusWriter(server);
+        publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server));
+        expect(api.monitoringStatus(server)).toMatchObject({
+            container: { observedAt: new Date(now).toISOString(), result: healthy },
+            game: { observedAt: new Date(now).toISOString(), result: { ok: false, reason: 'timeout' } }
+        });
+        vi.setSystemTime(now + 40_000);
+        expect(api.monitoringStatus(server)?.game?.observedAt).toBe(new Date(now).toISOString());
+        expect(api.inspect).toHaveBeenCalledTimes(1);
+        expect(api.rcon).toHaveBeenCalledTimes(1);
+        vi.setSystemTime(now + 45_001);
+        expect(api.monitoringStatus(server)).toBeNull();
+        vi.setSystemTime(now - 1);
+        expect(api.monitoringStatus(server)).toBeNull();
+    });
+
+    it.each(['registration', 'generation', 'lifecycle', 'endpoint', 'manual', 'offline'] as const)('invalidates a completed status snapshot after %s changes', async change => {
+        const api = await setup();
+        const host = await native(api);
+        const publish = api.monitoringStatusWriter(server);
+        publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server));
+        expect(api.monitoringStatus(server)).not.toBeNull();
+        if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server);
+        if (change === 'generation') api.database.prepare('UPDATE native_instances SET generation=2').run();
+        if (change === 'lifecycle') api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString());
+        if (change === 'endpoint') api.database.prepare('UPDATE native_instances SET port=28961').run();
+        if (change === 'manual') api.database.prepare('UPDATE servers SET container_checked_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server);
+        if (change === 'offline') api.database.prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run(new Date(now - 300_000).toISOString(), host.host.id);
+        expect(api.monitoringStatus(server)).toBeNull();
+    });
+
+    it('does not publish a status pass across an instance replacement', async () => {
+        const api = await setup();
+        await native(api);
+        const publish = api.monitoringStatusWriter(server);
+        const container = await api.inspectForMonitoring(server);
+        const game = await api.rconForMonitoring(server);
+        api.database.prepare('UPDATE native_instances SET generation=2').run();
+        publish(container, game);
+        expect(api.monitoringStatus(server)).toBeNull();
+    });
+
+    it.each(['inspect', 'rcon'] as const)('invalidates the routine snapshot when a forced %s check starts', async kind => {
+        const api = await setup();
+        const publish = api.monitoringStatusWriter(server);
+        const container = await api.inspectForMonitoring(server);
+        const game = await api.rconForMonitoring(server);
+        publish(container, game);
+        expect(api.monitoringStatus(server)).not.toBeNull();
+        await (kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring)(server, true);
+        publish(container, game);
+        expect(api.monitoringStatus(server)).toBeNull();
+    });
+
+    it('excludes old game evidence even when the container sample is fresh', async () => {
+        const api = await setup();
+        const game = await api.rconForMonitoring(server);
+        vi.setSystemTime(now + 46_000);
+        const publish = api.monitoringStatusWriter(server);
+        publish(await api.inspectForMonitoring(server), game);
+        expect(api.monitoringStatus(server)?.container.result).toEqual(healthy);
+        expect(api.monitoringStatus(server)?.game).toBeUndefined();
+        expect(api.rcon).toHaveBeenCalledTimes(1);
+    });
+
     it.each(['inspect', 'rcon'] as const)('reuses %s results until 30 seconds after request start without refreshing their timestamp', async kind => {
         const api = await setup();
         const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring;

tests/unit/host-status-batch.test.ts

@@ -59,6 +59,10 @@ describe('host status observations', () => {
         } finally { api.complete(); await Promise.all(passes); }
         expect(api.inspect).not.toHaveBeenCalled();
         expect(api.getServer('server-2-4')).toMatchObject({ last_players: 0, last_map: 'mp_kino', container_checked_at: new Date(at).toISOString() });
+        expect(api.monitoringStatus('server-2-4')).toMatchObject({
+            container: { observedAt: new Date(at).toISOString(), result: { ok: true } },
+            game: { observedAt: api.getServer('server-2-4')?.last_seen_at, result: { ok: true } }
+        });
         expect(api.db.prepare('SELECT COUNT(*) AS n FROM metric_samples').get()).toEqual({ n: 15 });
         await api.fanoutStatus();
         expect(api.operations()).toHaveLength(3);
@@ -90,6 +94,9 @@ describe('host status observations', () => {
         api.heartbeat(host.id, { ...host.inventory, instanceStates: [{ ...host.inventory.instanceStates![0], healthy: false, state: 'exited' }] });
         await api.fanoutStatus();
         expect(api.getServer('server-0-0')?.container_state).toBe('exited');
+        expect(api.monitoringStatus('server-0-0')).toMatchObject({ container: {
+            observedAt: new Date(at).toISOString(), result: { ok: false, state: 'exited' }
+        } });
         expect(api.operations()).toEqual([]);
     });
 
@@ -120,6 +127,7 @@ describe('host status observations', () => {
         } finally { api.complete(); await pass; }
         expect(api.getServer('server-0-0')?.last_error).toBe('bad-reply:wrong RCON password');
         expect(api.getServer('server-0-4')?.last_error).toContain('timeout:');
+        expect(api.monitoringStatus('server-0-4')?.game?.result).toMatchObject({ ok: false, reason: 'timeout' });
         expect(api.operations()).toHaveLength(1);
     });
 

tests/unit/watchdog.test.ts

@@ -58,6 +58,10 @@ const onlyIncident = () => { expect(incidents()).toHaveLength(1); return inciden
 const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[];
 function advance(ms: number) { vi.setSystemTime(Date.now() + ms); }
 function down() { vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
+async function pollStatus() {
+    const worker = await vi.importActual<typeof import('../../src/lib/server/pipeline/worker.js')>('../../src/lib/server/pipeline/worker.js');
+    await worker.fanoutStatus();
+}
 async function sustainedFailure() {
     down();
     await watchdog.watchdogTick(true);
@@ -68,24 +72,148 @@ async function sustainedFailure() {
 }
 
 describe('Watchdog detection and ownership', () => {
+    it('waits for status-monitor evidence without issuing its own routine probes', async () => {
+        down();
+        for (let minute = 0; minute < 4; minute++) {
+            await watchdog.watchdogTick();
+            advance(watchdog.WATCHDOG_SCAN_MS);
+        }
+        expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        expect(incidents()).toEqual([]);
+        expect(operations()).toEqual([]);
+    });
+
+    it.each(['container', 'game'])('requests recovery from sustained %s failures collected by status polling', async kind => {
+        if (kind === 'container') down();
+        else vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
+        for (let minute = 0; minute < 3; minute++) {
+            await pollStatus();
+            await watchdog.watchdogTick();
+            if (minute < 2) {
+                expect(onlyIncident().failures).toBe(minute + 1);
+                expect(operations()).toEqual([]);
+                advance(watchdog.WATCHDOG_SCAN_MS);
+            }
+        }
+        expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
+        expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
+        expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(3);
+        expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0);
+    });
+
+    it('does not count the same sample again or use stale data, and resets after a monitoring gap', async () => {
+        down();
+        await pollStatus();
+        await watchdog.watchdogTick();
+        const first = onlyIncident();
+        advance(30_000);
+        database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
+        await watchdog.watchdogTick();
+        advance(60_000);
+        await watchdog.watchdogTick();
+        expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at });
+        expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(1);
+        advance(60_000);
+        await pollStatus();
+        await watchdog.watchdogTick();
+        expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
+        expect(operations()).toEqual([]);
+    });
+
+    it('uses sample times across polling offsets without counting early or losing sustained failures', async () => {
+        down();
+        await pollStatus();
+        advance(20_000);
+        await watchdog.watchdogTick();
+        advance(10_000);
+        await pollStatus();
+        advance(50_000);
+        // The first failure is stale until the monitor supplies another sample.
+        await pollStatus();
+        await watchdog.watchdogTick();
+        expect(onlyIncident().failures).toBe(2);
+        advance(40_000);
+        await pollStatus();
+        advance(20_000);
+        await watchdog.watchdogTick();
+        expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
+        expect(operations()).toHaveLength(1);
+    });
+
+    it('breaks a routine failure streak on a successful status poll between watchdog passes', async () => {
+        vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
+        await pollStatus();
+        await watchdog.watchdogTick();
+        advance(30_000);
+        vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
+        await pollStatus();
+        advance(30_000);
+        await pollStatus();
+        await watchdog.watchdogTick();
+        expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
+        expect(operations()).toEqual([]);
+        expect(rcon.rconStatusResult).toHaveBeenCalledTimes(3);
+    });
+
+    it('uses fresh telemetry alongside the polled container and waits for polled RCON when telemetry expires', async () => {
+        const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
+        saveTelemetry({ ...gameEvent('heartbeat', 17, 4, server), receivedAt: new Date().toISOString() });
+        await pollStatus();
+        await watchdog.watchdogTick();
+        expect(incidents()).toEqual([]);
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        advance(watchdog.WATCHDOG_SCAN_MS);
+        await watchdog.watchdogTick();
+        expect(incidents()).toEqual([]);
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
+        await pollStatus();
+        advance(30_000);
+        database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
+        await watchdog.watchdogTick();
+        expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open' });
+        expect(rcon.rconStatusResult).toHaveBeenCalledTimes(1);
+    });
+
+    it('starts a new failure window for a replacement registration', async () => {
+        down();
+        await pollStatus();
+        await watchdog.watchdogTick();
+        advance(watchdog.WATCHDOG_SCAN_MS);
+        await pollStatus();
+        await watchdog.watchdogTick();
+        expect(onlyIncident().failures).toBe(2);
+        advance(watchdog.WATCHDOG_SCAN_MS);
+        database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server);
+        await pollStatus();
+        await watchdog.watchdogTick();
+        expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
+        expect(operations()).toEqual([]);
+    });
+
     it('scans files every five minutes while retaining minute health checks and immediate manual scans', async () => {
         const drift = await import('../../src/lib/server/fleet/drift.js');
         const scan = vi.spyOn(drift, 'prepareDriftCampaign');
+        await pollStatus();
         await watchdog.watchdogTick();
         expect(scan).toHaveBeenCalledTimes(1);
         for (let minute = 1; minute < 5; minute++) {
             advance(watchdog.WATCHDOG_SCAN_MS);
+            await pollStatus();
             await watchdog.watchdogTick();
         }
         expect(scan).toHaveBeenCalledTimes(1);
         expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(5);
         advance(watchdog.WATCHDOG_SCAN_MS);
+        await pollStatus();
         await watchdog.watchdogTick();
         expect(scan).toHaveBeenCalledTimes(2);
         await watchdog.watchdogTick(true);
         expect(scan).toHaveBeenCalledTimes(3);
         advance(watchdog.WATCHDOG_SCAN_MS);
         database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server);
+        await pollStatus();
         await watchdog.watchdogTick();
         expect(scan).toHaveBeenCalledTimes(4);
         scan.mockRestore();
@@ -94,9 +222,11 @@ describe('Watchdog detection and ownership', () => {
     it('retries failed file scans on the next health scan instead of retaining successful freshness', async () => {
         const drift = await import('../../src/lib/server/fleet/drift.js');
         const scan = vi.spyOn(drift, 'prepareDriftCampaign').mockRejectedValueOnce(new Error('Agent unavailable'));
+        await pollStatus();
         await watchdog.watchdogTick();
         expect(watchdog.watchdogOverview().scanError).toContain('Agent unavailable');
         advance(watchdog.WATCHDOG_SCAN_MS);
+        await pollStatus();
         await watchdog.watchdogTick();
         expect(scan).toHaveBeenCalledTimes(2);
         expect(watchdog.watchdogOverview().scanError).toBeNull();
@@ -112,6 +242,7 @@ describe('Watchdog detection and ownership', () => {
         expect(onlyIncident().failures).toBe(1);
         expect(operations()).toEqual([]);
         advance(watchdog.WATCHDOG_SCAN_MS);
+        await pollStatus();
         await watchdog.watchdogTick();
         expect(onlyIncident().failures).toBe(2);
         advance(watchdog.WATCHDOG_SCAN_MS - 1);