import { randomUUID } from 'node:crypto'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn() })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn() })); const server = 'managed'; const now = Date.parse('2026-10-05T16:00:00Z'); const healthy = { ok: true as const, state: 'running' }; const playing = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now); }); afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.restoreAllMocks(); }); async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); const database = db(); database.prepare('INSERT INTO servers(name,komodo_name,created_at,game_port) VALUES(?,?,?,28960)') .run(server, server, new Date(now).toISOString()); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); const inspect = vi.mocked(komodoInspectHealthy).mockResolvedValue(healthy); const rcon = vi.mocked(rconStatusResult).mockResolvedValue(playing); return { database, inspect, rcon, ...await import('../../src/lib/server/fleet/health-observations.js') }; } async function native(api: Awaited>) { const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Monitoring host' }); hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 }); const instanceId = randomUUID(); api.database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'library','image',28960,?)`).run(instanceId, server, host.id, new Date(now).toISOString()); return { host, instanceId, hosts }; } function deferred() { let resolve!: (value: T) => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; } describe('shared monitoring observations', () => { it('retains failures for watchdog consumption without caching them as reusable probes', async () => { const api = await setup(); api.rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'No reply' }); const publish = api.monitoringStatusWriter(server); publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server)); expect(api.monitoringStatus(server)).toMatchObject({ container: { observedAt: new Date(now).toISOString(), result: healthy }, game: { observedAt: new Date(now).toISOString(), result: { ok: false, reason: 'timeout' } } }); vi.setSystemTime(now + 40_000); expect(api.monitoringStatus(server)?.game?.observedAt).toBe(new Date(now).toISOString()); expect(api.inspect).toHaveBeenCalledTimes(1); expect(api.rcon).toHaveBeenCalledTimes(1); vi.setSystemTime(now + 45_001); expect(api.monitoringStatus(server)).toBeNull(); vi.setSystemTime(now - 1); expect(api.monitoringStatus(server)).toBeNull(); }); it.each(['registration', 'generation', 'lifecycle', 'endpoint', 'manual', 'offline'] as const)('invalidates a completed status snapshot after %s changes', async change => { const api = await setup(); const host = await native(api); const publish = api.monitoringStatusWriter(server); publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server)); expect(api.monitoringStatus(server)).not.toBeNull(); if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server); if (change === 'generation') api.database.prepare('UPDATE native_instances SET generation=2').run(); if (change === 'lifecycle') api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString()); if (change === 'endpoint') api.database.prepare('UPDATE native_instances SET port=28961').run(); if (change === 'manual') api.database.prepare('UPDATE servers SET container_checked_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server); if (change === 'offline') api.database.prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run(new Date(now - 300_000).toISOString(), host.host.id); expect(api.monitoringStatus(server)).toBeNull(); }); it('does not publish a status pass across an instance replacement', async () => { const api = await setup(); await native(api); const publish = api.monitoringStatusWriter(server); const container = await api.inspectForMonitoring(server); const game = await api.rconForMonitoring(server); api.database.prepare('UPDATE native_instances SET generation=2').run(); publish(container, game); expect(api.monitoringStatus(server)).toBeNull(); }); it.each(['inspect', 'rcon'] as const)('invalidates the routine snapshot when a forced %s check starts', async kind => { const api = await setup(); const publish = api.monitoringStatusWriter(server); const container = await api.inspectForMonitoring(server); const game = await api.rconForMonitoring(server); publish(container, game); expect(api.monitoringStatus(server)).not.toBeNull(); await (kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring)(server, true); publish(container, game); expect(api.monitoringStatus(server)).toBeNull(); }); it('excludes old game evidence even when the container sample is fresh', async () => { const api = await setup(); const game = await api.rconForMonitoring(server); vi.setSystemTime(now + 46_000); const publish = api.monitoringStatusWriter(server); publish(await api.inspectForMonitoring(server), game); expect(api.monitoringStatus(server)?.container.result).toEqual(healthy); expect(api.monitoringStatus(server)?.game).toBeUndefined(); expect(api.rcon).toHaveBeenCalledTimes(1); }); it.each(['inspect', 'rcon'] as const)('reuses %s results until 30 seconds after request start without refreshing their timestamp', async kind => { const api = await setup(); const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring; const request = kind === 'inspect' ? api.inspect : api.rcon; const first = await call(server); expect(first.observedAt).toBe(new Date(now).toISOString()); vi.setSystemTime(now + 29_999); expect(await call(server)).toEqual(first); expect(request).toHaveBeenCalledTimes(1); vi.setSystemTime(now + 30_000); expect((await call(server)).observedAt).toBe(new Date(now + 30_000).toISOString()); expect(request).toHaveBeenCalledTimes(2); }); it('shares overlapping monitoring requests, but keeps inspection and RCON separate', async () => { const api = await setup(); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const observations = [api.inspectForMonitoring(server), api.inspectForMonitoring(server)]; await api.rconForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(1); expect(api.rcon).toHaveBeenCalledTimes(1); vi.setSystemTime(now + 5000); pending.resolve(healthy); expect(await Promise.all(observations)).toEqual(Array(2).fill({ result: healthy, observedAt: new Date(now + 5000).toISOString() })); }); it('timestamps a slow result on completion without making it reusable after its request expires', async () => { const api = await setup(); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const observation = api.inspectForMonitoring(server); await Promise.resolve(); vi.setSystemTime(now + 31_000); pending.resolve(healthy); expect(await observation).toEqual({ result: healthy, observedAt: new Date(now + 31_000).toISOString() }); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('reuses the saved completion timestamp while expiring from the earlier request start', async () => { const api = await setup(); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const observation = api.inspectForMonitoring(server); await Promise.resolve(); vi.setSystemTime(now + 20_000); pending.resolve(healthy); const completed = await observation; api.database.prepare('UPDATE servers SET container_checked_at=? WHERE name=?').run(completed.observedAt, server); vi.setSystemTime(now + 25_000); expect(await api.inspectForMonitoring(server)).toEqual(completed); expect(api.inspect).toHaveBeenCalledTimes(1); vi.setSystemTime(now + 30_000); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('does not reuse observations from a future clock', async () => { const api = await setup(); await api.inspectForMonitoring(server); vi.setSystemTime(now - 1); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('does not cache failed results or rejected requests', async () => { const api = await setup(); api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' }); expect((await api.inspectForMonitoring(server)).result.ok).toBe(false); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); api.rcon.mockRejectedValueOnce(new Error('Unavailable')); await expect(api.rconForMonitoring(server)).rejects.toThrow('Unavailable'); await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); it('forced checks bypass completed results and evict old success when the forced result fails', async () => { const api = await setup(); await api.inspectForMonitoring(server); api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' }); expect((await api.inspectForMonitoring(server, true)).result.ok).toBe(false); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(3); }); it('forced checks bypass wrapper overlap and an older reply cannot replace the newer cache entry', async () => { const api = await setup(); const old = deferred(); api.inspect.mockReturnValueOnce(old.promise); const first = api.inspectForMonitoring(server); await Promise.resolve(); vi.setSystemTime(now + 1000); const latest = await api.inspectForMonitoring(server, true); old.resolve(healthy); await first; expect(await api.inspectForMonitoring(server)).toEqual(latest); expect(api.inspect).toHaveBeenCalledTimes(2); }); it.each(['registration', 'lifecycle', 'desired state', 'stop'] as const)('invalidates cached observations after a %s change', async change => { const api = await setup(); await api.inspectForMonitoring(server); if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server); if (change === 'lifecycle') api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString()); if (change === 'desired state') api.database.prepare("UPDATE servers SET desired_state='stopped' WHERE name=?").run(server); if (change === 'stop') api.database.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) VALUES('stop',?,'stop','awaiting-health','Stopping',?,?,'ui')`).run(server, new Date(now).toISOString(), new Date(now + 1).toISOString()); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('invalidates integration changes and accepts legacy port discovery by the successful RCON probe', async () => { const api = await setup(); await api.inspectForMonitoring(server); api.database.prepare("UPDATE servers SET komodo_name='replacement' WHERE name=?").run(server); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); api.database.prepare('UPDATE servers SET game_port=NULL WHERE name=?').run(server); api.rcon.mockImplementationOnce(async () => { api.database.prepare('UPDATE servers SET game_port=28960 WHERE name=?').run(server); return playing; }); expect((await api.rconForMonitoring(server)).result).toEqual(playing); await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(1); api.database.prepare('UPDATE servers SET game_port=28961 WHERE name=?').run(server); await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); it.each(['inspect', 'rcon'] as const)('does not reuse %s after a newer manual observation', async kind => { const api = await setup(); const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring; const request = kind === 'inspect' ? api.inspect : api.rcon; const field = kind === 'inspect' ? 'container_checked_at' : 'last_checked_at'; await call(server); vi.setSystemTime(now + 1000); api.database.prepare(`UPDATE servers SET ${field}=? WHERE name=?`).run(new Date(now + 1000).toISOString(), server); await call(server); expect(request).toHaveBeenCalledTimes(2); }); it('rejects a previous generation completing after replacement without discarding the new generation cache', async () => { const api = await setup(); const { instanceId } = await native(api); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const old = api.inspectForMonitoring(server); const rejected = expect(old).rejects.toThrow('Server changed'); await Promise.resolve(); api.database.prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId); const current = await api.inspectForMonitoring(server); pending.resolve(healthy); await rejected; expect(await api.inspectForMonitoring(server)).toEqual(current); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('rejects a result when a lifecycle request arrives while its probe is in flight', async () => { const api = await setup(); const pending = deferred(); api.rcon.mockReturnValueOnce(pending.promise); const old = api.rconForMonitoring(server); const rejected = expect(old).rejects.toThrow('Server changed'); await Promise.resolve(); api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString()); pending.resolve(playing); await rejected; await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); it('discards a slow result when a newer manual observation was saved during the request', async () => { const api = await setup(); const pending = deferred(); api.rcon.mockReturnValueOnce(pending.promise); const old = api.rconForMonitoring(server); const rejected = expect(old).rejects.toBeInstanceOf(api.MonitoringChangedError); await Promise.resolve(); vi.setSystemTime(now + 1000); api.database.prepare('UPDATE servers SET last_checked_at=? WHERE name=?').run(new Date().toISOString(), server); pending.resolve(playing); await rejected; await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); it('discards cached native success while its host is offline and probes again after reconnecting', async () => { const api = await setup(); const { host, hosts } = await native(api); await api.inspectForMonitoring(server); api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id); await expect(api.inspectForMonitoring(server)).rejects.toThrow('Host agent is offline'); expect(api.inspect).toHaveBeenCalledTimes(1); hosts.heartbeat(host.id, { protocolVersion: 1 }); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('rejects an otherwise successful result if the native host went offline during the probe', async () => { const api = await setup(); const { host } = await native(api); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const observation = api.inspectForMonitoring(server); const rejected = expect(observation).rejects.toThrow('Host agent is offline'); await Promise.resolve(); api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id); pending.resolve(healthy); await rejected; }); });