import { randomUUID } from 'node:crypto'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn() })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn() })); const server = 'managed'; const now = Date.parse('2026-10-05T16:00:00Z'); const healthy = { ok: true as const, state: 'running' }; const playing = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now); }); afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.restoreAllMocks(); }); async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); const database = db(); database.prepare('INSERT INTO servers(name,komodo_name,created_at,game_port) VALUES(?,?,?,28960)') .run(server, server, new Date(now).toISOString()); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); const inspect = vi.mocked(komodoInspectHealthy).mockResolvedValue(healthy); const rcon = vi.mocked(rconStatusResult).mockResolvedValue(playing); return { database, inspect, rcon, ...await import('../../src/lib/server/fleet/health-observations.js') }; } async function native(api: Awaited>) { const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Monitoring host' }); hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 }); const instanceId = randomUUID(); api.database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'library','image',28960,?)`).run(instanceId, server, host.id, new Date(now).toISOString()); return { host, instanceId, hosts }; } function deferred() { let resolve!: (value: T) => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; } describe('shared monitoring observations', () => { it.each(['inspect', 'rcon'] as const)('reuses %s results until 30 seconds after request start without refreshing their timestamp', async kind => { const api = await setup(); const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring; const request = kind === 'inspect' ? api.inspect : api.rcon; const first = await call(server); expect(first.observedAt).toBe(new Date(now).toISOString()); vi.setSystemTime(now + 29_999); expect(await call(server)).toEqual(first); expect(request).toHaveBeenCalledTimes(1); vi.setSystemTime(now + 30_000); expect((await call(server)).observedAt).toBe(new Date(now + 30_000).toISOString()); expect(request).toHaveBeenCalledTimes(2); }); it('shares overlapping monitoring requests, but keeps inspection and RCON separate', async () => { const api = await setup(); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const observations = [api.inspectForMonitoring(server), api.inspectForMonitoring(server)]; await api.rconForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(1); expect(api.rcon).toHaveBeenCalledTimes(1); vi.setSystemTime(now + 5000); pending.resolve(healthy); expect(await Promise.all(observations)).toEqual(Array(2).fill({ result: healthy, observedAt: new Date(now + 5000).toISOString() })); }); it('timestamps a slow result on completion without making it reusable after its request expires', async () => { const api = await setup(); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const observation = api.inspectForMonitoring(server); await Promise.resolve(); vi.setSystemTime(now + 31_000); pending.resolve(healthy); expect(await observation).toEqual({ result: healthy, observedAt: new Date(now + 31_000).toISOString() }); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('reuses the saved completion timestamp while expiring from the earlier request start', async () => { const api = await setup(); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const observation = api.inspectForMonitoring(server); await Promise.resolve(); vi.setSystemTime(now + 20_000); pending.resolve(healthy); const completed = await observation; api.database.prepare('UPDATE servers SET container_checked_at=? WHERE name=?').run(completed.observedAt, server); vi.setSystemTime(now + 25_000); expect(await api.inspectForMonitoring(server)).toEqual(completed); expect(api.inspect).toHaveBeenCalledTimes(1); vi.setSystemTime(now + 30_000); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('does not reuse observations from a future clock', async () => { const api = await setup(); await api.inspectForMonitoring(server); vi.setSystemTime(now - 1); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('does not cache failed results or rejected requests', async () => { const api = await setup(); api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' }); expect((await api.inspectForMonitoring(server)).result.ok).toBe(false); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); api.rcon.mockRejectedValueOnce(new Error('Unavailable')); await expect(api.rconForMonitoring(server)).rejects.toThrow('Unavailable'); await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); it('forced checks bypass completed results and evict old success when the forced result fails', async () => { const api = await setup(); await api.inspectForMonitoring(server); api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' }); expect((await api.inspectForMonitoring(server, true)).result.ok).toBe(false); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(3); }); it('forced checks bypass wrapper overlap and an older reply cannot replace the newer cache entry', async () => { const api = await setup(); const old = deferred(); api.inspect.mockReturnValueOnce(old.promise); const first = api.inspectForMonitoring(server); await Promise.resolve(); vi.setSystemTime(now + 1000); const latest = await api.inspectForMonitoring(server, true); old.resolve(healthy); await first; expect(await api.inspectForMonitoring(server)).toEqual(latest); expect(api.inspect).toHaveBeenCalledTimes(2); }); it.each(['registration', 'lifecycle', 'desired state', 'stop'] as const)('invalidates cached observations after a %s change', async change => { const api = await setup(); await api.inspectForMonitoring(server); if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server); if (change === 'lifecycle') api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString()); if (change === 'desired state') api.database.prepare("UPDATE servers SET desired_state='stopped' WHERE name=?").run(server); if (change === 'stop') api.database.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) VALUES('stop',?,'stop','awaiting-health','Stopping',?,?,'ui')`).run(server, new Date(now).toISOString(), new Date(now + 1).toISOString()); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('invalidates integration changes and accepts legacy port discovery by the successful RCON probe', async () => { const api = await setup(); await api.inspectForMonitoring(server); api.database.prepare("UPDATE servers SET komodo_name='replacement' WHERE name=?").run(server); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); api.database.prepare('UPDATE servers SET game_port=NULL WHERE name=?').run(server); api.rcon.mockImplementationOnce(async () => { api.database.prepare('UPDATE servers SET game_port=28960 WHERE name=?').run(server); return playing; }); expect((await api.rconForMonitoring(server)).result).toEqual(playing); await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(1); api.database.prepare('UPDATE servers SET game_port=28961 WHERE name=?').run(server); await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); it.each(['inspect', 'rcon'] as const)('does not reuse %s after a newer manual observation', async kind => { const api = await setup(); const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring; const request = kind === 'inspect' ? api.inspect : api.rcon; const field = kind === 'inspect' ? 'container_checked_at' : 'last_checked_at'; await call(server); vi.setSystemTime(now + 1000); api.database.prepare(`UPDATE servers SET ${field}=? WHERE name=?`).run(new Date(now + 1000).toISOString(), server); await call(server); expect(request).toHaveBeenCalledTimes(2); }); it('rejects a previous generation completing after replacement without discarding the new generation cache', async () => { const api = await setup(); const { instanceId } = await native(api); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const old = api.inspectForMonitoring(server); const rejected = expect(old).rejects.toThrow('Server changed'); await Promise.resolve(); api.database.prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId); const current = await api.inspectForMonitoring(server); pending.resolve(healthy); await rejected; expect(await api.inspectForMonitoring(server)).toEqual(current); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('rejects a result when a lifecycle request arrives while its probe is in flight', async () => { const api = await setup(); const pending = deferred(); api.rcon.mockReturnValueOnce(pending.promise); const old = api.rconForMonitoring(server); const rejected = expect(old).rejects.toThrow('Server changed'); await Promise.resolve(); api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString()); pending.resolve(playing); await rejected; await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); it('discards a slow result when a newer manual observation was saved during the request', async () => { const api = await setup(); const pending = deferred(); api.rcon.mockReturnValueOnce(pending.promise); const old = api.rconForMonitoring(server); const rejected = expect(old).rejects.toBeInstanceOf(api.MonitoringChangedError); await Promise.resolve(); vi.setSystemTime(now + 1000); api.database.prepare('UPDATE servers SET last_checked_at=? WHERE name=?').run(new Date().toISOString(), server); pending.resolve(playing); await rejected; await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); it('discards cached native success while its host is offline and probes again after reconnecting', async () => { const api = await setup(); const { host, hosts } = await native(api); await api.inspectForMonitoring(server); api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id); await expect(api.inspectForMonitoring(server)).rejects.toThrow('Host agent is offline'); expect(api.inspect).toHaveBeenCalledTimes(1); hosts.heartbeat(host.id, { protocolVersion: 1 }); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); it('rejects an otherwise successful result if the native host went offline during the probe', async () => { const api = await setup(); const { host } = await native(api); const pending = deferred(); api.inspect.mockReturnValueOnce(pending.promise); const observation = api.inspectForMonitoring(server); const rejected = expect(observation).rejects.toThrow('Host agent is offline'); await Promise.resolve(); api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id); pending.resolve(healthy); await rejected; }); });