josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Share recent health observations between background monitors

Browse Source
This commit is contained in:
josh
2026-10-05 13:00:53 -04:00
parent f16879036e
commit 2d069907f2
5 changed files with 452 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/server-status.test.ts
+47
View File
@@ -30,6 +30,51 @@ async function setup() {
}
describe('database fleet and container readiness', () => {
it('shares routine observations without renewing their timestamps and keeps manual probes fresh', async () => {
const { inspect, rcon, fanoutStatus, getServer } = await setup();
const { inspectForMonitoring, rconForMonitoring } = await import('../../src/lib/server/fleet/health-observations.js');
vi.useFakeTimers({ toFake: ['Date'] });
const at = Date.parse('2026-10-05T12:00:00Z');
vi.setSystemTime(at);
inspect.mockResolvedValue({ ok: true, state: 'running' });
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
await inspectForMonitoring('managed');
await rconForMonitoring('managed');
vi.setSystemTime(at + 10_000);
await fanoutStatus();
expect(inspect).toHaveBeenCalledTimes(1);
expect(rcon).toHaveBeenCalledTimes(1);
expect(getServer('managed')).toMatchObject({ container_checked_at: new Date(at).toISOString(), last_seen_at: new Date(at).toISOString() });
const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'manual failure' });
await POST({ params: { name: 'managed' } } as never);
expect(rcon).toHaveBeenCalledTimes(2);
vi.setSystemTime(at + 20_000);
await fanoutStatus();
expect(rcon).toHaveBeenCalledTimes(3);
expect(getServer('managed')).toMatchObject({ last_error: 'timeout:manual failure', last_seen_at: new Date(at).toISOString() });
});
it('does not overwrite a newer manual observation with an earlier background reply', async () => {
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
vi.useFakeTimers({ toFake: ['Date'] });
const at = Date.parse('2026-10-05T12:00:00Z');
vi.setSystemTime(at);
inspect.mockResolvedValue({ ok: true, state: 'running' });
let release!: (result: Awaited<ReturnType<typeof rcon>>) => void;
rcon.mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
const pending = fanoutStatus();
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
vi.setSystemTime(at + 10_000);
db.prepare("UPDATE servers SET last_checked_at=?,last_error='timeout:manual failure' WHERE name='managed'").run(new Date().toISOString());
release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old reply',
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
await pending;
expect(getServer('managed')).toMatchObject({ last_error: 'timeout:manual failure', last_seen_at: null, last_checked_at: new Date(at + 10_000).toISOString() });
});
it('refreshes a ten-server fleet while one check is slow, without duplicating or exceeding four checks', async () => {
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
@@ -391,6 +436,7 @@ describe('database fleet and container readiness', () => {
it.each(['unknown', undefined, 'throw'])('automatically recovers through RCON when inspection returns %s', async state => {
const { inspect, rcon, fanoutStatus, getServer } = await setup();
vi.useFakeTimers({ toFake: ['Date'] });
const { GET } = await import('../../src/routes/api/servers/+server.js');
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
if (state === 'throw') inspect.mockRejectedValue(new Error('Inspection timed out'));
@@ -409,6 +455,7 @@ describe('database fleet and container readiness', () => {
})] }));
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
vi.setSystemTime(Date.now() + 30_000);
await fanoutStatus();
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' });
});