Use status monitor evidence for routine watchdog health checks

This commit is contained in:
2026-10-05 16:13:47 -04:00
parent 91f4490d34
commit 87c5cf4b4b
8 changed files with 321 additions and 22 deletions
+68
View File
@@ -48,6 +48,74 @@ function deferred<T>() {
}
describe('shared monitoring observations', () => {
it('retains failures for watchdog consumption without caching them as reusable probes', async () => {
const api = await setup();
api.rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'No reply' });
const publish = api.monitoringStatusWriter(server);
publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server));
expect(api.monitoringStatus(server)).toMatchObject({
container: { observedAt: new Date(now).toISOString(), result: healthy },
game: { observedAt: new Date(now).toISOString(), result: { ok: false, reason: 'timeout' } }
});
vi.setSystemTime(now + 40_000);
expect(api.monitoringStatus(server)?.game?.observedAt).toBe(new Date(now).toISOString());
expect(api.inspect).toHaveBeenCalledTimes(1);
expect(api.rcon).toHaveBeenCalledTimes(1);
vi.setSystemTime(now + 45_001);
expect(api.monitoringStatus(server)).toBeNull();
vi.setSystemTime(now - 1);
expect(api.monitoringStatus(server)).toBeNull();
});
it.each(['registration', 'generation', 'lifecycle', 'endpoint', 'manual', 'offline'] as const)('invalidates a completed status snapshot after %s changes', async change => {
const api = await setup();
const host = await native(api);
const publish = api.monitoringStatusWriter(server);
publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server));
expect(api.monitoringStatus(server)).not.toBeNull();
if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server);
if (change === 'generation') api.database.prepare('UPDATE native_instances SET generation=2').run();
if (change === 'lifecycle') api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString());
if (change === 'endpoint') api.database.prepare('UPDATE native_instances SET port=28961').run();
if (change === 'manual') api.database.prepare('UPDATE servers SET container_checked_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server);
if (change === 'offline') api.database.prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run(new Date(now - 300_000).toISOString(), host.host.id);
expect(api.monitoringStatus(server)).toBeNull();
});
it('does not publish a status pass across an instance replacement', async () => {
const api = await setup();
await native(api);
const publish = api.monitoringStatusWriter(server);
const container = await api.inspectForMonitoring(server);
const game = await api.rconForMonitoring(server);
api.database.prepare('UPDATE native_instances SET generation=2').run();
publish(container, game);
expect(api.monitoringStatus(server)).toBeNull();
});
it.each(['inspect', 'rcon'] as const)('invalidates the routine snapshot when a forced %s check starts', async kind => {
const api = await setup();
const publish = api.monitoringStatusWriter(server);
const container = await api.inspectForMonitoring(server);
const game = await api.rconForMonitoring(server);
publish(container, game);
expect(api.monitoringStatus(server)).not.toBeNull();
await (kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring)(server, true);
publish(container, game);
expect(api.monitoringStatus(server)).toBeNull();
});
it('excludes old game evidence even when the container sample is fresh', async () => {
const api = await setup();
const game = await api.rconForMonitoring(server);
vi.setSystemTime(now + 46_000);
const publish = api.monitoringStatusWriter(server);
publish(await api.inspectForMonitoring(server), game);
expect(api.monitoringStatus(server)?.container.result).toEqual(healthy);
expect(api.monitoringStatus(server)?.game).toBeUndefined();
expect(api.rcon).toHaveBeenCalledTimes(1);
});
it.each(['inspect', 'rcon'] as const)('reuses %s results until 30 seconds after request start without refreshing their timestamp', async kind => {
const api = await setup();
const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring;