Share recent health observations between background monitors
This commit is contained in:
@@ -0,0 +1,259 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn() }));
|
||||
|
||||
const server = 'managed';
|
||||
const now = Date.parse('2026-10-05T16:00:00Z');
|
||||
const healthy = { ok: true as const, state: 'running' };
|
||||
const playing = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
||||
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } };
|
||||
let fixture: Fixture;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now);
|
||||
});
|
||||
afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.restoreAllMocks(); });
|
||||
|
||||
async function setup() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const database = db();
|
||||
database.prepare('INSERT INTO servers(name,komodo_name,created_at,game_port) VALUES(?,?,?,28960)')
|
||||
.run(server, server, new Date(now).toISOString());
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const inspect = vi.mocked(komodoInspectHealthy).mockResolvedValue(healthy);
|
||||
const rcon = vi.mocked(rconStatusResult).mockResolvedValue(playing);
|
||||
return { database, inspect, rcon, ...await import('../../src/lib/server/fleet/health-observations.js') };
|
||||
}
|
||||
|
||||
async function native(api: Awaited<ReturnType<typeof setup>>) {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.createHost({ name: 'Monitoring host' });
|
||||
hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 });
|
||||
const instanceId = randomUUID();
|
||||
api.database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES(?,?,?,'library','image',28960,?)`).run(instanceId, server, host.id, new Date(now).toISOString());
|
||||
return { host, instanceId, hosts };
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>(done => { resolve = done; });
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
describe('shared monitoring observations', () => {
|
||||
it.each(['inspect', 'rcon'] as const)('reuses %s results until 30 seconds after request start without refreshing their timestamp', async kind => {
|
||||
const api = await setup();
|
||||
const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring;
|
||||
const request = kind === 'inspect' ? api.inspect : api.rcon;
|
||||
const first = await call(server);
|
||||
expect(first.observedAt).toBe(new Date(now).toISOString());
|
||||
vi.setSystemTime(now + 29_999);
|
||||
expect(await call(server)).toEqual(first);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
vi.setSystemTime(now + 30_000);
|
||||
expect((await call(server)).observedAt).toBe(new Date(now + 30_000).toISOString());
|
||||
expect(request).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('shares overlapping monitoring requests, but keeps inspection and RCON separate', async () => {
|
||||
const api = await setup();
|
||||
const pending = deferred<typeof healthy>();
|
||||
api.inspect.mockReturnValueOnce(pending.promise);
|
||||
const observations = [api.inspectForMonitoring(server), api.inspectForMonitoring(server)];
|
||||
await api.rconForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(1);
|
||||
expect(api.rcon).toHaveBeenCalledTimes(1);
|
||||
vi.setSystemTime(now + 5000);
|
||||
pending.resolve(healthy);
|
||||
expect(await Promise.all(observations)).toEqual(Array(2).fill({ result: healthy, observedAt: new Date(now + 5000).toISOString() }));
|
||||
});
|
||||
|
||||
it('timestamps a slow result on completion without making it reusable after its request expires', async () => {
|
||||
const api = await setup();
|
||||
const pending = deferred<typeof healthy>();
|
||||
api.inspect.mockReturnValueOnce(pending.promise);
|
||||
const observation = api.inspectForMonitoring(server);
|
||||
await Promise.resolve();
|
||||
vi.setSystemTime(now + 31_000);
|
||||
pending.resolve(healthy);
|
||||
expect(await observation).toEqual({ result: healthy, observedAt: new Date(now + 31_000).toISOString() });
|
||||
await api.inspectForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('reuses the saved completion timestamp while expiring from the earlier request start', async () => {
|
||||
const api = await setup();
|
||||
const pending = deferred<typeof healthy>();
|
||||
api.inspect.mockReturnValueOnce(pending.promise);
|
||||
const observation = api.inspectForMonitoring(server);
|
||||
await Promise.resolve();
|
||||
vi.setSystemTime(now + 20_000);
|
||||
pending.resolve(healthy);
|
||||
const completed = await observation;
|
||||
api.database.prepare('UPDATE servers SET container_checked_at=? WHERE name=?').run(completed.observedAt, server);
|
||||
vi.setSystemTime(now + 25_000);
|
||||
expect(await api.inspectForMonitoring(server)).toEqual(completed);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(1);
|
||||
vi.setSystemTime(now + 30_000);
|
||||
await api.inspectForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not reuse observations from a future clock', async () => {
|
||||
const api = await setup();
|
||||
await api.inspectForMonitoring(server);
|
||||
vi.setSystemTime(now - 1);
|
||||
await api.inspectForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not cache failed results or rejected requests', async () => {
|
||||
const api = await setup();
|
||||
api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' });
|
||||
expect((await api.inspectForMonitoring(server)).result.ok).toBe(false);
|
||||
await api.inspectForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
api.rcon.mockRejectedValueOnce(new Error('Unavailable'));
|
||||
await expect(api.rconForMonitoring(server)).rejects.toThrow('Unavailable');
|
||||
await api.rconForMonitoring(server);
|
||||
expect(api.rcon).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('forced checks bypass completed results and evict old success when the forced result fails', async () => {
|
||||
const api = await setup();
|
||||
await api.inspectForMonitoring(server);
|
||||
api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' });
|
||||
expect((await api.inspectForMonitoring(server, true)).result.ok).toBe(false);
|
||||
await api.inspectForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('forced checks bypass wrapper overlap and an older reply cannot replace the newer cache entry', async () => {
|
||||
const api = await setup();
|
||||
const old = deferred<typeof healthy>();
|
||||
api.inspect.mockReturnValueOnce(old.promise);
|
||||
const first = api.inspectForMonitoring(server);
|
||||
await Promise.resolve();
|
||||
vi.setSystemTime(now + 1000);
|
||||
const latest = await api.inspectForMonitoring(server, true);
|
||||
old.resolve(healthy); await first;
|
||||
expect(await api.inspectForMonitoring(server)).toEqual(latest);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each(['registration', 'lifecycle', 'desired state', 'stop'] as const)('invalidates cached observations after a %s change', async change => {
|
||||
const api = await setup();
|
||||
await api.inspectForMonitoring(server);
|
||||
if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server);
|
||||
if (change === 'lifecycle') api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString());
|
||||
if (change === 'desired state') api.database.prepare("UPDATE servers SET desired_state='stopped' WHERE name=?").run(server);
|
||||
if (change === 'stop') api.database.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
|
||||
VALUES('stop',?,'stop','awaiting-health','Stopping',?,?,'ui')`).run(server, new Date(now).toISOString(), new Date(now + 1).toISOString());
|
||||
await api.inspectForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('invalidates integration changes and accepts legacy port discovery by the successful RCON probe', async () => {
|
||||
const api = await setup();
|
||||
await api.inspectForMonitoring(server);
|
||||
api.database.prepare("UPDATE servers SET komodo_name='replacement' WHERE name=?").run(server);
|
||||
await api.inspectForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
api.database.prepare('UPDATE servers SET game_port=NULL WHERE name=?').run(server);
|
||||
api.rcon.mockImplementationOnce(async () => {
|
||||
api.database.prepare('UPDATE servers SET game_port=28960 WHERE name=?').run(server);
|
||||
return playing;
|
||||
});
|
||||
expect((await api.rconForMonitoring(server)).result).toEqual(playing);
|
||||
await api.rconForMonitoring(server);
|
||||
expect(api.rcon).toHaveBeenCalledTimes(1);
|
||||
api.database.prepare('UPDATE servers SET game_port=28961 WHERE name=?').run(server);
|
||||
await api.rconForMonitoring(server);
|
||||
expect(api.rcon).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each(['inspect', 'rcon'] as const)('does not reuse %s after a newer manual observation', async kind => {
|
||||
const api = await setup();
|
||||
const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring;
|
||||
const request = kind === 'inspect' ? api.inspect : api.rcon;
|
||||
const field = kind === 'inspect' ? 'container_checked_at' : 'last_checked_at';
|
||||
await call(server);
|
||||
vi.setSystemTime(now + 1000);
|
||||
api.database.prepare(`UPDATE servers SET ${field}=? WHERE name=?`).run(new Date(now + 1000).toISOString(), server);
|
||||
await call(server);
|
||||
expect(request).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('rejects a previous generation completing after replacement without discarding the new generation cache', async () => {
|
||||
const api = await setup();
|
||||
const { instanceId } = await native(api);
|
||||
const pending = deferred<typeof healthy>();
|
||||
api.inspect.mockReturnValueOnce(pending.promise);
|
||||
const old = api.inspectForMonitoring(server);
|
||||
const rejected = expect(old).rejects.toThrow('Server changed');
|
||||
await Promise.resolve();
|
||||
api.database.prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId);
|
||||
const current = await api.inspectForMonitoring(server);
|
||||
pending.resolve(healthy); await rejected;
|
||||
expect(await api.inspectForMonitoring(server)).toEqual(current);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('rejects a result when a lifecycle request arrives while its probe is in flight', async () => {
|
||||
const api = await setup();
|
||||
const pending = deferred<typeof playing>();
|
||||
api.rcon.mockReturnValueOnce(pending.promise);
|
||||
const old = api.rconForMonitoring(server);
|
||||
const rejected = expect(old).rejects.toThrow('Server changed');
|
||||
await Promise.resolve();
|
||||
api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString());
|
||||
pending.resolve(playing); await rejected;
|
||||
await api.rconForMonitoring(server);
|
||||
expect(api.rcon).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('discards a slow result when a newer manual observation was saved during the request', async () => {
|
||||
const api = await setup();
|
||||
const pending = deferred<typeof playing>();
|
||||
api.rcon.mockReturnValueOnce(pending.promise);
|
||||
const old = api.rconForMonitoring(server);
|
||||
const rejected = expect(old).rejects.toBeInstanceOf(api.MonitoringChangedError);
|
||||
await Promise.resolve();
|
||||
vi.setSystemTime(now + 1000);
|
||||
api.database.prepare('UPDATE servers SET last_checked_at=? WHERE name=?').run(new Date().toISOString(), server);
|
||||
pending.resolve(playing); await rejected;
|
||||
await api.rconForMonitoring(server);
|
||||
expect(api.rcon).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('discards cached native success while its host is offline and probes again after reconnecting', async () => {
|
||||
const api = await setup();
|
||||
const { host, hosts } = await native(api);
|
||||
await api.inspectForMonitoring(server);
|
||||
api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id);
|
||||
await expect(api.inspectForMonitoring(server)).rejects.toThrow('Host agent is offline');
|
||||
expect(api.inspect).toHaveBeenCalledTimes(1);
|
||||
hosts.heartbeat(host.id, { protocolVersion: 1 });
|
||||
await api.inspectForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('rejects an otherwise successful result if the native host went offline during the probe', async () => {
|
||||
const api = await setup();
|
||||
const { host } = await native(api);
|
||||
const pending = deferred<typeof healthy>();
|
||||
api.inspect.mockReturnValueOnce(pending.promise);
|
||||
const observation = api.inspectForMonitoring(server);
|
||||
const rejected = expect(observation).rejects.toThrow('Host agent is offline');
|
||||
await Promise.resolve();
|
||||
api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id);
|
||||
pending.resolve(healthy); await rejected;
|
||||
});
|
||||
});
|
||||
@@ -30,6 +30,51 @@ async function setup() {
|
||||
}
|
||||
|
||||
describe('database fleet and container readiness', () => {
|
||||
it('shares routine observations without renewing their timestamps and keeps manual probes fresh', async () => {
|
||||
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
const { inspectForMonitoring, rconForMonitoring } = await import('../../src/lib/server/fleet/health-observations.js');
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
const at = Date.parse('2026-10-05T12:00:00Z');
|
||||
vi.setSystemTime(at);
|
||||
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
||||
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
||||
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
||||
await inspectForMonitoring('managed');
|
||||
await rconForMonitoring('managed');
|
||||
vi.setSystemTime(at + 10_000);
|
||||
await fanoutStatus();
|
||||
expect(inspect).toHaveBeenCalledTimes(1);
|
||||
expect(rcon).toHaveBeenCalledTimes(1);
|
||||
expect(getServer('managed')).toMatchObject({ container_checked_at: new Date(at).toISOString(), last_seen_at: new Date(at).toISOString() });
|
||||
|
||||
const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
|
||||
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'manual failure' });
|
||||
await POST({ params: { name: 'managed' } } as never);
|
||||
expect(rcon).toHaveBeenCalledTimes(2);
|
||||
vi.setSystemTime(at + 20_000);
|
||||
await fanoutStatus();
|
||||
expect(rcon).toHaveBeenCalledTimes(3);
|
||||
expect(getServer('managed')).toMatchObject({ last_error: 'timeout:manual failure', last_seen_at: new Date(at).toISOString() });
|
||||
});
|
||||
|
||||
it('does not overwrite a newer manual observation with an earlier background reply', async () => {
|
||||
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
const at = Date.parse('2026-10-05T12:00:00Z');
|
||||
vi.setSystemTime(at);
|
||||
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
||||
let release!: (result: Awaited<ReturnType<typeof rcon>>) => void;
|
||||
rcon.mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
||||
const pending = fanoutStatus();
|
||||
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
|
||||
vi.setSystemTime(at + 10_000);
|
||||
db.prepare("UPDATE servers SET last_checked_at=?,last_error='timeout:manual failure' WHERE name='managed'").run(new Date().toISOString());
|
||||
release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old reply',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
||||
await pending;
|
||||
expect(getServer('managed')).toMatchObject({ last_error: 'timeout:manual failure', last_seen_at: null, last_checked_at: new Date(at + 10_000).toISOString() });
|
||||
});
|
||||
|
||||
it('refreshes a ten-server fleet while one check is slow, without duplicating or exceeding four checks', async () => {
|
||||
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
||||
@@ -391,6 +436,7 @@ describe('database fleet and container readiness', () => {
|
||||
|
||||
it.each(['unknown', undefined, 'throw'])('automatically recovers through RCON when inspection returns %s', async state => {
|
||||
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
||||
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
||||
if (state === 'throw') inspect.mockRejectedValue(new Error('Inspection timed out'));
|
||||
@@ -409,6 +455,7 @@ describe('database fleet and container readiness', () => {
|
||||
})] }));
|
||||
|
||||
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
|
||||
vi.setSystemTime(Date.now() + 30_000);
|
||||
await fanoutStatus();
|
||||
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' });
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user