385 lines
17 KiB
TypeScript
385 lines
17 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn() }));
|
|
|
|
const server = 'managed';
|
|
const now = Date.parse('2026-10-05T16:00:00Z');
|
|
const healthy = { ok: true as const, state: 'running' };
|
|
const playing = {
|
|
ok: true as const,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
};
|
|
let fixture: Fixture;
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.clearAllMocks();
|
|
fixture = await newFixture([]);
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(now);
|
|
});
|
|
afterEach(async () => {
|
|
vi.useRealTimers();
|
|
await fixture.cleanup();
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
async function setup() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const database = db();
|
|
database
|
|
.prepare('INSERT INTO servers(name,created_at,game_port) VALUES(?,?,28960)')
|
|
.run(server, new Date(now).toISOString());
|
|
vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth');
|
|
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const inspect = vi.mocked(nativeHealth).mockResolvedValue(healthy);
|
|
const rcon = vi.mocked(rconStatusResult).mockResolvedValue(playing);
|
|
return {
|
|
database,
|
|
inspect,
|
|
rcon,
|
|
...(await import('../../src/lib/server/fleet/health-observations.js'))
|
|
};
|
|
}
|
|
|
|
async function native(api: Awaited<ReturnType<typeof setup>>) {
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Monitoring host' });
|
|
hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 });
|
|
const instanceId = randomUUID();
|
|
api.database
|
|
.prepare(
|
|
`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
|
VALUES(?,?,?,'library','image',28960,?)`
|
|
)
|
|
.run(instanceId, server, host.id, new Date(now).toISOString());
|
|
return { host, instanceId, hosts };
|
|
}
|
|
|
|
function deferred<T>() {
|
|
let resolve!: (value: T) => void;
|
|
const promise = new Promise<T>((done) => {
|
|
resolve = done;
|
|
});
|
|
return { promise, resolve };
|
|
}
|
|
|
|
describe('shared monitoring observations', () => {
|
|
it('retains failures for watchdog consumption without caching them as reusable probes', async () => {
|
|
const api = await setup();
|
|
api.rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'timeout',
|
|
detail: 'No reply'
|
|
});
|
|
const publish = api.monitoringStatusWriter(server);
|
|
publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server));
|
|
expect(api.monitoringStatus(server)).toMatchObject({
|
|
container: { observedAt: new Date(now).toISOString(), result: healthy },
|
|
game: {
|
|
observedAt: new Date(now).toISOString(),
|
|
result: { ok: false, reason: 'timeout' }
|
|
}
|
|
});
|
|
vi.setSystemTime(now + 40_000);
|
|
expect(api.monitoringStatus(server)?.game?.observedAt).toBe(new Date(now).toISOString());
|
|
expect(api.inspect).toHaveBeenCalledTimes(1);
|
|
expect(api.rcon).toHaveBeenCalledTimes(1);
|
|
vi.setSystemTime(now + 90_001);
|
|
expect(api.monitoringStatus(server)).toBeNull();
|
|
vi.setSystemTime(now - 1);
|
|
expect(api.monitoringStatus(server)).toBeNull();
|
|
});
|
|
|
|
it.each(['registration', 'generation', 'lifecycle', 'endpoint', 'manual', 'offline'] as const)(
|
|
'invalidates a completed status snapshot after %s changes',
|
|
async (change) => {
|
|
const api = await setup();
|
|
const host = await native(api);
|
|
const publish = api.monitoringStatusWriter(server);
|
|
publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server));
|
|
expect(api.monitoringStatus(server)).not.toBeNull();
|
|
if (change === 'registration')
|
|
api.database
|
|
.prepare('UPDATE servers SET created_at=? WHERE name=?')
|
|
.run(new Date(now + 1).toISOString(), server);
|
|
if (change === 'generation')
|
|
api.database.prepare('UPDATE native_instances SET generation=2').run();
|
|
if (change === 'lifecycle')
|
|
api.database
|
|
.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)')
|
|
.run(server, new Date(now + 1).toISOString());
|
|
if (change === 'endpoint')
|
|
api.database.prepare('UPDATE native_instances SET port=28961').run();
|
|
if (change === 'manual')
|
|
api.database
|
|
.prepare('UPDATE servers SET container_checked_at=? WHERE name=?')
|
|
.run(new Date(now + 1).toISOString(), server);
|
|
if (change === 'offline')
|
|
api.database
|
|
.prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?')
|
|
.run(new Date(now - 300_000).toISOString(), host.host.id);
|
|
expect(api.monitoringStatus(server)).toBeNull();
|
|
}
|
|
);
|
|
|
|
it('does not publish a status pass across an instance replacement', async () => {
|
|
const api = await setup();
|
|
await native(api);
|
|
const publish = api.monitoringStatusWriter(server);
|
|
const container = await api.inspectForMonitoring(server);
|
|
const game = await api.rconForMonitoring(server);
|
|
api.database.prepare('UPDATE native_instances SET generation=2').run();
|
|
publish(container, game);
|
|
expect(api.monitoringStatus(server)).toBeNull();
|
|
});
|
|
|
|
it.each(['inspect', 'rcon'] as const)(
|
|
'invalidates the routine snapshot when a forced %s check starts',
|
|
async (kind) => {
|
|
const api = await setup();
|
|
const publish = api.monitoringStatusWriter(server);
|
|
const container = await api.inspectForMonitoring(server);
|
|
const game = await api.rconForMonitoring(server);
|
|
publish(container, game);
|
|
expect(api.monitoringStatus(server)).not.toBeNull();
|
|
await (kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring)(
|
|
server,
|
|
true
|
|
);
|
|
publish(container, game);
|
|
expect(api.monitoringStatus(server)).toBeNull();
|
|
}
|
|
);
|
|
|
|
it('excludes old game evidence even when the container sample is fresh', async () => {
|
|
const api = await setup();
|
|
const game = await api.rconForMonitoring(server);
|
|
vi.setSystemTime(now + 91_000);
|
|
const publish = api.monitoringStatusWriter(server);
|
|
publish(await api.inspectForMonitoring(server), game);
|
|
expect(api.monitoringStatus(server)?.container.result).toEqual(healthy);
|
|
expect(api.monitoringStatus(server)?.game).toBeUndefined();
|
|
expect(api.rcon).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it.each(['inspect', 'rcon'] as const)(
|
|
'reuses %s results until 30 seconds after request start without refreshing their timestamp',
|
|
async (kind) => {
|
|
const api = await setup();
|
|
const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring;
|
|
const request = kind === 'inspect' ? api.inspect : api.rcon;
|
|
const first = await call(server);
|
|
expect(first.observedAt).toBe(new Date(now).toISOString());
|
|
vi.setSystemTime(now + 29_999);
|
|
expect(await call(server)).toEqual(first);
|
|
expect(request).toHaveBeenCalledTimes(1);
|
|
vi.setSystemTime(now + 30_000);
|
|
expect((await call(server)).observedAt).toBe(new Date(now + 30_000).toISOString());
|
|
expect(request).toHaveBeenCalledTimes(2);
|
|
}
|
|
);
|
|
|
|
it('shares overlapping monitoring requests, but keeps inspection and RCON separate', async () => {
|
|
const api = await setup();
|
|
const pending = deferred<typeof healthy>();
|
|
api.inspect.mockReturnValueOnce(pending.promise);
|
|
const observations = [api.inspectForMonitoring(server), api.inspectForMonitoring(server)];
|
|
await api.rconForMonitoring(server);
|
|
expect(api.inspect).toHaveBeenCalledTimes(1);
|
|
expect(api.rcon).toHaveBeenCalledTimes(1);
|
|
vi.setSystemTime(now + 5000);
|
|
pending.resolve(healthy);
|
|
expect(await Promise.all(observations)).toEqual(
|
|
Array(2).fill({ result: healthy, observedAt: new Date(now + 5000).toISOString() })
|
|
);
|
|
});
|
|
|
|
it('timestamps a slow result on completion without making it reusable after its request expires', async () => {
|
|
const api = await setup();
|
|
const pending = deferred<typeof healthy>();
|
|
api.inspect.mockReturnValueOnce(pending.promise);
|
|
const observation = api.inspectForMonitoring(server);
|
|
await Promise.resolve();
|
|
vi.setSystemTime(now + 31_000);
|
|
pending.resolve(healthy);
|
|
expect(await observation).toEqual({
|
|
result: healthy,
|
|
observedAt: new Date(now + 31_000).toISOString()
|
|
});
|
|
await api.inspectForMonitoring(server);
|
|
expect(api.inspect).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('reuses the saved completion timestamp while expiring from the earlier request start', async () => {
|
|
const api = await setup();
|
|
const pending = deferred<typeof healthy>();
|
|
api.inspect.mockReturnValueOnce(pending.promise);
|
|
const observation = api.inspectForMonitoring(server);
|
|
await Promise.resolve();
|
|
vi.setSystemTime(now + 20_000);
|
|
pending.resolve(healthy);
|
|
const completed = await observation;
|
|
api.database
|
|
.prepare('UPDATE servers SET container_checked_at=? WHERE name=?')
|
|
.run(completed.observedAt, server);
|
|
vi.setSystemTime(now + 25_000);
|
|
expect(await api.inspectForMonitoring(server)).toEqual(completed);
|
|
expect(api.inspect).toHaveBeenCalledTimes(1);
|
|
vi.setSystemTime(now + 30_000);
|
|
await api.inspectForMonitoring(server);
|
|
expect(api.inspect).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('does not reuse observations from a future clock', async () => {
|
|
const api = await setup();
|
|
await api.inspectForMonitoring(server);
|
|
vi.setSystemTime(now - 1);
|
|
await api.inspectForMonitoring(server);
|
|
expect(api.inspect).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('does not cache failed results or rejected requests', async () => {
|
|
const api = await setup();
|
|
api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' });
|
|
expect((await api.inspectForMonitoring(server)).result.ok).toBe(false);
|
|
await api.inspectForMonitoring(server);
|
|
expect(api.inspect).toHaveBeenCalledTimes(2);
|
|
api.rcon.mockRejectedValueOnce(new Error('Unavailable'));
|
|
await expect(api.rconForMonitoring(server)).rejects.toThrow('Unavailable');
|
|
await api.rconForMonitoring(server);
|
|
expect(api.rcon).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('forced checks bypass completed results and evict old success when the forced result fails', async () => {
|
|
const api = await setup();
|
|
await api.inspectForMonitoring(server);
|
|
api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' });
|
|
expect((await api.inspectForMonitoring(server, true)).result.ok).toBe(false);
|
|
await api.inspectForMonitoring(server);
|
|
expect(api.inspect).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it('forced checks bypass wrapper overlap and an older reply cannot replace the newer cache entry', async () => {
|
|
const api = await setup();
|
|
const old = deferred<typeof healthy>();
|
|
api.inspect.mockReturnValueOnce(old.promise);
|
|
const first = api.inspectForMonitoring(server);
|
|
await Promise.resolve();
|
|
vi.setSystemTime(now + 1000);
|
|
const latest = await api.inspectForMonitoring(server, true);
|
|
old.resolve(healthy);
|
|
await first;
|
|
expect(await api.inspectForMonitoring(server)).toEqual(latest);
|
|
expect(api.inspect).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it.each(['registration', 'lifecycle', 'desired state', 'stop'] as const)(
|
|
'invalidates cached observations after a %s change',
|
|
async (change) => {
|
|
const api = await setup();
|
|
await api.inspectForMonitoring(server);
|
|
if (change === 'registration')
|
|
api.database
|
|
.prepare('UPDATE servers SET created_at=? WHERE name=?')
|
|
.run(new Date(now + 1).toISOString(), server);
|
|
if (change === 'lifecycle')
|
|
api.database
|
|
.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)')
|
|
.run(server, new Date(now + 1).toISOString());
|
|
if (change === 'desired state')
|
|
api.database
|
|
.prepare("UPDATE servers SET desired_state='stopped' WHERE name=?")
|
|
.run(server);
|
|
if (change === 'stop')
|
|
api.database
|
|
.prepare(
|
|
`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
|
|
VALUES('stop',?,'stop','awaiting-health','Stopping',?,?,'ui')`
|
|
)
|
|
.run(server, new Date(now).toISOString(), new Date(now + 1).toISOString());
|
|
await api.inspectForMonitoring(server);
|
|
expect(api.inspect).toHaveBeenCalledTimes(2);
|
|
}
|
|
);
|
|
|
|
it('rejects a previous generation completing after replacement without discarding the new generation cache', async () => {
|
|
const api = await setup();
|
|
const { instanceId } = await native(api);
|
|
const pending = deferred<typeof healthy>();
|
|
api.inspect.mockReturnValueOnce(pending.promise);
|
|
const old = api.inspectForMonitoring(server);
|
|
const rejected = expect(old).rejects.toThrow('Server changed');
|
|
await Promise.resolve();
|
|
api.database.prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId);
|
|
const current = await api.inspectForMonitoring(server);
|
|
pending.resolve(healthy);
|
|
await rejected;
|
|
expect(await api.inspectForMonitoring(server)).toEqual(current);
|
|
expect(api.inspect).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('rejects a result when a lifecycle request arrives while its probe is in flight', async () => {
|
|
const api = await setup();
|
|
const pending = deferred<typeof playing>();
|
|
api.rcon.mockReturnValueOnce(pending.promise);
|
|
const old = api.rconForMonitoring(server);
|
|
const rejected = expect(old).rejects.toThrow('Server changed');
|
|
await Promise.resolve();
|
|
api.database
|
|
.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)')
|
|
.run(server, new Date(now + 1).toISOString());
|
|
pending.resolve(playing);
|
|
await rejected;
|
|
await api.rconForMonitoring(server);
|
|
expect(api.rcon).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('discards a slow result when a newer manual observation was saved during the request', async () => {
|
|
const api = await setup();
|
|
const pending = deferred<typeof playing>();
|
|
api.rcon.mockReturnValueOnce(pending.promise);
|
|
const old = api.rconForMonitoring(server);
|
|
const rejected = expect(old).rejects.toBeInstanceOf(api.MonitoringChangedError);
|
|
await Promise.resolve();
|
|
vi.setSystemTime(now + 1000);
|
|
api.database
|
|
.prepare('UPDATE servers SET last_checked_at=? WHERE name=?')
|
|
.run(new Date().toISOString(), server);
|
|
pending.resolve(playing);
|
|
await rejected;
|
|
await api.rconForMonitoring(server);
|
|
expect(api.rcon).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('discards cached native success while its host is offline and probes again after reconnecting', async () => {
|
|
const api = await setup();
|
|
const { host, hosts } = await native(api);
|
|
await api.inspectForMonitoring(server);
|
|
api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id);
|
|
await expect(api.inspectForMonitoring(server)).rejects.toThrow('Host agent is offline');
|
|
expect(api.inspect).toHaveBeenCalledTimes(1);
|
|
hosts.heartbeat(host.id, { protocolVersion: 1 });
|
|
await api.inspectForMonitoring(server);
|
|
expect(api.inspect).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('rejects an otherwise successful result if the native host went offline during the probe', async () => {
|
|
const api = await setup();
|
|
const { host } = await native(api);
|
|
const pending = deferred<typeof healthy>();
|
|
api.inspect.mockReturnValueOnce(pending.promise);
|
|
const observation = api.inspectForMonitoring(server);
|
|
const rejected = expect(observation).rejects.toThrow('Host agent is offline');
|
|
await Promise.resolve();
|
|
api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id);
|
|
pending.resolve(healthy);
|
|
await rejected;
|
|
});
|
|
});
|