josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Use status monitor evidence for routine watchdog health checks

Browse Source
This commit is contained in:
josh
2026-10-05 16:13:47 -04:00
parent 91f4490d34
commit 87c5cf4b4b
8 changed files with 321 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/watchdog.test.ts
+131
View File
@@ -58,6 +58,10 @@ const onlyIncident = () => { expect(incidents()).toHaveLength(1); return inciden
const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[];
function advance(ms: number) { vi.setSystemTime(Date.now() + ms); }
function down() { vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
async function pollStatus() {
const worker = await vi.importActual<typeof import('../../src/lib/server/pipeline/worker.js')>('../../src/lib/server/pipeline/worker.js');
await worker.fanoutStatus();
}
async function sustainedFailure() {
down();
await watchdog.watchdogTick(true);
@@ -68,24 +72,148 @@ async function sustainedFailure() {
}
describe('Watchdog detection and ownership', () => {
it('waits for status-monitor evidence without issuing its own routine probes', async () => {
down();
for (let minute = 0; minute < 4; minute++) {
await watchdog.watchdogTick();
advance(watchdog.WATCHDOG_SCAN_MS);
}
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
expect(incidents()).toEqual([]);
expect(operations()).toEqual([]);
});
it.each(['container', 'game'])('requests recovery from sustained %s failures collected by status polling', async kind => {
if (kind === 'container') down();
else vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
for (let minute = 0; minute < 3; minute++) {
await pollStatus();
await watchdog.watchdogTick();
if (minute < 2) {
expect(onlyIncident().failures).toBe(minute + 1);
expect(operations()).toEqual([]);
advance(watchdog.WATCHDOG_SCAN_MS);
}
}
expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(3);
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0);
});
it('does not count the same sample again or use stale data, and resets after a monitoring gap', async () => {
down();
await pollStatus();
await watchdog.watchdogTick();
const first = onlyIncident();
advance(30_000);
database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
await watchdog.watchdogTick();
advance(60_000);
await watchdog.watchdogTick();
expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at });
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(1);
advance(60_000);
await pollStatus();
await watchdog.watchdogTick();
expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
expect(operations()).toEqual([]);
});
it('uses sample times across polling offsets without counting early or losing sustained failures', async () => {
down();
await pollStatus();
advance(20_000);
await watchdog.watchdogTick();
advance(10_000);
await pollStatus();
advance(50_000);
// The first failure is stale until the monitor supplies another sample.
await pollStatus();
await watchdog.watchdogTick();
expect(onlyIncident().failures).toBe(2);
advance(40_000);
await pollStatus();
advance(20_000);
await watchdog.watchdogTick();
expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
expect(operations()).toHaveLength(1);
});
it('breaks a routine failure streak on a successful status poll between watchdog passes', async () => {
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
await pollStatus();
await watchdog.watchdogTick();
advance(30_000);
vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
await pollStatus();
advance(30_000);
await pollStatus();
await watchdog.watchdogTick();
expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
expect(operations()).toEqual([]);
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(3);
});
it('uses fresh telemetry alongside the polled container and waits for polled RCON when telemetry expires', async () => {
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
saveTelemetry({ ...gameEvent('heartbeat', 17, 4, server), receivedAt: new Date().toISOString() });
await pollStatus();
await watchdog.watchdogTick();
expect(incidents()).toEqual([]);
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
advance(watchdog.WATCHDOG_SCAN_MS);
await watchdog.watchdogTick();
expect(incidents()).toEqual([]);
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
await pollStatus();
advance(30_000);
database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
await watchdog.watchdogTick();
expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open' });
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(1);
});
it('starts a new failure window for a replacement registration', async () => {
down();
await pollStatus();
await watchdog.watchdogTick();
advance(watchdog.WATCHDOG_SCAN_MS);
await pollStatus();
await watchdog.watchdogTick();
expect(onlyIncident().failures).toBe(2);
advance(watchdog.WATCHDOG_SCAN_MS);
database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server);
await pollStatus();
await watchdog.watchdogTick();
expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
expect(operations()).toEqual([]);
});
it('scans files every five minutes while retaining minute health checks and immediate manual scans', async () => {
const drift = await import('../../src/lib/server/fleet/drift.js');
const scan = vi.spyOn(drift, 'prepareDriftCampaign');
await pollStatus();
await watchdog.watchdogTick();
expect(scan).toHaveBeenCalledTimes(1);
for (let minute = 1; minute < 5; minute++) {
advance(watchdog.WATCHDOG_SCAN_MS);
await pollStatus();
await watchdog.watchdogTick();
}
expect(scan).toHaveBeenCalledTimes(1);
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(5);
advance(watchdog.WATCHDOG_SCAN_MS);
await pollStatus();
await watchdog.watchdogTick();
expect(scan).toHaveBeenCalledTimes(2);
await watchdog.watchdogTick(true);
expect(scan).toHaveBeenCalledTimes(3);
advance(watchdog.WATCHDOG_SCAN_MS);
database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server);
await pollStatus();
await watchdog.watchdogTick();
expect(scan).toHaveBeenCalledTimes(4);
scan.mockRestore();
@@ -94,9 +222,11 @@ describe('Watchdog detection and ownership', () => {
it('retries failed file scans on the next health scan instead of retaining successful freshness', async () => {
const drift = await import('../../src/lib/server/fleet/drift.js');
const scan = vi.spyOn(drift, 'prepareDriftCampaign').mockRejectedValueOnce(new Error('Agent unavailable'));
await pollStatus();
await watchdog.watchdogTick();
expect(watchdog.watchdogOverview().scanError).toContain('Agent unavailable');
advance(watchdog.WATCHDOG_SCAN_MS);
await pollStatus();
await watchdog.watchdogTick();
expect(scan).toHaveBeenCalledTimes(2);
expect(watchdog.watchdogOverview().scanError).toBeNull();
@@ -112,6 +242,7 @@ describe('Watchdog detection and ownership', () => {
expect(onlyIncident().failures).toBe(1);
expect(operations()).toEqual([]);
advance(watchdog.WATCHDOG_SCAN_MS);
await pollStatus();
await watchdog.watchdogTick();
expect(onlyIncident().failures).toBe(2);
advance(watchdog.WATCHDOG_SCAN_MS - 1);