josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Relax freshness limits and tolerate transient RCON failures

Browse Source
This commit is contained in:
josh
2026-10-05 16:36:23 -04:00
parent d8b972cfc3
commit b1b3e8b958
28 changed files with 221 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/server-status.test.ts
+77 -2
View File
@@ -30,6 +30,77 @@ async function setup() {
}
describe('database fleet and container readiness', () => {
it('requires two distinct failed monitoring checks, preserves diagnostics, and resets on success', async () => {
const { inspect, rcon, fanoutStatus, getServer } = await setup();
const { GET } = await import('../../src/routes/api/servers/+server.js');
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
const snapshot = async () => (await (await GET()).json()).servers[0];
vi.useFakeTimers({ toFake: ['Date'] });
const at = Date.parse('2026-10-05T12:00:00Z');
vi.setSystemTime(at);
inspect.mockResolvedValue({ ok: true, state: 'running' });
const success = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } };
rcon.mockResolvedValue(success);
await fanoutStatus();
const events: unknown[] = [];
const unsubscribe = subscribe(event => events.push(event));
try {
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
vi.setSystemTime(at + 30_000);
await fanoutStatus();
expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:no reply' });
expect(await snapshot()).toMatchObject({ health: 'online', last_error: null,
rcon_status: { last_error: 'timeout:no reply', rcon_failures: 1 } });
expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
health: 'online', lastError: null, rconStatus: expect.objectContaining({ rcon_failures: 1, last_error: 'timeout:no reply' })
})] }));
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } });
// Reprocessing a sample with the same timestamp cannot count it twice.
await fanoutStatus();
expect(getServer('managed')?.rcon_failures).toBe(1);
vi.setSystemTime(at + 60_000);
await fanoutStatus();
expect(await snapshot()).toMatchObject({ health: 'stale', last_error: 'timeout:no reply', rcon_status: { rcon_failures: 2 } });
rcon.mockResolvedValue(success);
vi.setSystemTime(at + 90_000);
await fanoutStatus();
expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: null });
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'send-error', detail: 'unreachable' });
vi.setSystemTime(at + 120_000);
await fanoutStatus();
expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } });
vi.setSystemTime(at + 180_001);
expect(await snapshot()).toMatchObject({ health: 'stale' });
vi.setSystemTime(at + 210_001);
expect(await snapshot()).toMatchObject({ health: 'offline' });
} finally { unsubscribe(); }
});
it('adds the failure counter to an existing database without suppressing legacy errors', async () => {
const { db, getServer } = await setup();
const { closeDb } = await import('../../src/lib/server/db/index.js');
db.exec('ALTER TABLE servers DROP COLUMN rcon_failures');
closeDb();
expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: 'timeout:old process' });
});
it('does not carry a failure streak across a new lifecycle submission', async () => {
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
vi.useFakeTimers({ toFake: ['Date'] });
const at = Date.parse('2026-10-05T12:00:00Z');
vi.setSystemTime(at);
db.prepare("UPDATE servers SET rcon_failures=1,last_checked_at=?").run(new Date(at - 30_000).toISOString());
db.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
VALUES('new-start','managed','restart','done','done',?,?,'test')`).run(new Date(at - 10_000).toISOString(), new Date(at - 10_000).toISOString());
inspect.mockResolvedValue({ ok: true, state: 'running' });
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'new process' });
await fanoutStatus();
expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:new process' });
});
it('shares routine observations without renewing their timestamps and keeps manual probes fresh', async () => {
const { inspect, rcon, fanoutStatus, getServer } = await setup();
const { inspectForMonitoring, rconForMonitoring } = await import('../../src/lib/server/fleet/health-observations.js');
@@ -193,7 +264,7 @@ describe('database fleet and container readiness', () => {
const pending = fanoutStatus();
try {
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
vi.setSystemTime(now + 46_000);
vi.setSystemTime(now + 91_000);
} finally {
release({ ok: true, state: 'running' });
await pending;
@@ -444,7 +515,7 @@ describe('database fleet and container readiness', () => {
const { GET } = await import('../../src/routes/api/servers/+server.js');
inspect.mockResolvedValue({ ok: true, state: 'running' });
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 46_000).toISOString());
db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 91_000).toISOString());
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
await fanoutStatus();
@@ -515,6 +586,10 @@ describe('database fleet and container readiness', () => {
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
vi.setSystemTime(Date.now() + 30_000);
await fanoutStatus();
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null,
rcon_status: { rcon_failures: 1, last_error: 'timeout:no reply' } });
vi.setSystemTime(Date.now() + 30_000);
await fanoutStatus();
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' });
});