josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix transient health checks and avoid unnecessary recovery

Browse Source
This commit is contained in:
josh
2026-10-02 13:06:18 -04:00
parent 221c4c10cf
commit 65d3a99846
16 changed files with 842 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/watchdog.test.ts
+172
View File
@@ -153,6 +153,105 @@ describe('Watchdog detection and ownership', () => {
expect(operations()).toEqual([]);
});
it.each(['no-config', 'no-port', 'bad-reply', 'no-players'] as const)('keeps %s RCON failures observational', async reason => {
vi.mocked(rcon.rconStatusResult).mockResolvedValue({ ...missingRcon, reason });
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
expect(onlyIncident()).toMatchObject({ state: 'open' });
expect(operations()).toEqual([]);
});
it('breaks the sustained failure window on a successful probe between watchdog samples', async () => {
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
await watchdog.watchdogTick(true);
advance(watchdog.WATCHDOG_SCAN_MS);
await watchdog.watchdogTick(true);
advance(20_000);
const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
await POST({ params: { name: server } } as Parameters<typeof POST>[0]);
advance(40_000);
await watchdog.watchdogTick(true);
expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open', first_seen_at: new Date().toISOString() });
expect(operations()).toEqual([]);
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
expect(operations()).toEqual([]);
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
});
it.each([0, 2])('cancels queued health recovery when fresh RCON responds with %i players', async players => {
const row = await sustainedFailure();
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(players);
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' })]);
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
expect(komodo.komodoRestart).not.toHaveBeenCalled();
});
it('holds queued health recovery when container readiness becomes unknown', async () => {
await sustainedFailure();
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
expect(operations()[0].state).toBe('waiting');
expect(komodo.komodoRestart).not.toHaveBeenCalled();
});
it('does not apply a recovered occupancy probe after its operation is replaced', async () => {
const row = await sustainedFailure();
const restart = operations()[0];
const { cancelProvisioning, requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
vi.mocked(rcon.rconGetPlayerCount).mockImplementationOnce(async () => {
expect(cancelProvisioning(restart.id)).toBe(true);
expect(requestLifecycle(server, 'stop').ok).toBe(true);
return 0;
});
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' }),
expect.objectContaining({ action: 'stop', state: 'waiting' })]);
expect(komodo.komodoRestart).not.toHaveBeenCalled();
});
it('cancels obsolete health recovery so a later queued maintenance restart can proceed', async () => {
const row = await sustainedFailure();
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const job = database.db().prepare('INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)')
.run(server, new Date().toISOString()).lastInsertRowid;
const maintenance = requestRestart(server, { kind: 'maintenance', id: String(job) });
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
expect(operations()).toEqual([expect.objectContaining({ state: 'cancelled' }),
expect.objectContaining({ id: maintenance.id, state: 'awaiting-health' })]);
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
});
it.each(['restart', 'rebuild'] as const)('preserves an operator request adopting a queued health %s', async action => {
await sustainedFailure();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
if (action === 'rebuild') {
await processProvisioning();
advance(watchdog.WATCHDOG_RECOVERY_MS);
await watchdog.watchdogTick(true);
}
const queued = operations().at(-1)!;
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
expect(requestLifecycle(server, action)).toEqual({ ok: true, deploymentId: queued.id });
expect(database.db().prepare('SELECT triggered_by FROM server_deployments WHERE id = ?').get(queued.id)).toEqual({ triggered_by: 'ui' });
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
if (action === 'rebuild') vi.mocked(komodo.komodoDestroyDeployment).mockImplementationOnce(async () => { down(); return { ok: true }; });
await processProvisioning();
expect(operations().at(-1)).toMatchObject({ id: queued.id, action, state: action === 'restart' ? 'awaiting-health' : 'queued' });
if (action === 'restart') expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
else expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
});
it('preserves intentional stop intent after completion and database reopening', async () => {
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
const stop = requestLifecycle(server, 'stop');
@@ -241,6 +340,79 @@ describe('Watchdog detection and ownership', () => {
});
describe('Watchdog recovery persistence', () => {
it('verifies recovery at the deadline before deciding whether to rebuild', async () => {
const row = await sustainedFailure();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
advance(watchdog.WATCHDOG_RECOVERY_MS);
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
await watchdog.watchdogTick(true);
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'done' })]);
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
.toEqual({ healthy_at: new Date().toISOString() });
});
it('defers escalation while current container health is unknown', async () => {
await sustainedFailure();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
advance(watchdog.WATCHDOG_RECOVERY_MS);
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
await watchdog.watchdogTick(true);
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'awaiting-health' })]);
});
it('keeps failed submission history when fresh evidence later releases recovery', async () => {
const row = await sustainedFailure();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
vi.mocked(komodo.komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Reply lost' });
await processProvisioning();
advance(watchdog.WATCHDOG_RECOVERY_MS);
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
await watchdog.watchdogTick(true);
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'failed' })]);
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
.toEqual({ healthy_at: new Date().toISOString() });
expect(watchdog.watchdogDetail(row.id)?.actions[0].operation_error).toContain('Reply lost');
});
it('discards escalation evidence if another lifecycle submission appears during its probe', async () => {
const row = await sustainedFailure();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
advance(watchdog.WATCHDOG_RECOVERY_MS);
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
vi.mocked(rcon.rconStatusResult).mockImplementationOnce(async () => {
database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('newer', ?, 'restart', 'done', 'Complete', ?, ?, 'manual')`).run(server, new Date().toISOString(), new Date().toISOString());
return healthyRcon;
});
await watchdog.watchdogTick(true);
expect(operations()[0]).toMatchObject({ action: 'restart', state: 'awaiting-health' });
expect(operations().some(op => op.action === 'rebuild')).toBe(false);
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
});
it('cancels a queued health rebuild and releases its transferred slot when the game recovers', async () => {
const row = await sustainedFailure();
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
advance(watchdog.WATCHDOG_RECOVERY_MS);
await watchdog.watchdogTick(true);
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
const rebuild = operations()[1];
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
await processProvisioning();
expect(operations()).toEqual([expect.objectContaining({ state: 'failed' }), expect.objectContaining({ state: 'cancelled' })]);
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
expect(database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
.toEqual({ operation_id: rebuild.id, healthy_at: new Date().toISOString() });
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
});
it('preserves the original finding as deleted history after an intentional successful destroy', async () => {
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');