Fix transient health checks and avoid unnecessary recovery
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { komodoInspectHealthy } from '../../src/lib/server/integrations/komodo.js';
|
||||
|
||||
vi.mock('../../src/lib/server/config.js', () => ({
|
||||
config: {
|
||||
komodo: { address: 'http://komodo.test', apiKey: 'test', apiSecret: 'test' },
|
||||
httpTimeoutMs: 1000
|
||||
},
|
||||
komodoConfigured: () => true
|
||||
}));
|
||||
vi.mock('../../src/lib/server/fleet/servers.js', () => ({
|
||||
getServer: () => ({ komodo_type: 'deployment', komodo_name: 'saved-resource' })
|
||||
}));
|
||||
|
||||
const fetchMock = vi.fn<typeof fetch>();
|
||||
|
||||
beforeEach(() => {
|
||||
fetchMock.mockReset();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('Komodo health endpoint fallback', () => {
|
||||
it('limits an unreachable probe to one timeout and preserves its diagnosis', async () => {
|
||||
vi.useFakeTimers();
|
||||
fetchMock.mockImplementation((_url, options) => new Promise((_resolve, reject) => {
|
||||
options?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')));
|
||||
}));
|
||||
|
||||
const result = komodoInspectHealthy('server');
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(await result).toMatchObject({
|
||||
ok: false,
|
||||
reason: 'timed out after 1000ms dialing http://komodo.test/read/GetDeploymentContainer'
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves the network cause without probing alternate endpoints', async () => {
|
||||
fetchMock.mockRejectedValue(new TypeError('fetch failed', {
|
||||
cause: Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' })
|
||||
}));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toMatchObject({
|
||||
ok: false,
|
||||
reason: expect.stringContaining('ECONNREFUSED: connection refused')
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([400, 401, 403, 429, 500, 502, 503])('stops on HTTP %i without hiding the response', async status => {
|
||||
fetchMock.mockResolvedValue(new Response('request failed', { status }));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toEqual({
|
||||
ok: false, reason: `HTTP ${status}`, snippet: 'request failed'
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[404, 'Not Found'],
|
||||
[400, 'unknown variant `GetDeploymentContainer`'],
|
||||
[405, 'unsupported method'],
|
||||
[500, 'unknown method'],
|
||||
[501, 'method not found']
|
||||
])('tries a compatible endpoint after HTTP %i: %s', async (status, message) => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(new Response(message as string, { status: status as number }))
|
||||
.mockResolvedValueOnce(Response.json({ State: { Status: 'running' } }));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' });
|
||||
expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
|
||||
'http://komodo.test/read/GetDeploymentContainer',
|
||||
'http://komodo.test/read/InspectDeploymentContainer'
|
||||
]);
|
||||
});
|
||||
|
||||
it('continues past successful responses with an unknown shape', async () => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(Response.json({ config: { image: 'server' } }))
|
||||
.mockResolvedValueOnce(Response.json({ State: { Status: 'running' } }));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' });
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not overwrite an unhealthy container state with a later response', async () => {
|
||||
fetchMock.mockResolvedValueOnce(Response.json({ State: { Status: 'exited' } }));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toMatchObject({ ok: false, state: 'exited', reason: 'container state: exited' });
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves malformed response diagnostics', async () => {
|
||||
fetchMock.mockResolvedValueOnce(new Response('not json'));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toMatchObject({
|
||||
ok: false, reason: expect.stringContaining('unparseable json:'), snippet: 'not json'
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -256,6 +256,10 @@ describe('provisioning operations', () => {
|
||||
|
||||
async function incident(id = 'incident') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
// These lifecycle tests exercise an actual container failure. Healthy
|
||||
// containers with fresh game evidence now cancel obsolete health work.
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'unhealthy', reason: 'Container unhealthy' });
|
||||
const now = new Date().toISOString();
|
||||
db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
|
||||
VALUES (?,?,'health','open','Server down',?,?)`).run(id, server, now, now);
|
||||
@@ -323,7 +327,7 @@ describe('Watchdog provisioning and telemetry evidence', () => {
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
vi.spyOn(config.rcon, 'password', 'get').mockReturnValue('');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
@@ -333,6 +337,7 @@ describe('Watchdog provisioning and telemetry evidence', () => {
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await processProvisioning();
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { createSocket, type Socket } from 'node:dgram';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { komodoConfigured, rconConfigured } from '../../src/lib/server/config.js';
|
||||
import { getServer, type ServerRow } from '../../src/lib/server/fleet/servers.js';
|
||||
import { latestGameSubmission } from '../../src/lib/server/fleet/lifecycle-evidence.js';
|
||||
import { rconGetPlayerCount, rconPlutoniumVersion, rconStatusResult } from '../../src/lib/server/integrations/rcon.js';
|
||||
|
||||
vi.mock('node:dgram', () => ({ createSocket: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/config.js', () => ({
|
||||
config: { rcon: { host: 'localhost', password: 'test', timeoutMs: 1_000 } },
|
||||
komodoConfigured: vi.fn(), rconConfigured: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn(), setServerField: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/fleet/lifecycle-evidence.js', () => ({ latestGameSubmission: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoResolveGamePort: vi.fn() }));
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
send = vi.fn();
|
||||
close = vi.fn();
|
||||
}
|
||||
|
||||
let sockets: FakeSocket[];
|
||||
let server: ServerRow;
|
||||
const packet = (body: string) => Buffer.concat([Buffer.from([255, 255, 255, 255]), Buffer.from(body)]);
|
||||
const reply = (index = 0) => sockets[index].emit('message', packet('statusResponse\n\\mapname\\mp_kino\n10 45 "joe"'));
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.useFakeTimers();
|
||||
sockets = [];
|
||||
server = { name: 'test-server', created_at: '2026-10-01T00:00:00Z', game_port: 28960, rcon_host: null } as ServerRow;
|
||||
vi.mocked(getServer).mockImplementation(() => ({ ...server }));
|
||||
vi.mocked(latestGameSubmission).mockReturnValue(null);
|
||||
vi.mocked(komodoConfigured).mockReturnValue(false);
|
||||
vi.mocked(rconConfigured).mockReturnValue(true);
|
||||
vi.mocked(createSocket).mockImplementation(() => {
|
||||
const socket = new FakeSocket();
|
||||
sockets.push(socket);
|
||||
return socket as unknown as Socket;
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('RCON status transport', () => {
|
||||
it('shares overlapping health and occupancy probes and probes again after completion', async () => {
|
||||
const health = rconStatusResult(server.name);
|
||||
const occupancy = rconGetPlayerCount(server.name);
|
||||
const duplicate = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sockets).toHaveLength(1);
|
||||
expect(sockets[0].send.mock.calls[0][0].toString()).toContain('rcon test status\n');
|
||||
reply();
|
||||
expect(await health).toMatchObject({ ok: true, status: { players: 1, map: 'mp_kino' } });
|
||||
expect(await occupancy).toBe(1);
|
||||
expect(await duplicate).toEqual(await health);
|
||||
expect(sockets[0].close).toHaveBeenCalledOnce();
|
||||
|
||||
const fresh = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sockets).toHaveLength(2);
|
||||
reply(1);
|
||||
expect(await fresh).toMatchObject({ ok: true });
|
||||
});
|
||||
|
||||
it('recovers a dropped UDP reply after the first backoff', async () => {
|
||||
const request = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(sockets).toHaveLength(1);
|
||||
expect(sockets[0].close).toHaveBeenCalledOnce();
|
||||
await vi.advanceTimersByTimeAsync(249);
|
||||
expect(sockets).toHaveLength(1);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(sockets).toHaveLength(2);
|
||||
reply(1);
|
||||
expect(await request).toMatchObject({ ok: true });
|
||||
});
|
||||
|
||||
it('bounds retries at three attempts with 250ms then 500ms backoffs', async () => {
|
||||
const request = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(1_250);
|
||||
expect(sockets).toHaveLength(2);
|
||||
await vi.advanceTimersByTimeAsync(1_499);
|
||||
expect(sockets).toHaveLength(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(sockets).toHaveLength(3);
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(await request).toMatchObject({ ok: false, reason: 'timeout' });
|
||||
expect(sockets.every(socket => socket.close.mock.calls.length === 1)).toBe(true);
|
||||
|
||||
const fresh = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sockets).toHaveLength(4);
|
||||
reply(3);
|
||||
expect(await fresh).toMatchObject({ ok: true });
|
||||
});
|
||||
|
||||
it.each(['event', 'callback'])('retries transient send errors reported through the %s', async source => {
|
||||
const request = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
if (source === 'event') sockets[0].emit('error', new Error('UDP unavailable'));
|
||||
else sockets[0].send.mock.calls[0][5](new Error('UDP unavailable'));
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
expect(sockets).toHaveLength(2);
|
||||
reply(1);
|
||||
expect(await request).toMatchObject({ ok: true });
|
||||
});
|
||||
|
||||
it.each([
|
||||
['missing prefix', Buffer.from('wrong listener')],
|
||||
['wrong password', packet('print\nBad rcon_password')],
|
||||
['unknown dialect', packet('unexpected reply')]
|
||||
])('does not retry a %s reply', async (_name, response) => {
|
||||
const request = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
sockets[0].emit('message', response);
|
||||
expect(await request).toMatchObject({ ok: false, reason: 'bad-reply' });
|
||||
expect(sockets).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not retry missing configuration or endpoint failures', async () => {
|
||||
vi.mocked(rconConfigured).mockReturnValue(false);
|
||||
expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-config' });
|
||||
server.game_port = null;
|
||||
expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-port' });
|
||||
vi.mocked(getServer).mockReturnValue(undefined);
|
||||
expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-port' });
|
||||
expect(sockets).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('cleans up a rejected in-flight probe so the next one can proceed', async () => {
|
||||
vi.mocked(createSocket).mockImplementationOnce(() => { throw new Error('Socket initialization failed'); });
|
||||
await expect(rconStatusResult(server.name)).rejects.toThrow('Socket initialization failed');
|
||||
const request = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
reply();
|
||||
expect(await request).toMatchObject({ ok: true });
|
||||
});
|
||||
|
||||
it.each(['restart', 'registration', 'host', 'port', 'server'])('does not share probes across a changed %s', async change => {
|
||||
const original = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
if (change === 'restart') vi.mocked(latestGameSubmission).mockReturnValue('2026-10-02T00:00:00Z');
|
||||
if (change === 'registration') server.created_at = '2026-10-02T00:00:00Z';
|
||||
if (change === 'host') server.rcon_host = 'new-host';
|
||||
if (change === 'port') server.game_port = 28961;
|
||||
if (change === 'server') server.name = 'other-server';
|
||||
const current = rconStatusResult(server.name);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(sockets).toHaveLength(2);
|
||||
reply();
|
||||
sockets[1].emit('message', packet('statusResponse\n\\mapname\\mp_ascension\n'));
|
||||
expect(await original).toMatchObject({ ok: true, status: { players: 1 } });
|
||||
expect(await current).toMatchObject({ ok: true, status: { players: 0 } });
|
||||
});
|
||||
|
||||
it('leaves other RCON commands on their existing single-attempt behavior', async () => {
|
||||
const request = rconPlutoniumVersion(server.name);
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(await request).toMatchObject({ revision: null, error: 'no reply within 1000ms' });
|
||||
expect(sockets).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -30,6 +30,110 @@ async function setup() {
|
||||
}
|
||||
|
||||
describe('database fleet and container readiness', () => {
|
||||
it('refreshes a ten-server fleet while one check is slow, without duplicating or exceeding four checks', async () => {
|
||||
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
const now = Date.parse('2026-10-02T12:00:00Z');
|
||||
vi.setSystemTime(now);
|
||||
for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(`fast-${i}`, `fast-${i}`);
|
||||
let release!: (result: { ok: true; state: string }) => void;
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
inspect.mockImplementation(async name => {
|
||||
peak = Math.max(peak, ++active);
|
||||
try {
|
||||
await Promise.resolve();
|
||||
return name === 'managed'
|
||||
? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; })
|
||||
: { ok: true as const, state: 'running' };
|
||||
} finally { active--; }
|
||||
});
|
||||
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
||||
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
||||
const events: unknown[] = [];
|
||||
const unsubscribe = subscribe(event => events.push(event));
|
||||
const first = fanoutStatus();
|
||||
try {
|
||||
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(9));
|
||||
expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: expect.arrayContaining([
|
||||
expect.objectContaining({ server: 'fast-8', health: 'online', players: 2 })
|
||||
]) }));
|
||||
vi.setSystemTime(now + 30_000);
|
||||
const second = fanoutStatus();
|
||||
await vi.waitFor(() => expect(Date.parse(getServer('fast-8')?.last_seen_at ?? '')).toBeGreaterThanOrEqual(now + 30_000));
|
||||
expect(rcon).toHaveBeenCalledTimes(18);
|
||||
expect(inspect.mock.calls.filter(([name]) => name === 'managed')).toHaveLength(1);
|
||||
expect(peak).toBe(4);
|
||||
release({ ok: true, state: 'running' });
|
||||
await Promise.all([first, second]);
|
||||
expect(rcon).toHaveBeenCalledTimes(19);
|
||||
} finally {
|
||||
release({ ok: true, state: 'running' });
|
||||
await first;
|
||||
unsubscribe();
|
||||
}
|
||||
});
|
||||
|
||||
it('expires early results before recording a fleet collection delayed by a slow check', async () => {
|
||||
const { db, inspect, rcon, fanoutStatus } = await setup();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
const now = Date.parse('2026-10-02T12:00:00Z');
|
||||
vi.setSystemTime(now);
|
||||
db.prepare("INSERT INTO servers (name, komodo_name) VALUES ('slow', 'slow')").run();
|
||||
let release!: (result: { ok: true; state: string }) => void;
|
||||
inspect.mockImplementation(async name => name === 'slow'
|
||||
? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; })
|
||||
: { ok: true as const, state: 'running' });
|
||||
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
||||
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
||||
const pending = fanoutStatus();
|
||||
try {
|
||||
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
|
||||
vi.setSystemTime(now + 46_000);
|
||||
} finally {
|
||||
release({ ok: true, state: 'running' });
|
||||
await pending;
|
||||
}
|
||||
expect(db.prepare('SELECT server, players FROM metric_samples ORDER BY server').all()).toEqual([
|
||||
{ server: 'managed', players: null }, { server: 'slow', players: 2 }
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps a manual probe diagnostic without persisting it across a restart', async () => {
|
||||
const { db, rcon, getServer } = await setup();
|
||||
const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
|
||||
let release!: (result: Awaited<ReturnType<typeof rcon>>) => void;
|
||||
rcon.mockImplementation(() => new Promise(resolve => { release = resolve; }));
|
||||
const pending = POST({ params: { name: 'managed' } } as never);
|
||||
db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString());
|
||||
release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
||||
expect(await (await pending).json()).toMatchObject({ ok: true });
|
||||
expect(getServer('managed')).toMatchObject({ last_seen_at: null, last_error: 'timeout:old process' });
|
||||
});
|
||||
|
||||
it.each(['container', 'rcon'])('ignores an old registration whose %s check finishes after replacement', async phase => {
|
||||
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
let release!: () => void;
|
||||
const blocked = new Promise<void>(resolve => { release = resolve; });
|
||||
inspect.mockImplementation(async () => {
|
||||
if (phase === 'container') await blocked;
|
||||
return { ok: true, state: 'running' };
|
||||
});
|
||||
rcon.mockImplementation(async () => {
|
||||
if (phase === 'rcon') await blocked;
|
||||
return { ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };
|
||||
});
|
||||
const pending = fanoutStatus();
|
||||
try {
|
||||
if (phase === 'rcon') await vi.waitFor(() => expect(rcon).toHaveBeenCalled());
|
||||
db.prepare("UPDATE servers SET created_at = 'replacement', container_state = NULL, last_error = NULL WHERE name = 'managed'").run();
|
||||
} finally { release(); await pending; }
|
||||
expect(getServer('managed')).toMatchObject({ container_state: null, last_seen_at: null, last_error: null });
|
||||
});
|
||||
|
||||
it.each(['queued', 'running', 'awaiting-health'])('shows provisioning during %s creation and resumes health reporting afterward', async (state) => {
|
||||
const { db, inspect, rcon, fanoutStatus } = await setup();
|
||||
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
||||
|
||||
@@ -153,6 +153,105 @@ describe('Watchdog detection and ownership', () => {
|
||||
expect(operations()).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['no-config', 'no-port', 'bad-reply', 'no-players'] as const)('keeps %s RCON failures observational', async reason => {
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue({ ...missingRcon, reason });
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(onlyIncident()).toMatchObject({ state: 'open' });
|
||||
expect(operations()).toEqual([]);
|
||||
});
|
||||
|
||||
it('breaks the sustained failure window on a successful probe between watchdog samples', async () => {
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
||||
await watchdog.watchdogTick(true);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
advance(20_000);
|
||||
const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
|
||||
await POST({ params: { name: server } } as Parameters<typeof POST>[0]);
|
||||
advance(40_000);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open', first_seen_at: new Date().toISOString() });
|
||||
expect(operations()).toEqual([]);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([]);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
||||
});
|
||||
|
||||
it.each([0, 2])('cancels queued health recovery when fresh RCON responds with %i players', async players => {
|
||||
const row = await sustainedFailure();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(players);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('holds queued health recovery when container readiness becomes unknown', async () => {
|
||||
await sustainedFailure();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(operations()[0].state).toBe('waiting');
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not apply a recovered occupancy probe after its operation is replaced', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const restart = operations()[0];
|
||||
const { cancelProvisioning, requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockImplementationOnce(async () => {
|
||||
expect(cancelProvisioning(restart.id)).toBe(true);
|
||||
expect(requestLifecycle(server, 'stop').ok).toBe(true);
|
||||
return 0;
|
||||
});
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' }),
|
||||
expect.objectContaining({ action: 'stop', state: 'waiting' })]);
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('cancels obsolete health recovery so a later queued maintenance restart can proceed', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const job = database.db().prepare('INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)')
|
||||
.run(server, new Date().toISOString()).lastInsertRowid;
|
||||
const maintenance = requestRestart(server, { kind: 'maintenance', id: String(job) });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(operations()).toEqual([expect.objectContaining({ state: 'cancelled' }),
|
||||
expect.objectContaining({ id: maintenance.id, state: 'awaiting-health' })]);
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each(['restart', 'rebuild'] as const)('preserves an operator request adopting a queued health %s', async action => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
if (action === 'rebuild') {
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
}
|
||||
const queued = operations().at(-1)!;
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
expect(requestLifecycle(server, action)).toEqual({ ok: true, deploymentId: queued.id });
|
||||
expect(database.db().prepare('SELECT triggered_by FROM server_deployments WHERE id = ?').get(queued.id)).toEqual({ triggered_by: 'ui' });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
if (action === 'rebuild') vi.mocked(komodo.komodoDestroyDeployment).mockImplementationOnce(async () => { down(); return { ok: true }; });
|
||||
await processProvisioning();
|
||||
expect(operations().at(-1)).toMatchObject({ id: queued.id, action, state: action === 'restart' ? 'awaiting-health' : 'queued' });
|
||||
if (action === 'restart') expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
else expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves intentional stop intent after completion and database reopening', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const stop = requestLifecycle(server, 'stop');
|
||||
@@ -241,6 +340,79 @@ describe('Watchdog detection and ownership', () => {
|
||||
});
|
||||
|
||||
describe('Watchdog recovery persistence', () => {
|
||||
it('verifies recovery at the deadline before deciding whether to rebuild', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'done' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ healthy_at: new Date().toISOString() });
|
||||
});
|
||||
|
||||
it('defers escalation while current container health is unknown', async () => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'awaiting-health' })]);
|
||||
});
|
||||
|
||||
it('keeps failed submission history when fresh evidence later releases recovery', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
vi.mocked(komodo.komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Reply lost' });
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'failed' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ healthy_at: new Date().toISOString() });
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions[0].operation_error).toContain('Reply lost');
|
||||
});
|
||||
|
||||
it('discards escalation evidence if another lifecycle submission appears during its probe', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rcon.rconStatusResult).mockImplementationOnce(async () => {
|
||||
database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
||||
VALUES ('newer', ?, 'restart', 'done', 'Complete', ?, ?, 'manual')`).run(server, new Date().toISOString(), new Date().toISOString());
|
||||
return healthyRcon;
|
||||
});
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()[0]).toMatchObject({ action: 'restart', state: 'awaiting-health' });
|
||||
expect(operations().some(op => op.action === 'rebuild')).toBe(false);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
||||
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
||||
});
|
||||
|
||||
it('cancels a queued health rebuild and releases its transferred slot when the game recovers', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
const rebuild = operations()[1];
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await processProvisioning();
|
||||
expect(operations()).toEqual([expect.objectContaining({ state: 'failed' }), expect.objectContaining({ state: 'cancelled' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: rebuild.id, healthy_at: new Date().toISOString() });
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves the original finding as deleted history after an intentional successful destroy', async () => {
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
|
||||
Reference in New Issue
Block a user