688 lines
43 KiB
TypeScript
688 lines
43 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { access, mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { gameEvent } from '../helpers/telemetry.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
|
komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
|
|
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
|
komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
|
|
}));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
|
|
|
const server = 't5_zombie_theater_1';
|
|
const healthyRcon = { ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };
|
|
const missingRcon = { ok: false as const, endpoint: { host: 'test', port: 28960 }, reason: 'timeout' as const, detail: 'No reply' };
|
|
let fixture: Fixture;
|
|
type Engine = typeof import('../../src/lib/server/watchdog/index.js');
|
|
let watchdog: Engine;
|
|
let database: typeof import('../../src/lib/server/db/index.js');
|
|
let komodo: typeof import('../../src/lib/server/integrations/komodo.js');
|
|
let rcon: typeof import('../../src/lib/server/integrations/rcon.js');
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules(); vi.resetAllMocks();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
|
fixture = await newFixture([server]);
|
|
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
|
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
|
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
|
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
|
database = await import('../../src/lib/server/db/index.js');
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
database.db().prepare('UPDATE servers SET game_port = 28960 WHERE name = ?').run(server);
|
|
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
saveServerKey(server, 'saved-key');
|
|
komodo = await import('../../src/lib/server/integrations/komodo.js');
|
|
for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) {
|
|
vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
|
|
}
|
|
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
|
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
|
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
|
watchdog = await import('../../src/lib/server/watchdog/index.js');
|
|
});
|
|
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
|
|
|
const incidents = () => watchdog.watchdogOverview().incidents;
|
|
const onlyIncident = () => { expect(incidents()).toHaveLength(1); return incidents()[0]; };
|
|
const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[];
|
|
function advance(ms: number) { vi.setSystemTime(Date.now() + ms); }
|
|
function down() { vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
|
|
async function sustainedFailure() {
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
|
|
return onlyIncident();
|
|
}
|
|
|
|
describe('Watchdog detection and ownership', () => {
|
|
it('defaults to automatic recovery and debounces manual scans without accelerating the failure window', async () => {
|
|
expect(watchdog.watchdogOverview().enabled).toBe(true);
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({ state: 'open', failures: 1 });
|
|
for (let n = 0; n < 5; n++) await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().failures).toBe(1);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident().failures).toBe(2);
|
|
advance(watchdog.WATCHDOG_SCAN_MS - 1);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
advance(1);
|
|
await watchdog.watchdogTick(true);
|
|
const row = onlyIncident();
|
|
expect(row).toMatchObject({ state: 'recovering', failures: 3 });
|
|
expect(watchdog.watchdogDetail(row.id)?.actions).toEqual([expect.objectContaining({ action: 'restart', operation_state: 'waiting' })]);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toHaveLength(1);
|
|
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('keeps paused recovery observational and resumes one pending incident', async () => {
|
|
watchdog.setWatchdogEnabled(false);
|
|
down();
|
|
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
|
expect(onlyIncident()).toMatchObject({ state: 'open', failures: 4 });
|
|
expect(operations()).toEqual([]);
|
|
database.closeDb();
|
|
expect(watchdog.watchdogOverview().enabled).toBe(false);
|
|
watchdog.setWatchdogEnabled(true);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
watchdog.setWatchdogEnabled(false);
|
|
await processProvisioning();
|
|
expect(operations()[0].state).toBe('waiting');
|
|
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
|
watchdog.setWatchdogEnabled(true);
|
|
await processProvisioning();
|
|
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('does not restart an unknown Komodo state even after repeated sustained observations', async () => {
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo request timed out' });
|
|
for (let n = 0; n < 8; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
|
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' });
|
|
expect(operations()).toEqual([]);
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().failures).toBe(1);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
|
});
|
|
|
|
it('allows a longer initial startup grace period before requesting recovery', async () => {
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
|
|
for (let n = 0; n < 5; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
|
expect(operations()).toEqual([]);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
|
});
|
|
|
|
it('accepts fresh validated telemetry when RCON is unavailable and rechecks after it becomes stale', async () => {
|
|
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
|
saveTelemetry({ ...gameEvent('heartbeat', 17, 4, server), receivedAt: new Date().toISOString() });
|
|
await watchdog.watchdogTick(true);
|
|
expect(incidents()).toEqual([]);
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
advance(60_000);
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 1 });
|
|
saveTelemetry({ ...gameEvent('heartbeat', 18, 4, server), receivedAt: new Date().toISOString() });
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().state).toBe('resolved');
|
|
expect(operations()).toEqual([]);
|
|
});
|
|
|
|
it.each(['no-config', 'no-port', 'bad-reply', 'no-players'] as const)('keeps %s RCON failures observational', async reason => {
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue({ ...missingRcon, reason });
|
|
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
|
expect(onlyIncident()).toMatchObject({ state: 'open' });
|
|
expect(operations()).toEqual([]);
|
|
});
|
|
|
|
it('breaks the sustained failure window on a successful probe between watchdog samples', async () => {
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
advance(20_000);
|
|
const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
|
|
await POST({ params: { name: server } } as Parameters<typeof POST>[0]);
|
|
advance(40_000);
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open', first_seen_at: new Date().toISOString() });
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
|
});
|
|
|
|
it.each([0, 2])('cancels queued health recovery when fresh RCON responds with %i players', async players => {
|
|
const row = await sustainedFailure();
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(players);
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' })]);
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
|
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('holds queued health recovery when container readiness becomes unknown', async () => {
|
|
await sustainedFailure();
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
expect(operations()[0].state).toBe('waiting');
|
|
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('does not apply a recovered occupancy probe after its operation is replaced', async () => {
|
|
const row = await sustainedFailure();
|
|
const restart = operations()[0];
|
|
const { cancelProvisioning, requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(rcon.rconGetPlayerCount).mockImplementationOnce(async () => {
|
|
expect(cancelProvisioning(restart.id)).toBe(true);
|
|
expect(requestLifecycle(server, 'stop').ok).toBe(true);
|
|
return 0;
|
|
});
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' }),
|
|
expect.objectContaining({ action: 'stop', state: 'waiting' })]);
|
|
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('cancels obsolete health recovery so a later queued maintenance restart can proceed', async () => {
|
|
const row = await sustainedFailure();
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const job = database.db().prepare('INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)')
|
|
.run(server, new Date().toISOString()).lastInsertRowid;
|
|
const maintenance = requestRestart(server, { kind: 'maintenance', id: String(job) });
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
|
expect(operations()).toEqual([expect.objectContaining({ state: 'cancelled' }),
|
|
expect.objectContaining({ id: maintenance.id, state: 'awaiting-health' })]);
|
|
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it.each(['restart', 'rebuild'] as const)('preserves an operator request adopting a queued health %s', async action => {
|
|
await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
if (action === 'rebuild') {
|
|
await processProvisioning();
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
|
await watchdog.watchdogTick(true);
|
|
}
|
|
const queued = operations().at(-1)!;
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
expect(requestLifecycle(server, action)).toEqual({ ok: true, deploymentId: queued.id });
|
|
expect(database.db().prepare('SELECT triggered_by FROM server_deployments WHERE id = ?').get(queued.id)).toEqual({ triggered_by: 'ui' });
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
if (action === 'rebuild') vi.mocked(komodo.komodoDestroyDeployment).mockImplementationOnce(async () => { down(); return { ok: true }; });
|
|
await processProvisioning();
|
|
expect(operations().at(-1)).toMatchObject({ id: queued.id, action, state: action === 'restart' ? 'awaiting-health' : 'queued' });
|
|
if (action === 'restart') expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
|
else expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('preserves intentional stop intent after completion and database reopening', async () => {
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const stop = requestLifecycle(server, 'stop');
|
|
expect(stop.ok).toBe(true);
|
|
if (!stop.ok) throw new Error(stop.error);
|
|
database.db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ? WHERE id = ?").run(new Date().toISOString(), stop.deploymentId);
|
|
down();
|
|
database.closeDb();
|
|
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
|
expect(incidents()).toEqual([]);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
|
|
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it.each(['beta', 'campaign'] as const)('defers a server owned by %s work', async (owner) => {
|
|
if (owner === 'beta') {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { script, version } = createScript({ name: 'beta', content: Buffer.from('experimental'), targetRelPath: 'beta.gsc' });
|
|
await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server], waitForHealth: false, waitForMatch: false });
|
|
} else {
|
|
database.db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', ?)").run(new Date().toISOString());
|
|
database.db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', ?, 1, 'held')").run(server);
|
|
}
|
|
down();
|
|
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
|
expect(incidents()).toEqual([]);
|
|
expect(operations()).toEqual([]);
|
|
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('rechecks lifecycle ownership after a health probe yields to an operator stop', async () => {
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => {
|
|
expect(requestLifecycle(server, 'stop').ok).toBe(true);
|
|
return { ok: false, state: 'exited', reason: 'Container exited' };
|
|
});
|
|
await watchdog.watchdogTick(true);
|
|
expect(incidents()).toEqual([]);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
|
|
});
|
|
|
|
it.each(['queued', 'waiting', 'running', 'awaiting-health', 'retained recovery'])(
|
|
'suppresses health and drift throughout %s provisioning, then resumes', async state => {
|
|
const retained = state === 'retained recovery';
|
|
const now = new Date().toISOString();
|
|
database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('manual-operation', ?, ?, ?, 'Operator work', ?, 'manual')`)
|
|
.run(server, retained ? 'restart' : 'rebuild', retained ? 'failed' : state, now);
|
|
if (retained) database.db().prepare(`INSERT INTO restart_rollout (server, operation_id, requested_at)
|
|
VALUES (?, 'manual-operation', ?)`).run(server, now);
|
|
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
|
|
down();
|
|
for (let n = 0; n < 4; n++) {
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
expect(incidents()).toEqual([]);
|
|
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
expect(operations()).toHaveLength(1);
|
|
|
|
database.db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'manual-operation'").run();
|
|
if (retained) database.db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), server);
|
|
await watchdog.watchdogTick(true);
|
|
expect(incidents()).toEqual(expect.arrayContaining([
|
|
expect.objectContaining({ kind: 'health', state: 'open', failures: 1 }),
|
|
expect.objectContaining({ kind: 'drift', state: 'recovering' })
|
|
]));
|
|
expect(operations()).toHaveLength(2);
|
|
expect(operations()[1]).toMatchObject({ action: 'rebuild', state: 'waiting' });
|
|
}
|
|
);
|
|
|
|
it('stops probing when provisioning takes ownership during the container inspection', async () => {
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => {
|
|
expect(requestLifecycle(server, 'restart').ok).toBe(true);
|
|
return { ok: true, state: 'running' };
|
|
});
|
|
await watchdog.watchdogTick(true);
|
|
expect(incidents()).toEqual([]);
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
|
|
});
|
|
});
|
|
|
|
describe('Watchdog recovery persistence', () => {
|
|
it('verifies recovery at the deadline before deciding whether to rebuild', async () => {
|
|
const row = await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'done' })]);
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
|
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
|
.toEqual({ healthy_at: new Date().toISOString() });
|
|
});
|
|
|
|
it('defers escalation while current container health is unknown', async () => {
|
|
await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'awaiting-health' })]);
|
|
});
|
|
|
|
it('keeps failed submission history when fresh evidence later releases recovery', async () => {
|
|
const row = await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
vi.mocked(komodo.komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Reply lost' });
|
|
await processProvisioning();
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'failed' })]);
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
|
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
|
.toEqual({ healthy_at: new Date().toISOString() });
|
|
expect(watchdog.watchdogDetail(row.id)?.actions[0].operation_error).toContain('Reply lost');
|
|
});
|
|
|
|
it('discards escalation evidence if another lifecycle submission appears during its probe', async () => {
|
|
const row = await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(rcon.rconStatusResult).mockImplementationOnce(async () => {
|
|
database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
|
VALUES ('newer', ?, 'restart', 'done', 'Complete', ?, ?, 'manual')`).run(server, new Date().toISOString(), new Date().toISOString());
|
|
return healthyRcon;
|
|
});
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()[0]).toMatchObject({ action: 'restart', state: 'awaiting-health' });
|
|
expect(operations().some(op => op.action === 'rebuild')).toBe(false);
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
|
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
|
});
|
|
|
|
it('cancels a queued health rebuild and releases its transferred slot when the game recovers', async () => {
|
|
const row = await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
|
const rebuild = operations()[1];
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
await processProvisioning();
|
|
expect(operations()).toEqual([expect.objectContaining({ state: 'failed' }), expect.objectContaining({ state: 'cancelled' })]);
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
|
expect(database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
|
.toEqual({ operation_id: rebuild.id, healthy_at: new Date().toISOString() });
|
|
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('preserves the original finding as deleted history after an intentional successful destroy', async () => {
|
|
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
|
removeServerKey(server);
|
|
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
|
|
await watchdog.watchdogTick(true);
|
|
const id = onlyIncident().id;
|
|
expect(onlyIncident().state).toBe('attention');
|
|
expect(destroyServer(server)).toMatchObject({ ok: true });
|
|
down();
|
|
await processDestructions();
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'destroy', state: 'done' })]);
|
|
expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
|
await watchdog.watchdogTick(true);
|
|
expect(watchdog.watchdogDetail(id)?.incident).toMatchObject({ state: 'attention', server_deleted_at: expect.any(String) });
|
|
expect(watchdog.updateWatchdogIncident(id, 'retry')).toMatchObject({ ok: false, status: 409 });
|
|
});
|
|
|
|
it('requests one restart, escalates once to rebuild, and retains exhausted attention across reopening', async () => {
|
|
const row = await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
|
expect(operations()[0].state).toBe('awaiting-health');
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS - 1);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toHaveLength(1);
|
|
advance(1);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
|
expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(2);
|
|
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' });
|
|
await processProvisioning();
|
|
expect(operations()[1].state).toBe('failed');
|
|
await watchdog.watchdogTick(true);
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention');
|
|
database.closeDb();
|
|
for (let n = 0; n < 4; n++) { advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true); }
|
|
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention');
|
|
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
|
expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('does not escalate a timed out restart while automatic recovery is paused', async () => {
|
|
await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
watchdog.setWatchdogEnabled(false);
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toHaveLength(1);
|
|
watchdog.setWatchdogEnabled(true);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
|
});
|
|
|
|
it('reclaims a failed attempt recovery slot only after an explicit incident retry', async () => {
|
|
const row = await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
|
await watchdog.watchdogTick(true);
|
|
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' });
|
|
await processProvisioning();
|
|
await watchdog.watchdogTick(true);
|
|
const failed = operations()[1];
|
|
expect(failed).toMatchObject({ action: 'rebuild', state: 'failed' });
|
|
expect(onlyIncident()).toMatchObject({ id: row.id, state: 'attention', cycle: 1 });
|
|
const slot = () => database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server);
|
|
expect(slot()).toEqual({ operation_id: failed.id, healthy_at: null });
|
|
|
|
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
|
|
vi.mocked(komodo.komodoInspectHealthy).mockClear();
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
|
expect(operations()).toHaveLength(2);
|
|
|
|
expect(watchdog.updateWatchdogIncident(row.id, 'retry')).toEqual({ ok: true });
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toHaveLength(2);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
const retry = operations()[2];
|
|
expect(retry).toMatchObject({ action: 'restart', state: 'waiting' });
|
|
expect(onlyIncident()).toMatchObject({ id: row.id, state: 'recovering', cycle: 2 });
|
|
expect(slot()).toEqual({ operation_id: retry.id, healthy_at: null });
|
|
expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(3);
|
|
|
|
await processProvisioning();
|
|
expect(komodo.komodoRestart).toHaveBeenCalledTimes(2);
|
|
expect(operations()[2].state).toBe('awaiting-health');
|
|
});
|
|
|
|
it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
const id = onlyIncident().id;
|
|
database.closeDb();
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().id).toBe(id);
|
|
const queued = operations();
|
|
expect(queued).toHaveLength(1);
|
|
database.closeDb();
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual(queued);
|
|
expect(watchdog.watchdogDetail(id)?.actions).toHaveLength(1);
|
|
});
|
|
|
|
it('permits an explicit new attempt after dismissal without silently rearming a sustained condition', async () => {
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
const id = onlyIncident().id;
|
|
expect(watchdog.updateWatchdogIncident(id, 'dismiss')).toEqual({ ok: true });
|
|
for (let n = 0; n < 4; n++) { advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); }
|
|
expect(operations()).toEqual([]);
|
|
expect(onlyIncident().state).toBe('dismissed');
|
|
expect(watchdog.updateWatchdogIncident(id, 'retry')).toEqual({ ok: true });
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({ id, state: 'recovering', cycle: 2 });
|
|
expect(watchdog.watchdogDetail(id)?.actions).toEqual([expect.objectContaining({ action: 'restart', cycle: 2 })]);
|
|
});
|
|
});
|
|
|
|
describe('Watchdog APIs', () => {
|
|
const request = (body: unknown, raw = false) => new Request('http://localhost/api/watchdog', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: raw ? String(body) : JSON.stringify(body)
|
|
});
|
|
|
|
it('returns overview state and validates scan actions and pause booleans', async () => {
|
|
const { GET, POST } = await import('../../src/routes/api/watchdog/+server.js');
|
|
expect(await GET().json()).toEqual({ enabled: true, lastScanAt: null, scanError: null, incidents: [] });
|
|
expect((await POST({ request: request('{', true) })).status).toBe(400);
|
|
for (const body of [null, [], {}, { action: 'restart' }, { enabled: 'false' }, { enabled: 0 }, { action: 'scan', enabled: 'false' }]) {
|
|
expect((await POST({ request: request(body) })).status).toBe(400);
|
|
}
|
|
expect(await (await POST({ request: request({ enabled: false }) })).json()).toMatchObject({ enabled: false });
|
|
database.closeDb();
|
|
expect(await GET().json()).toMatchObject({ enabled: false });
|
|
down();
|
|
const scanned = await POST({ request: request({ action: 'scan' }) });
|
|
expect(scanned.status).toBe(200);
|
|
expect(await scanned.json()).toMatchObject({ enabled: false, lastScanAt: expect.any(String), incidents: [expect.objectContaining({ kind: 'health', state: 'open' })] });
|
|
expect(operations()).toEqual([]);
|
|
});
|
|
|
|
it('validates incident actions and refuses retries while provisioning is pending', async () => {
|
|
const { GET, POST } = await import('../../src/routes/api/watchdog/[id]/+server.js');
|
|
expect(GET({ params: { id: 'missing' } }).status).toBe(404);
|
|
expect((await POST({ params: { id: 'missing' }, request: request({ action: 'retry' }) })).status).toBe(404);
|
|
for (const body of [null, {}, [], { action: 'restart' }]) {
|
|
expect((await POST({ params: { id: 'missing' }, request: request(body) })).status).toBe(400);
|
|
}
|
|
expect((await POST({ params: { id: 'missing' }, request: request('{', true) })).status).toBe(400);
|
|
const row = await sustainedFailure();
|
|
expect(await GET({ params: { id: row.id } }).json()).toMatchObject({
|
|
incident: { id: row.id, state: 'recovering' }, actions: [expect.objectContaining({ action: 'restart' })], events: expect.any(Array)
|
|
});
|
|
for (const action of ['retry', 'dismiss']) {
|
|
const response = await POST({ params: { id: row.id }, request: request({ action }) });
|
|
expect(response.status).toBe(409);
|
|
expect(await response.json()).toMatchObject({ error: expect.stringContaining('Provisioning is still active') });
|
|
}
|
|
expect(operations()).toHaveLength(1);
|
|
});
|
|
|
|
it('keeps legacy drift previews readable and directs mutations to Watchdog without changing files', async () => {
|
|
const { GET, POST } = await import('../../src/routes/api/drift/+server.js');
|
|
const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc');
|
|
await writeFile(path, 'unexpected content');
|
|
expect(await (await GET()).json()).toMatchObject({ totalDrift: 1, servers: [{ server, files: [expect.objectContaining({ relPath: 'stray.gsc' })] }] });
|
|
const preview = await POST({ request: request({ dryRun: true, servers: [server] }) });
|
|
expect(preview.status).toBe(200);
|
|
const body = await preview.json();
|
|
expect(body).toMatchObject({ ok: true, plan: { servers: [server], fingerprint: expect.any(String), files: [expect.objectContaining({ relPath: 'stray.gsc' })] } });
|
|
const mutation = await POST({ request: request({ servers: [server], expectedFingerprint: body.plan.fingerprint }) });
|
|
expect(mutation.status).toBe(410);
|
|
expect(await mutation.json()).toMatchObject({ ok: false, watchdogUrl: '/watchdog', error: expect.stringContaining('Watchdog') });
|
|
expect(await readFile(path, 'utf8')).toBe('unexpected content');
|
|
expect(operations()).toEqual([]);
|
|
expect(database.db().prepare('SELECT id FROM deployments').all()).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe('Watchdog drift campaigns', () => {
|
|
it('queues an audited rebuild without changing files until provisioning executes and retains its campaign through cleanup', async () => {
|
|
await mkdir(join(fixture.t5, server, 'configs'), { recursive: true });
|
|
const path = join(fixture.t5, server, 'configs', 'stray.cfg');
|
|
await writeFile(path, 'unexpected content');
|
|
await watchdog.watchdogTick(true);
|
|
const row = onlyIncident();
|
|
expect(row).toMatchObject({ kind: 'drift', state: 'recovering' });
|
|
const action = watchdog.watchdogDetail(row.id)!.actions[0];
|
|
expect(action).toMatchObject({ action: 'rebuild', campaign_id: expect.any(String), operation_state: 'waiting' });
|
|
expect(await readFile(path, 'utf8')).toBe('unexpected content');
|
|
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
|
expect(database.db().prepare('SELECT rel_path, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(action.campaign_id))
|
|
.toEqual([{ rel_path: 'stray.cfg', content_before: Buffer.from('unexpected content'), content_after: null }]);
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
down();
|
|
await processProvisioning();
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'queued' })]);
|
|
await expect(access(path)).rejects.toThrow();
|
|
expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
|
database.closeDb();
|
|
await sweep(); await watchdog.watchdogTick(true);
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
|
expect(database.db().prepare('SELECT provisioning_id, restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id))
|
|
.toEqual({ provisioning_id: action.operation_id, restart_state: 'pending' });
|
|
await processCreations();
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
advance(1);
|
|
await processCreations(); await sweep(); await watchdog.watchdogTick(true);
|
|
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
|
expect(database.db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id)).toEqual({ restart_state: 'done' });
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'done' })]);
|
|
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('keeps a missing rebuild key visible as attention without deleting files or retrying automatically', async () => {
|
|
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
removeServerKey(server);
|
|
const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc');
|
|
await writeFile(path, 'unexpected');
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('Save a server key') });
|
|
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
expect(await readFile(path, 'utf8')).toBe('unexpected');
|
|
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe('Maintenance recovery policy boundaries', () => {
|
|
it('separates health recovery from drift repair and holds an already queued drift rebuild', async () => {
|
|
const policies = await import('../../src/lib/server/maintenance/policies.js');
|
|
policies.setMaintenancePolicy('health', false);
|
|
down();
|
|
for (let n = 0; n < 3; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
|
expect(operations()).toEqual([]);
|
|
expect(onlyIncident().kind).toBe('health');
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'drift');
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
|
|
policies.setMaintenancePolicy('drift', false);
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
expect(operations()[0].state).toBe('waiting');
|
|
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
|
});
|
|
it('global pause blocks escalation without disabling the saved health policy', async () => {
|
|
await sustainedFailure();
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
const policies = await import('../../src/lib/server/maintenance/policies.js');
|
|
policies.pauseMaintenance(true);
|
|
advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true);
|
|
expect(operations()).toHaveLength(1);
|
|
expect(policies.maintenancePolicies().health).toBe(true);
|
|
policies.pauseMaintenance(false); await watchdog.watchdogTick(true);
|
|
expect(operations()).toHaveLength(2);
|
|
expect(operations()[1].action).toBe('rebuild');
|
|
});
|
|
});
|