Remove Komodo backend in favor of native hosts
This commit is contained in:
+26
-362
@@ -1,14 +1,10 @@
|
||||
import { prepareNativeHost } from '../helpers/native-host.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { access, mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { gameEvent } from '../helpers/telemetry.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
|
||||
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
@@ -20,7 +16,8 @@ let fixture: Fixture;
|
||||
type Engine = typeof import('../../src/lib/server/watchdog/index.js');
|
||||
let watchdog: Engine;
|
||||
let database: typeof import('../../src/lib/server/db/index.js');
|
||||
let komodo: typeof import('../../src/lib/server/integrations/komodo.js');
|
||||
let backend: typeof import('../../src/lib/server/backends/native.js');
|
||||
let refreshHost: () => unknown;
|
||||
let rcon: typeof import('../../src/lib/server/integrations/rcon.js');
|
||||
|
||||
beforeEach(async () => {
|
||||
@@ -29,22 +26,18 @@ beforeEach(async () => {
|
||||
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
||||
fixture = await newFixture([server]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
||||
for (const [key, value] of Object.entries({ RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1' })) vi.stubEnv(key, value);
|
||||
database = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
database.db().prepare('UPDATE servers SET game_port = 28960 WHERE name = ?').run(server);
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) {
|
||||
vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
|
||||
}
|
||||
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
refreshHost = (await prepareNativeHost([server])).heartbeat;
|
||||
backend = await import('../../src/lib/server/backends/native.js');
|
||||
vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
|
||||
const { mockAgentFiles } = await import('../helpers/agent-files.js');
|
||||
await mockAgentFiles();
|
||||
rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
||||
@@ -56,8 +49,8 @@ afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAl
|
||||
const incidents = () => watchdog.watchdogOverview().incidents;
|
||||
const onlyIncident = () => { expect(incidents()).toHaveLength(1); return incidents()[0]; };
|
||||
const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[];
|
||||
function advance(ms: number) { vi.setSystemTime(Date.now() + ms); }
|
||||
function down() { vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
|
||||
function advance(ms: number) { vi.setSystemTime(Date.now() + ms); refreshHost(); }
|
||||
function down() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
|
||||
async function pollStatus() {
|
||||
const worker = await vi.importActual<typeof import('../../src/lib/server/pipeline/worker.js')>('../../src/lib/server/pipeline/worker.js');
|
||||
await worker.fanoutStatus();
|
||||
@@ -78,7 +71,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
await watchdog.watchdogTick();
|
||||
advance(watchdog.WATCHDOG_SCAN_MS);
|
||||
}
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
||||
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(operations()).toEqual([]);
|
||||
@@ -98,7 +91,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
}
|
||||
expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
|
||||
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(3);
|
||||
expect(backend.nativeHealth).toHaveBeenCalledTimes(3);
|
||||
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0);
|
||||
});
|
||||
|
||||
@@ -113,7 +106,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
advance(60_000);
|
||||
await watchdog.watchdogTick();
|
||||
expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at });
|
||||
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(1);
|
||||
expect(backend.nativeHealth).toHaveBeenCalledTimes(1);
|
||||
advance(90_001);
|
||||
await pollStatus();
|
||||
await watchdog.watchdogTick();
|
||||
@@ -205,7 +198,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
await watchdog.watchdogTick();
|
||||
}
|
||||
expect(scan).toHaveBeenCalledTimes(1);
|
||||
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(5);
|
||||
expect(backend.nativeHealth).toHaveBeenCalledTimes(5);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS);
|
||||
await pollStatus();
|
||||
await watchdog.watchdogTick();
|
||||
@@ -256,32 +249,11 @@ describe('Watchdog detection and ownership', () => {
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions).toEqual([expect.objectContaining({ action: 'restart', operation_state: 'waiting' })]);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps paused recovery observational and resumes one pending incident', async () => {
|
||||
watchdog.setWatchdogEnabled(false);
|
||||
down();
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(onlyIncident()).toMatchObject({ state: 'open', failures: 4 });
|
||||
expect(operations()).toEqual([]);
|
||||
database.closeDb();
|
||||
expect(watchdog.watchdogOverview().enabled).toBe(false);
|
||||
watchdog.setWatchdogEnabled(true);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
watchdog.setWatchdogEnabled(false);
|
||||
await processProvisioning();
|
||||
expect(operations()[0].state).toBe('waiting');
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
watchdog.setWatchdogEnabled(true);
|
||||
await processProvisioning();
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not restart an unknown Komodo state even after repeated sustained observations', async () => {
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo request timed out' });
|
||||
it('does not restart an unknown host agent state even after repeated sustained observations', async () => {
|
||||
vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, reason: 'host agent request timed out' });
|
||||
for (let n = 0; n < 8; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' });
|
||||
expect(operations()).toEqual([]);
|
||||
@@ -297,7 +269,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
});
|
||||
|
||||
it('allows a longer initial startup grace period before requesting recovery', async () => {
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
|
||||
vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
|
||||
for (let n = 0; n < 5; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(operations()).toEqual([]);
|
||||
await watchdog.watchdogTick(true);
|
||||
@@ -346,79 +318,6 @@ describe('Watchdog detection and ownership', () => {
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
||||
});
|
||||
|
||||
it.each([0, 2])('cancels queued health recovery when fresh RCON responds with %i players', async players => {
|
||||
const row = await sustainedFailure();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(players);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('holds queued health recovery when container readiness becomes unknown', async () => {
|
||||
await sustainedFailure();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(operations()[0].state).toBe('waiting');
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not apply a recovered occupancy probe after its operation is replaced', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const restart = operations()[0];
|
||||
const { cancelProvisioning, requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockImplementationOnce(async () => {
|
||||
expect(cancelProvisioning(restart.id)).toBe(true);
|
||||
expect(requestLifecycle(server, 'stop').ok).toBe(true);
|
||||
return 0;
|
||||
});
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' }),
|
||||
expect.objectContaining({ action: 'stop', state: 'waiting' })]);
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('cancels obsolete health recovery so a later queued maintenance restart can proceed', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const job = database.db().prepare('INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)')
|
||||
.run(server, new Date().toISOString()).lastInsertRowid;
|
||||
const maintenance = requestRestart(server, { kind: 'maintenance', id: String(job) });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(operations()).toEqual([expect.objectContaining({ state: 'cancelled' }),
|
||||
expect.objectContaining({ id: maintenance.id, state: 'awaiting-health' })]);
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each(['restart', 'rebuild'] as const)('preserves an operator request adopting a queued health %s', async action => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
if (action === 'rebuild') {
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
}
|
||||
const queued = operations().at(-1)!;
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
expect(requestLifecycle(server, action)).toEqual({ ok: true, deploymentId: queued.id });
|
||||
expect(database.db().prepare('SELECT triggered_by FROM server_deployments WHERE id = ?').get(queued.id)).toEqual({ triggered_by: 'ui' });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
if (action === 'rebuild') vi.mocked(komodo.komodoDestroyDeployment).mockImplementationOnce(async () => { down(); return { ok: true }; });
|
||||
await processProvisioning();
|
||||
expect(operations().at(-1)).toMatchObject({ id: queued.id, action, state: action === 'restart' ? 'awaiting-health' : 'queued' });
|
||||
if (action === 'restart') expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
else expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves intentional stop intent after completion and database reopening', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const stop = requestLifecycle(server, 'stop');
|
||||
@@ -430,7 +329,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['beta', 'campaign'] as const)('defers a server owned by %s work', async (owner) => {
|
||||
@@ -447,12 +346,12 @@ describe('Watchdog detection and ownership', () => {
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(operations()).toEqual([]);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rechecks lifecycle ownership after a health probe yields to an operator stop', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => {
|
||||
vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
|
||||
expect(requestLifecycle(server, 'stop').ok).toBe(true);
|
||||
return { ok: false, state: 'exited', reason: 'Container exited' };
|
||||
});
|
||||
@@ -477,7 +376,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
advance(watchdog.WATCHDOG_SCAN_MS);
|
||||
}
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
||||
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
||||
expect(operations()).toHaveLength(1);
|
||||
|
||||
@@ -495,7 +394,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
|
||||
it('stops probing when provisioning takes ownership during the container inspection', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => {
|
||||
vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
|
||||
expect(requestLifecycle(server, 'restart').ok).toBe(true);
|
||||
return { ok: true, state: 'running' };
|
||||
});
|
||||
@@ -507,173 +406,6 @@ describe('Watchdog detection and ownership', () => {
|
||||
});
|
||||
|
||||
describe('Watchdog recovery persistence', () => {
|
||||
it('verifies recovery at the deadline before deciding whether to rebuild', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'done' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ healthy_at: new Date().toISOString() });
|
||||
});
|
||||
|
||||
it('defers escalation while current container health is unknown', async () => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'awaiting-health' })]);
|
||||
});
|
||||
|
||||
it('keeps failed submission history when fresh evidence later releases recovery', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
vi.mocked(komodo.komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Reply lost' });
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'failed' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ healthy_at: new Date().toISOString() });
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions[0].operation_error).toContain('Reply lost');
|
||||
});
|
||||
|
||||
it('discards escalation evidence if another lifecycle submission appears during its probe', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rcon.rconStatusResult).mockImplementationOnce(async () => {
|
||||
database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
||||
VALUES ('newer', ?, 'restart', 'done', 'Complete', ?, ?, 'manual')`).run(server, new Date().toISOString(), new Date().toISOString());
|
||||
return healthyRcon;
|
||||
});
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()[0]).toMatchObject({ action: 'restart', state: 'awaiting-health' });
|
||||
expect(operations().some(op => op.action === 'rebuild')).toBe(false);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
||||
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
||||
});
|
||||
|
||||
it('cancels a queued health rebuild and releases its transferred slot when the game recovers', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
const rebuild = operations()[1];
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await processProvisioning();
|
||||
expect(operations()).toEqual([expect.objectContaining({ state: 'failed' }), expect.objectContaining({ state: 'cancelled' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: rebuild.id, healthy_at: new Date().toISOString() });
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves the original finding as deleted history after an intentional successful destroy', async () => {
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
removeServerKey(server);
|
||||
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
|
||||
await watchdog.watchdogTick(true);
|
||||
const id = onlyIncident().id;
|
||||
expect(onlyIncident().state).toBe('attention');
|
||||
expect(destroyServer(server)).toMatchObject({ ok: true });
|
||||
down();
|
||||
await processDestructions();
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'destroy', state: 'done' })]);
|
||||
expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(id)?.incident).toMatchObject({ state: 'attention', server_deleted_at: expect.any(String) });
|
||||
expect(watchdog.updateWatchdogIncident(id, 'retry')).toMatchObject({ ok: false, status: 409 });
|
||||
});
|
||||
|
||||
it('requests one restart, escalates once to rebuild, and retains exhausted attention across reopening', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(operations()[0].state).toBe('awaiting-health');
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS - 1);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
advance(1);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(2);
|
||||
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' });
|
||||
await processProvisioning();
|
||||
expect(operations()[1].state).toBe('failed');
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention');
|
||||
database.closeDb();
|
||||
for (let n = 0; n < 4; n++) { advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true); }
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention');
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not escalate a timed out restart while automatic recovery is paused', async () => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
watchdog.setWatchdogEnabled(false);
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
watchdog.setWatchdogEnabled(true);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
});
|
||||
|
||||
it('reclaims a failed attempt recovery slot only after an explicit incident retry', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' });
|
||||
await processProvisioning();
|
||||
await watchdog.watchdogTick(true);
|
||||
const failed = operations()[1];
|
||||
expect(failed).toMatchObject({ action: 'rebuild', state: 'failed' });
|
||||
expect(onlyIncident()).toMatchObject({ id: row.id, state: 'attention', cycle: 1 });
|
||||
const slot = () => database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server);
|
||||
expect(slot()).toEqual({ operation_id: failed.id, healthy_at: null });
|
||||
|
||||
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockClear();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(operations()).toHaveLength(2);
|
||||
|
||||
expect(watchdog.updateWatchdogIncident(row.id, 'retry')).toEqual({ ok: true });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(2);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
const retry = operations()[2];
|
||||
expect(retry).toMatchObject({ action: 'restart', state: 'waiting' });
|
||||
expect(onlyIncident()).toMatchObject({ id: row.id, state: 'recovering', cycle: 2 });
|
||||
expect(slot()).toEqual({ operation_id: retry.id, healthy_at: null });
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(3);
|
||||
|
||||
await processProvisioning();
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect(operations()[2].state).toBe('awaiting-health');
|
||||
});
|
||||
|
||||
it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
|
||||
down();
|
||||
@@ -770,41 +502,6 @@ describe('Watchdog APIs', () => {
|
||||
});
|
||||
|
||||
describe('Watchdog drift campaigns', () => {
|
||||
it('queues an audited rebuild without changing files until provisioning executes and retains its campaign through cleanup', async () => {
|
||||
await mkdir(join(fixture.t5, server, 'configs'), { recursive: true });
|
||||
const path = join(fixture.t5, server, 'configs', 'stray.cfg');
|
||||
await writeFile(path, 'unexpected content');
|
||||
await watchdog.watchdogTick(true);
|
||||
const row = onlyIncident();
|
||||
expect(row).toMatchObject({ kind: 'drift', state: 'recovering' });
|
||||
const action = watchdog.watchdogDetail(row.id)!.actions[0];
|
||||
expect(action).toMatchObject({ action: 'rebuild', campaign_id: expect.any(String), operation_state: 'waiting' });
|
||||
expect(await readFile(path, 'utf8')).toBe('unexpected content');
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(database.db().prepare('SELECT rel_path, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(action.campaign_id))
|
||||
.toEqual([{ rel_path: 'stray.cfg', content_before: Buffer.from('unexpected content'), content_after: null }]);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
down();
|
||||
await processProvisioning();
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'queued' })]);
|
||||
await expect(access(path)).rejects.toThrow();
|
||||
expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
||||
database.closeDb();
|
||||
await sweep(); await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
||||
expect(database.db().prepare('SELECT provisioning_id, restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id))
|
||||
.toEqual({ provisioning_id: action.operation_id, restart_state: 'pending' });
|
||||
await processCreations();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
advance(1);
|
||||
await processCreations(); await sweep(); await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id)).toEqual({ restart_state: 'done' });
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'done' })]);
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps a missing rebuild key visible as attention without deleting files or retrying automatically', async () => {
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
@@ -812,43 +509,10 @@ describe('Watchdog drift campaigns', () => {
|
||||
const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc');
|
||||
await writeFile(path, 'unexpected');
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('Save a server key') });
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('server key are required') });
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([]);
|
||||
expect(await readFile(path, 'utf8')).toBe('unexpected');
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Maintenance recovery policy boundaries', () => {
|
||||
it('separates health recovery from drift repair and holds an already queued drift rebuild', async () => {
|
||||
const policies = await import('../../src/lib/server/maintenance/policies.js');
|
||||
policies.setMaintenancePolicy('health', false);
|
||||
down();
|
||||
for (let n = 0; n < 3; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(operations()).toEqual([]);
|
||||
expect(onlyIncident().kind).toBe('health');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'drift');
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
|
||||
policies.setMaintenancePolicy('drift', false);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(operations()[0].state).toBe('waiting');
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
it('global pause blocks escalation without disabling the saved health policy', async () => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
const policies = await import('../../src/lib/server/maintenance/policies.js');
|
||||
policies.pauseMaintenance(true);
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
expect(policies.maintenancePolicies().health).toBe(true);
|
||||
policies.pauseMaintenance(false); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(2);
|
||||
expect(operations()[1].action).toBe('rebuild');
|
||||
expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user