1029 lines
43 KiB
TypeScript
1029 lines
43 KiB
TypeScript
import { prepareNativeHost } from '../helpers/native-host.js';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { gameEvent } from '../helpers/telemetry.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconGetPlayerCount: vi.fn(),
|
|
rconStatusResult: vi.fn(),
|
|
rconPlutoniumVersion: vi.fn()
|
|
}));
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
|
|
|
const server = 't5_zombie_theater_1';
|
|
const healthyRcon = {
|
|
ok: true as const,
|
|
endpoint: { host: 'test', port: 28960 },
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
};
|
|
const missingRcon = {
|
|
ok: false as const,
|
|
endpoint: { host: 'test', port: 28960 },
|
|
reason: 'timeout' as const,
|
|
detail: 'No reply'
|
|
};
|
|
let fixture: Fixture;
|
|
type Engine = typeof import('../../src/lib/server/watchdog/index.js');
|
|
let watchdog: Engine;
|
|
let database: typeof import('../../src/lib/server/db/index.js');
|
|
let backend: typeof import('../../src/lib/server/backends/native.js');
|
|
let refreshHost: () => unknown;
|
|
let rcon: typeof import('../../src/lib/server/integrations/rcon.js');
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.resetAllMocks();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
|
fixture = await newFixture([server]);
|
|
vi.stubGlobal(
|
|
'fetch',
|
|
vi.fn(async () =>
|
|
Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })
|
|
)
|
|
);
|
|
for (const [key, value] of Object.entries({
|
|
RCON_PASSWORD: 'test',
|
|
DEPLOYMENT_CREATE_CONCURRENCY: '1',
|
|
DEPLOYMENT_RESTART_CONCURRENCY: '1'
|
|
}))
|
|
vi.stubEnv(key, value);
|
|
database = await import('../../src/lib/server/db/index.js');
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
database.db().prepare('UPDATE servers SET game_port = 28960 WHERE name = ?').run(server);
|
|
const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
reserveServerKey(server, 't5', 'saved-key');
|
|
refreshHost = (await prepareNativeHost([server])).heartbeat;
|
|
backend = await import('../../src/lib/server/backends/native.js');
|
|
vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
|
|
const { mockAgentFiles } = await import('../helpers/agent-files.js');
|
|
await mockAgentFiles();
|
|
rcon = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
|
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
|
watchdog = await import('../../src/lib/server/watchdog/index.js');
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
vi.useRealTimers();
|
|
vi.unstubAllEnvs();
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
const incidents = () => watchdog.watchdogOverview().incidents;
|
|
const onlyIncident = () => {
|
|
expect(incidents()).toHaveLength(1);
|
|
return incidents()[0];
|
|
};
|
|
const operations = () =>
|
|
database
|
|
.db()
|
|
.prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid')
|
|
.all() as { id: string; action: string; state: string }[];
|
|
function advance(ms: number) {
|
|
vi.setSystemTime(Date.now() + ms);
|
|
refreshHost();
|
|
}
|
|
function down() {
|
|
vi.mocked(backend.nativeHealth).mockResolvedValue({
|
|
ok: false,
|
|
state: 'exited',
|
|
reason: 'Container exited'
|
|
});
|
|
}
|
|
function missing() {
|
|
vi.mocked(backend.nativeHealth).mockResolvedValue({
|
|
ok: false,
|
|
state: 'missing',
|
|
reason: 'Container missing'
|
|
});
|
|
}
|
|
async function pollStatus() {
|
|
const worker = await vi.importActual<typeof import('../../src/lib/server/pipeline/worker.js')>(
|
|
'../../src/lib/server/pipeline/worker.js'
|
|
);
|
|
await worker.fanoutStatus();
|
|
}
|
|
async function sustainedFailure() {
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([
|
|
expect.objectContaining({ action: 'restart', state: 'waiting' })
|
|
]);
|
|
return onlyIncident();
|
|
}
|
|
|
|
describe('Watchdog detection and ownership', () => {
|
|
it.each([false, true])(
|
|
'rescans legacy bank drift and preserves real differences (mixed=%s)',
|
|
async (mixed) => {
|
|
const { mkdir } = await import('node:fs/promises');
|
|
const { destOfKind } = await import('../../src/lib/server/config.js');
|
|
const { setMaintenancePolicy, pauseMaintenance } =
|
|
await import('../../src/lib/server/maintenance/policies.js');
|
|
setMaintenancePolicy('drift', false);
|
|
pauseMaintenance(true);
|
|
const dest = destOfKind(server, 'config');
|
|
await mkdir(dest, { recursive: true });
|
|
const secret = 'set tws_bank_token "ec_bank_watchdog_private"';
|
|
await writeFile(join(dest, 'electric_cherry_bank.cfg'), secret);
|
|
if (mixed) await writeFile(join(dest, 'stray.cfg'), 'real drift');
|
|
database
|
|
.db()
|
|
.prepare(
|
|
`INSERT INTO watchdog_incidents
|
|
(id,server,kind,state,summary,details,first_seen_at,last_seen_at,failures)
|
|
VALUES('legacy-bank',?,'drift','open','Old drift',?,?,?,1)`
|
|
)
|
|
.run(
|
|
server,
|
|
JSON.stringify({
|
|
files: [{ kind: 'config', relPath: 'electric_cherry_bank.cfg' }]
|
|
}),
|
|
new Date().toISOString(),
|
|
new Date().toISOString()
|
|
);
|
|
await watchdog.watchdogTick(true);
|
|
const row = database
|
|
.db()
|
|
.prepare('SELECT state,details FROM watchdog_incidents WHERE id=?')
|
|
.get('legacy-bank') as { state: string; details: string };
|
|
expect(row.state).toBe(mixed ? 'open' : 'resolved');
|
|
expect(
|
|
JSON.parse(row.details).files.map((file: { relPath: string }) => file.relPath)
|
|
).toEqual(mixed ? ['stray.cfg'] : []);
|
|
expect(operations()).toEqual([]);
|
|
expect(
|
|
database
|
|
.db()
|
|
.prepare('SELECT 1 FROM file_blobs WHERE instr(content,?)>0')
|
|
.all(Buffer.from(secret))
|
|
).toEqual([]);
|
|
expect(await readFile(join(dest, 'electric_cherry_bank.cfg'), 'utf8')).toBe(secret);
|
|
}
|
|
);
|
|
|
|
it('waits for status-monitor evidence without issuing its own routine probes', async () => {
|
|
down();
|
|
for (let minute = 0; minute < 4; minute++) {
|
|
await watchdog.watchdogTick();
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
expect(incidents()).toEqual([]);
|
|
expect(operations()).toEqual([]);
|
|
});
|
|
|
|
it.each(['container', 'game'])(
|
|
'requests recovery from sustained %s failures collected by status polling',
|
|
async (kind) => {
|
|
if (kind === 'container') down();
|
|
else vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
|
for (let minute = 0; minute < 3; minute++) {
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
if (minute < 2) {
|
|
expect(onlyIncident().failures).toBe(minute + 1);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
}
|
|
expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
|
|
expect(operations()).toEqual([
|
|
expect.objectContaining({ action: 'restart', state: 'waiting' })
|
|
]);
|
|
expect(backend.nativeHealth).toHaveBeenCalledTimes(3);
|
|
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0);
|
|
}
|
|
);
|
|
|
|
it.each([false, true])(
|
|
'rebuilds a confirmed missing container once after three samples over two minutes (manual=%s)',
|
|
async (manual) => {
|
|
missing();
|
|
const drift = await import('../../src/lib/server/fleet/drift.js');
|
|
const scan = vi.spyOn(drift, 'prepareDriftCampaign');
|
|
for (let sample = 0; sample < 3; sample++) {
|
|
if (!manual) await pollStatus();
|
|
await watchdog.watchdogTick(manual);
|
|
expect(onlyIncident().failures).toBe(sample + 1);
|
|
if (sample < 2) {
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
}
|
|
const incident = onlyIncident();
|
|
const queued = operations();
|
|
expect(incident).toMatchObject({ kind: 'health', state: 'recovering', failures: 3 });
|
|
expect(queued).toEqual([
|
|
expect.objectContaining({ action: 'rebuild', state: 'waiting' })
|
|
]);
|
|
expect(watchdog.watchdogDetail(incident.id)?.actions).toEqual([
|
|
expect.objectContaining({
|
|
action: 'rebuild',
|
|
operation_id: queued[0].id,
|
|
campaign_id: null
|
|
})
|
|
]);
|
|
expect(
|
|
database
|
|
.db()
|
|
.prepare('SELECT kind,source_id FROM provisioning_sources WHERE operation_id=?')
|
|
.all(queued[0].id)
|
|
).toEqual([{ kind: 'watchdog', source_id: incident.id }]);
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
expect(scan).not.toHaveBeenCalled();
|
|
expect(backend.nativeRequest).not.toHaveBeenCalled();
|
|
database.closeDb();
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual(queued);
|
|
expect(watchdog.watchdogDetail(incident.id)?.actions).toHaveLength(1);
|
|
}
|
|
);
|
|
|
|
it('does not accelerate missing-container recovery through repeated manual checks', async () => {
|
|
missing();
|
|
await watchdog.watchdogTick(true);
|
|
for (let check = 0; check < 5; check++) await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().failures).toBe(1);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS - 1);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
advance(1);
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().failures).toBe(3);
|
|
expect(operations()).toEqual([
|
|
expect.objectContaining({ action: 'rebuild', state: 'waiting' })
|
|
]);
|
|
});
|
|
|
|
it('dispatches missing-container recovery as a rebuild without requesting an empty-player count', async () => {
|
|
missing();
|
|
for (let sample = 0; sample < 3; sample++) {
|
|
await watchdog.watchdogTick(true);
|
|
if (sample < 2) advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
const operation = operations()[0];
|
|
expect(operation).toMatchObject({ action: 'rebuild', state: 'waiting' });
|
|
const readFiles = vi.mocked(backend.nativeRequest).getMockImplementation()!;
|
|
vi.mocked(backend.nativeRequest).mockImplementation(async (instance, kind, payload) => {
|
|
if (kind === 'instance.inspect')
|
|
return { exists: false, healthy: false, state: 'missing' } as never;
|
|
return readFiles(instance, kind, payload);
|
|
});
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
const binding = database
|
|
.db()
|
|
.prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?')
|
|
.get(operation.id) as { operation_id: string | null };
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
expect(binding.operation_id).toEqual(expect.any(String));
|
|
expect(getHostOperation(binding.operation_id!)).toMatchObject({
|
|
kind: 'instance.rebuild',
|
|
generation: 1,
|
|
payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } }
|
|
});
|
|
expect(
|
|
database
|
|
.db()
|
|
.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'")
|
|
.all()
|
|
).toEqual([]);
|
|
expect(rcon.rconGetPlayerCount).not.toHaveBeenCalled();
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it.each(['health disabled', 'maintenance paused'])(
|
|
'keeps missing containers observational with %s even when files have drifted',
|
|
async (policy) => {
|
|
missing();
|
|
const policies = await import('../../src/lib/server/maintenance/policies.js');
|
|
policies.setMaintenancePolicy('drift', true);
|
|
if (policy === 'health disabled') policies.setMaintenancePolicy('health', false);
|
|
else policies.pauseMaintenance(true);
|
|
await writeFile(
|
|
join(
|
|
fixture.t5,
|
|
server,
|
|
'runtime/plutonium/storage/t5/raw/scripts/sp',
|
|
'stray.gsc'
|
|
),
|
|
'unexpected'
|
|
);
|
|
const drift = await import('../../src/lib/server/fleet/drift.js');
|
|
const scan = vi.spyOn(drift, 'prepareDriftCampaign');
|
|
for (let sample = 0; sample < 4; sample++) {
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 4 });
|
|
expect(operations()).toEqual([]);
|
|
expect(scan).not.toHaveBeenCalled();
|
|
expect(backend.nativeRequest).not.toHaveBeenCalled();
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
policies.setMaintenancePolicy('health', true);
|
|
policies.pauseMaintenance(false);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([
|
|
expect.objectContaining({ action: 'rebuild', state: 'waiting' })
|
|
]);
|
|
}
|
|
);
|
|
|
|
it('does not count the same sample again or use stale data, and resets after a monitoring gap', async () => {
|
|
down();
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
const first = onlyIncident();
|
|
advance(30_000);
|
|
database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
|
|
await watchdog.watchdogTick();
|
|
advance(60_000);
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at });
|
|
expect(backend.nativeHealth).toHaveBeenCalledTimes(1);
|
|
advance(90_001);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident()).toMatchObject({
|
|
failures: 1,
|
|
first_seen_at: new Date().toISOString()
|
|
});
|
|
expect(operations()).toEqual([]);
|
|
});
|
|
|
|
it('uses sample times across polling offsets without counting early or losing sustained failures', async () => {
|
|
down();
|
|
await pollStatus();
|
|
advance(20_000);
|
|
await watchdog.watchdogTick();
|
|
advance(10_000);
|
|
await pollStatus();
|
|
advance(50_000);
|
|
// The first failure is stale until the monitor supplies another sample.
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident().failures).toBe(2);
|
|
advance(40_000);
|
|
await pollStatus();
|
|
advance(20_000);
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
|
|
expect(operations()).toHaveLength(1);
|
|
});
|
|
|
|
it('breaks a routine failure streak on a successful status poll between watchdog passes', async () => {
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
advance(30_000);
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
|
|
await pollStatus();
|
|
advance(30_000);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident()).toMatchObject({
|
|
failures: 1,
|
|
first_seen_at: new Date().toISOString()
|
|
});
|
|
expect(operations()).toEqual([]);
|
|
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it.each([0, 4])(
|
|
'uses fresh telemetry with %i players without extra Watchdog probes and waits for polled RCON when telemetry expires',
|
|
async (players) => {
|
|
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
|
saveTelemetry({
|
|
...gameEvent('heartbeat', 17, players, server),
|
|
receivedAt: new Date().toISOString()
|
|
});
|
|
// Status polling supplements occupied rosters with pings; Watchdog adds no probes.
|
|
const supplementaryProbes = players > 0 ? 1 : 0;
|
|
await pollStatus();
|
|
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes);
|
|
await watchdog.watchdogTick();
|
|
expect(incidents()).toEqual([]);
|
|
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick();
|
|
expect(incidents()).toEqual([]);
|
|
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes);
|
|
advance(30_001);
|
|
await pollStatus();
|
|
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes + 1);
|
|
advance(30_000);
|
|
database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open' });
|
|
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes + 1);
|
|
expect(operations()).toEqual([]);
|
|
}
|
|
);
|
|
|
|
it('starts a new failure window for a replacement registration', async () => {
|
|
down();
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident().failures).toBe(2);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
database
|
|
.db()
|
|
.prepare('UPDATE servers SET created_at=? WHERE name=?')
|
|
.run(new Date().toISOString(), server);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident()).toMatchObject({
|
|
failures: 1,
|
|
first_seen_at: new Date().toISOString()
|
|
});
|
|
expect(operations()).toEqual([]);
|
|
});
|
|
|
|
it('scans files every five minutes while retaining minute health checks and immediate manual scans', async () => {
|
|
const drift = await import('../../src/lib/server/fleet/drift.js');
|
|
const scan = vi.spyOn(drift, 'prepareDriftCampaign');
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(scan).toHaveBeenCalledTimes(1);
|
|
for (let minute = 1; minute < 5; minute++) {
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
}
|
|
expect(scan).toHaveBeenCalledTimes(1);
|
|
expect(backend.nativeHealth).toHaveBeenCalledTimes(5);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(scan).toHaveBeenCalledTimes(2);
|
|
await watchdog.watchdogTick(true);
|
|
expect(scan).toHaveBeenCalledTimes(3);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
database
|
|
.db()
|
|
.prepare('UPDATE servers SET created_at=? WHERE name=?')
|
|
.run(new Date().toISOString(), server);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(scan).toHaveBeenCalledTimes(4);
|
|
scan.mockRestore();
|
|
});
|
|
|
|
it('retries failed file scans on the next health scan instead of retaining successful freshness', async () => {
|
|
const drift = await import('../../src/lib/server/fleet/drift.js');
|
|
const scan = vi
|
|
.spyOn(drift, 'prepareDriftCampaign')
|
|
.mockRejectedValueOnce(new Error('Agent unavailable'));
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(watchdog.watchdogOverview().scanError).toContain('Agent unavailable');
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(scan).toHaveBeenCalledTimes(2);
|
|
expect(watchdog.watchdogOverview().scanError).toBeNull();
|
|
scan.mockRestore();
|
|
});
|
|
|
|
it('defaults to automatic recovery and debounces manual scans without accelerating the failure window', async () => {
|
|
expect(watchdog.watchdogOverview().enabled).toBe(true);
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({ state: 'open', failures: 1 });
|
|
for (let n = 0; n < 5; n++) await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().failures).toBe(1);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await pollStatus();
|
|
await watchdog.watchdogTick();
|
|
expect(onlyIncident().failures).toBe(2);
|
|
advance(watchdog.WATCHDOG_SCAN_MS - 1);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
advance(1);
|
|
await watchdog.watchdogTick(true);
|
|
const row = onlyIncident();
|
|
expect(row).toMatchObject({ state: 'recovering', failures: 3 });
|
|
expect(watchdog.watchdogDetail(row.id)?.actions).toEqual([
|
|
expect.objectContaining({ action: 'restart', operation_state: 'waiting' })
|
|
]);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toHaveLength(1);
|
|
expect(
|
|
database
|
|
.db()
|
|
.prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'")
|
|
.all()
|
|
).toEqual([]);
|
|
});
|
|
|
|
it.each([
|
|
['unavailable', undefined, 'host agent request timed out'],
|
|
['unknown', 'unknown', 'Agent could not inspect the container'],
|
|
['offline', 'unknown', 'Host agent is disconnected']
|
|
] as const)(
|
|
'does not recover an %s host agent state even after repeated sustained observations',
|
|
async (_label, state, reason) => {
|
|
vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state, reason });
|
|
for (let n = 0; n < 8; n++) {
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' });
|
|
expect(operations()).toEqual([]);
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().failures).toBe(1);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
|
}
|
|
);
|
|
|
|
it('allows a longer initial startup grace period before requesting recovery', async () => {
|
|
vi.mocked(backend.nativeHealth).mockResolvedValue({
|
|
ok: false,
|
|
state: 'starting',
|
|
reason: 'Starting'
|
|
});
|
|
for (let n = 0; n < 5; n++) {
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
expect(operations()).toEqual([]);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
|
});
|
|
|
|
it('accepts fresh validated telemetry when RCON is unavailable and rechecks after it becomes stale', async () => {
|
|
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
|
saveTelemetry({
|
|
...gameEvent('heartbeat', 17, 4, server),
|
|
receivedAt: new Date().toISOString()
|
|
});
|
|
await watchdog.watchdogTick(true);
|
|
expect(incidents()).toEqual([]);
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
advance(90_001);
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 1 });
|
|
saveTelemetry({
|
|
...gameEvent('heartbeat', 18, 4, server),
|
|
receivedAt: new Date().toISOString()
|
|
});
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().state).toBe('resolved');
|
|
expect(operations()).toEqual([]);
|
|
});
|
|
|
|
it.each(['no-config', 'no-port', 'bad-reply', 'no-players'] as const)(
|
|
'keeps %s RCON failures observational',
|
|
async (reason) => {
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue({ ...missingRcon, reason });
|
|
for (let n = 0; n < 4; n++) {
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
expect(onlyIncident()).toMatchObject({ state: 'open' });
|
|
expect(operations()).toEqual([]);
|
|
}
|
|
);
|
|
|
|
it('breaks the sustained failure window on a successful probe between watchdog samples', async () => {
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
advance(20_000);
|
|
const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
|
|
await POST({ params: { name: server } } as Parameters<typeof POST>[0]);
|
|
advance(40_000);
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({
|
|
failures: 1,
|
|
state: 'open',
|
|
first_seen_at: new Date().toISOString()
|
|
});
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
|
});
|
|
|
|
it.each(['exited', 'missing'])(
|
|
'preserves intentional stop intent for %s containers after completion and database reopening',
|
|
async (state) => {
|
|
const { requestLifecycle } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
const stop = requestLifecycle(server, 'stop');
|
|
expect(stop.ok).toBe(true);
|
|
if (!stop.ok) throw new Error(stop.error);
|
|
database
|
|
.db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET state = 'done', finished_at = ? WHERE id = ?"
|
|
)
|
|
.run(new Date().toISOString(), stop.deploymentId);
|
|
if (state === 'missing') missing();
|
|
else down();
|
|
database.closeDb();
|
|
for (let n = 0; n < 4; n++) {
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
expect(incidents()).toEqual([]);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
|
|
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
|
}
|
|
);
|
|
|
|
it.each(['beta', 'campaign'] as const)('defers a server owned by %s work', async (owner) => {
|
|
if (owner === 'beta') {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { script, version } = createScript({
|
|
releaseVersion: '1.0.0',
|
|
name: 'beta',
|
|
content: Buffer.from('experimental'),
|
|
targetRelPath: 'beta.gsc'
|
|
});
|
|
await openBeta({
|
|
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
|
servers: [server],
|
|
waitForHealth: false,
|
|
waitForMatch: false
|
|
});
|
|
} else {
|
|
database
|
|
.db()
|
|
.prepare(
|
|
"INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', ?)"
|
|
)
|
|
.run(new Date().toISOString());
|
|
database
|
|
.db()
|
|
.prepare(
|
|
"INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', ?, 1, 'held')"
|
|
)
|
|
.run(server);
|
|
}
|
|
down();
|
|
for (let n = 0; n < 4; n++) {
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
expect(incidents()).toEqual([]);
|
|
expect(operations()).toEqual([]);
|
|
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it.each(['exited', 'missing'])(
|
|
'rechecks lifecycle ownership after a %s probe yields to an operator stop',
|
|
async (state) => {
|
|
const { requestLifecycle } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
|
|
expect(requestLifecycle(server, 'stop').ok).toBe(true);
|
|
return { ok: false, state, reason: `Container ${state}` };
|
|
});
|
|
await watchdog.watchdogTick(true);
|
|
expect(incidents()).toEqual([]);
|
|
expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
|
|
}
|
|
);
|
|
|
|
it.each(['queued', 'waiting', 'running', 'awaiting-health', 'retained recovery'])(
|
|
'suppresses health and drift throughout %s provisioning, then resumes',
|
|
async (state) => {
|
|
const retained = state === 'retained recovery';
|
|
const now = new Date().toISOString();
|
|
database
|
|
.db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('manual-operation', ?, ?, ?, 'Operator work', ?, 'manual')`
|
|
)
|
|
.run(server, retained ? 'restart' : 'rebuild', retained ? 'failed' : state, now);
|
|
if (retained)
|
|
database
|
|
.db()
|
|
.prepare(
|
|
`INSERT INTO restart_rollout (server, operation_id, requested_at)
|
|
VALUES (?, 'manual-operation', ?)`
|
|
)
|
|
.run(server, now);
|
|
await writeFile(
|
|
join(
|
|
fixture.t5,
|
|
server,
|
|
'runtime/plutonium/storage/t5/raw/scripts/sp',
|
|
'stray.gsc'
|
|
),
|
|
'unexpected'
|
|
);
|
|
down();
|
|
for (let n = 0; n < 4; n++) {
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
}
|
|
expect(incidents()).toEqual([]);
|
|
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
expect(operations()).toHaveLength(1);
|
|
|
|
database
|
|
.db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET state = 'done' WHERE id = 'manual-operation'"
|
|
)
|
|
.run();
|
|
if (retained)
|
|
database
|
|
.db()
|
|
.prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?')
|
|
.run(new Date().toISOString(), server);
|
|
await watchdog.watchdogTick(true);
|
|
expect(incidents()).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ kind: 'health', state: 'open', failures: 1 }),
|
|
expect.objectContaining({ kind: 'drift', state: 'recovering' })
|
|
])
|
|
);
|
|
expect(operations()).toHaveLength(2);
|
|
expect(operations()[1]).toMatchObject({ action: 'rebuild', state: 'waiting' });
|
|
}
|
|
);
|
|
|
|
it('stops probing when provisioning takes ownership during the container inspection', async () => {
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
|
|
expect(requestLifecycle(server, 'restart').ok).toBe(true);
|
|
return { ok: true, state: 'running' };
|
|
});
|
|
await watchdog.watchdogTick(true);
|
|
expect(incidents()).toEqual([]);
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
expect(operations()).toEqual([
|
|
expect.objectContaining({ action: 'restart', state: 'waiting' })
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe('Watchdog recovery persistence', () => {
|
|
it('escalates a submitted restart that becomes missing after its five-minute recovery window', async () => {
|
|
const incident = await sustainedFailure();
|
|
const restart = operations()[0];
|
|
const submittedAt = new Date().toISOString();
|
|
database
|
|
.db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET state='awaiting-health',submitted_at=? WHERE id=?"
|
|
)
|
|
.run(submittedAt, restart.id);
|
|
database
|
|
.db()
|
|
.prepare('INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES(?,?,?)')
|
|
.run(server, restart.id, submittedAt);
|
|
missing();
|
|
vi.mocked(rcon.rconStatusResult).mockClear();
|
|
advance(5 * 60_000 - 1);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([{ ...restart, state: 'awaiting-health' }]);
|
|
advance(1);
|
|
await watchdog.watchdogTick(true);
|
|
const rebuilt = operations();
|
|
expect(rebuilt).toEqual([
|
|
{ ...restart, state: 'failed' },
|
|
expect.objectContaining({ action: 'rebuild', state: 'waiting' })
|
|
]);
|
|
expect(
|
|
watchdog.watchdogDetail(incident.id)?.actions.map((action) => action.action)
|
|
).toEqual(['restart', 'rebuild']);
|
|
expect(
|
|
database
|
|
.db()
|
|
.prepare('SELECT operation_id FROM restart_rollout WHERE server=?')
|
|
.get(server)
|
|
).toEqual({ operation_id: rebuilt[1].id });
|
|
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual(rebuilt);
|
|
});
|
|
|
|
it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
const id = onlyIncident().id;
|
|
database.closeDb();
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident().id).toBe(id);
|
|
const queued = operations();
|
|
expect(queued).toHaveLength(1);
|
|
database.closeDb();
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual(queued);
|
|
expect(watchdog.watchdogDetail(id)?.actions).toHaveLength(1);
|
|
});
|
|
|
|
it('permits an explicit new attempt after dismissal without silently rearming a sustained condition', async () => {
|
|
down();
|
|
await watchdog.watchdogTick(true);
|
|
const id = onlyIncident().id;
|
|
expect(watchdog.updateWatchdogIncident(id, 'dismiss')).toEqual({ ok: true });
|
|
for (let n = 0; n < 4; n++) {
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
}
|
|
expect(operations()).toEqual([]);
|
|
expect(onlyIncident().state).toBe('dismissed');
|
|
expect(watchdog.updateWatchdogIncident(id, 'retry')).toEqual({ ok: true });
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({ id, state: 'recovering', cycle: 2 });
|
|
expect(watchdog.watchdogDetail(id)?.actions).toEqual([
|
|
expect.objectContaining({ action: 'restart', cycle: 2 })
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe('Watchdog APIs', () => {
|
|
const request = (body: unknown, raw = false) =>
|
|
new Request('http://localhost/api/watchdog', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: raw ? String(body) : JSON.stringify(body)
|
|
});
|
|
|
|
it('returns overview state and validates scan actions and pause booleans', async () => {
|
|
const { GET, POST } = await import('../../src/routes/api/watchdog/+server.js');
|
|
expect(await GET().json()).toEqual({
|
|
enabled: true,
|
|
lastScanAt: null,
|
|
scanError: null,
|
|
incidents: []
|
|
});
|
|
expect((await POST({ request: request('{', true) })).status).toBe(400);
|
|
for (const body of [
|
|
null,
|
|
[],
|
|
{},
|
|
{ action: 'restart' },
|
|
{ enabled: 'false' },
|
|
{ enabled: 0 },
|
|
{ action: 'scan', enabled: 'false' }
|
|
]) {
|
|
expect((await POST({ request: request(body) })).status).toBe(400);
|
|
}
|
|
expect(await (await POST({ request: request({ enabled: false }) })).json()).toMatchObject({
|
|
enabled: false
|
|
});
|
|
database.closeDb();
|
|
expect(await GET().json()).toMatchObject({ enabled: false });
|
|
down();
|
|
const scanned = await POST({ request: request({ action: 'scan' }) });
|
|
expect(scanned.status).toBe(200);
|
|
expect(await scanned.json()).toMatchObject({
|
|
enabled: false,
|
|
lastScanAt: expect.any(String),
|
|
incidents: [expect.objectContaining({ kind: 'health', state: 'open' })]
|
|
});
|
|
expect(operations()).toEqual([]);
|
|
});
|
|
|
|
it('validates incident actions and refuses retries while provisioning is pending', async () => {
|
|
const { GET, POST } = await import('../../src/routes/api/watchdog/[id]/+server.js');
|
|
expect(GET({ params: { id: 'missing' } }).status).toBe(404);
|
|
expect(
|
|
(await POST({ params: { id: 'missing' }, request: request({ action: 'retry' }) }))
|
|
.status
|
|
).toBe(404);
|
|
for (const body of [null, {}, [], { action: 'restart' }]) {
|
|
expect((await POST({ params: { id: 'missing' }, request: request(body) })).status).toBe(
|
|
400
|
|
);
|
|
}
|
|
expect(
|
|
(await POST({ params: { id: 'missing' }, request: request('{', true) })).status
|
|
).toBe(400);
|
|
const row = await sustainedFailure();
|
|
expect(await GET({ params: { id: row.id } }).json()).toMatchObject({
|
|
incident: { id: row.id, state: 'recovering' },
|
|
actions: [expect.objectContaining({ action: 'restart' })],
|
|
events: expect.any(Array)
|
|
});
|
|
for (const action of ['retry', 'dismiss']) {
|
|
const response = await POST({ params: { id: row.id }, request: request({ action }) });
|
|
expect(response.status).toBe(409);
|
|
expect(await response.json()).toMatchObject({
|
|
error: expect.stringContaining('Provisioning is still active')
|
|
});
|
|
}
|
|
expect(operations()).toHaveLength(1);
|
|
});
|
|
|
|
it('keeps legacy drift previews readable and directs mutations to Watchdog without changing files', async () => {
|
|
const { GET, POST } = await import('../../src/routes/api/drift/+server.js');
|
|
const path = join(
|
|
fixture.t5,
|
|
server,
|
|
'runtime/plutonium/storage/t5/raw/scripts/sp',
|
|
'stray.gsc'
|
|
);
|
|
await writeFile(path, 'unexpected content');
|
|
expect(await (await GET()).json()).toMatchObject({
|
|
totalDrift: 1,
|
|
servers: [{ server, files: [expect.objectContaining({ relPath: 'stray.gsc' })] }]
|
|
});
|
|
const preview = await POST({ request: request({ dryRun: true, servers: [server] }) });
|
|
expect(preview.status).toBe(200);
|
|
const body = await preview.json();
|
|
expect(body).toMatchObject({
|
|
ok: true,
|
|
plan: {
|
|
servers: [server],
|
|
fingerprint: expect.any(String),
|
|
files: [expect.objectContaining({ relPath: 'stray.gsc' })]
|
|
}
|
|
});
|
|
const mutation = await POST({
|
|
request: request({ servers: [server], expectedFingerprint: body.plan.fingerprint })
|
|
});
|
|
expect(mutation.status).toBe(410);
|
|
expect(await mutation.json()).toMatchObject({
|
|
ok: false,
|
|
watchdogUrl: '/watchdog',
|
|
error: expect.stringContaining('Watchdog')
|
|
});
|
|
expect(await readFile(path, 'utf8')).toBe('unexpected content');
|
|
expect(operations()).toEqual([]);
|
|
expect(database.db().prepare('SELECT id FROM deployments').all()).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe('Watchdog drift campaigns', () => {
|
|
it('keeps a missing rebuild key visible as attention without deleting files or retrying automatically', async () => {
|
|
// Simulate legacy registry damage; administrators cannot remove a leased key.
|
|
database.db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(server);
|
|
const path = join(
|
|
fixture.t5,
|
|
server,
|
|
'runtime/plutonium/storage/t5/raw/scripts/sp',
|
|
'stray.gsc'
|
|
);
|
|
await writeFile(path, 'unexpected');
|
|
await watchdog.watchdogTick(true);
|
|
expect(onlyIncident()).toMatchObject({
|
|
kind: 'drift',
|
|
state: 'attention',
|
|
summary: expect.stringContaining('server key are required')
|
|
});
|
|
advance(watchdog.WATCHDOG_SCAN_MS);
|
|
await watchdog.watchdogTick(true);
|
|
expect(operations()).toEqual([]);
|
|
expect(await readFile(path, 'utf8')).toBe('unexpected');
|
|
expect(
|
|
database
|
|
.db()
|
|
.prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'")
|
|
.all()
|
|
).toEqual([]);
|
|
});
|
|
});
|