Improve operational UX, recovery flows, and action safety

Implement the approved UX review across Fleet, Betas, Library, deployments, and Settings. Add reviewed promotion and reconciliation plans, atomic queued-removal cancellation, recoverable beta drafts and launch inputs, accessible controls, and mobile improvements.

Validated with 473 tests, a clean typecheck, production build, and isolated desktop/mobile browser walkthroughs. Include the review and implementation coverage notes.
This commit is contained in:
2026-09-19 21:34:34 -04:00
parent 940f81e6ca
commit 8ac420fe61
83 changed files with 2574 additions and 1190 deletions
+78
View File
@@ -0,0 +1,78 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
beforeEach(async () => {
vi.resetModules(); fixture = await newFixture(servers);
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('RCON_PASSWORD', '');
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
async function ready(kind: 'script' | 'baseline' = 'script', scope: 'game' | 'map' = 'game') {
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { script } = createScript({ name: 'Review test', kind, targetRelPath: kind === 'script' ? 'review.gsc' : undefined,
scopeType: scope, scopeValue: scope === 'game' ? 't5' : 'zombie_theater', content: Buffer.from(kind === 'script' ? 'main(){}' : 'set sv_voice "1"\n') });
const version = getLatestVersion(script.id)!;
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [servers[0]], waitForHealth: false, waitForMatch: false });
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(new Date().toISOString(), betaId);
return { betaId, script, version };
}
const destination = () => join(fixture.t5, servers[1], 'runtime/plutonium/storage/t5/raw/scripts/sp/review.gsc');
describe('reviewed beta promotion', () => {
it('previews exact versions and additional targets without writing, then links the activation rollout', async () => {
const { betaId, script, version } = await ready();
const { previewBetaPromotion, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
const preview = previewBetaPromotion(betaId);
expect(preview.files[0]).toMatchObject({ id: script.id, versionId: version.id, sha256: version.sha256, servers: [...servers].sort() });
expect(preview.servers).toEqual(expect.arrayContaining([expect.objectContaining({ name: servers[0], tested: true }), expect.objectContaining({ name: servers[1], tested: false })]));
expect(previewBetaPromotion(betaId).token).toBe(preview.token);
expect(existsSync(destination())).toBe(false);
const result = await promoteBeta(betaId, preview.token);
expect(result).toMatchObject({ awaitingActivation: 1, failedServers: [] });
expect(result.deploymentId).toContain('promote');
expect(await readFile(destination(), 'utf8')).toBe('main(){}');
});
it('rejects changed scope membership before creating a rollout or writing extra files', async () => {
const { betaId } = await ready('script', 'map');
const { previewBetaPromotion, promoteBeta, getBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
const preview = previewBetaPromotion(betaId);
expect(preview.servers).toHaveLength(1);
db().prepare('UPDATE servers SET map = ? WHERE name = ?').run('zombie_theater', servers[1]);
await expect(promoteBeta(betaId, preview.token)).rejects.toMatchObject({ kind: 'preview-changed', details: expect.objectContaining({ servers: expect.arrayContaining([expect.objectContaining({ name: servers[1] })]) }) });
expect(existsSync(destination())).toBe(false);
expect(getBeta(betaId)?.state).toBe('ready-promote');
expect(db().prepare('SELECT COUNT(*) n FROM deployments').get()).toEqual({ n: 0 });
await expect(promoteBeta(betaId, previewBetaPromotion(betaId).token)).resolves.toMatchObject({ awaitingActivation: 1 });
});
it('includes effective config contents in the guard, including map-specific composition', async () => {
const { betaId } = await ready('baseline');
const { previewBetaPromotion, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
const preview = previewBetaPromotion(betaId);
expect(preview.configWrites).toHaveLength(2);
expect(new Set(preview.configWrites.map(write => write.sha256)).size).toBe(2);
db().prepare('UPDATE servers SET map = ?, map_slug = ? WHERE name = ?').run('zombie_moon', 'moon', servers[1]);
await expect(promoteBeta(betaId, preview.token)).rejects.toMatchObject({ kind: 'preview-changed' });
expect(db().prepare('SELECT COUNT(*) n FROM deployments').get()).toEqual({ n: 0 });
});
it('requires the reviewed token at the HTTP boundary', async () => {
const { betaId } = await ready();
const { POST, GET } = await import('../../src/routes/api/betas/[id]/promote/+server.js');
const response = await POST({ params: { id: String(betaId) }, request: new Request('http://localhost/promote', { method: 'POST', body: '{}' }) });
expect(response.status).toBe(400);
const preview = await GET({ params: { id: String(betaId) } });
expect(preview.headers.get('cache-control')).toBe('no-store');
expect((await preview.json()).preview.token).toHaveLength(64);
});
});
+40
View File
@@ -0,0 +1,40 @@
import { describe, expect, it } from 'vitest';
import { defaultPlan, betaPlanIssues, requirement } from '../../src/lib/beta-plan.js';
import { readBetaDraft, compatibleBetaServers } from '../../src/lib/beta-draft.js';
import { betaWaitingCondition, betaEvidenceSummary } from '../../src/lib/beta-evidence.js';
describe('beta draft and plan recovery', () => {
it('retains exact version IDs and incomplete plan fields in a recoverable draft', () => {
const plan = defaultPlan();
const raw = JSON.stringify({ version: 1, savedAt: new Date().toISOString(), step: 2, picks: [{ scriptId: 12, versionId: 45 }], servers: ['town'], testPlan: { ...plan, timeoutHours: null } });
const draft = readBetaDraft(raw)!;
expect(draft.picks).toEqual([{ scriptId: 12, versionId: 45 }]);
expect(betaPlanIssues(draft.testPlan, 1)).toContainEqual(expect.objectContaining({ field: 'timeoutHours' }));
expect(readBetaDraft('{bad')).toBeNull();
expect(readBetaDraft(JSON.stringify({ version: 1, picks: [{ scriptId: '12', versionId: 45 }] }))).toBeNull();
});
it('keeps compatible explicit selections without selecting newly eligible servers', () => {
const result = compatibleBetaServers(['town1', 'town2'], ['town2', 'town3']);
expect([...result.kept]).toEqual(['town2']);
expect(result.removed).toEqual(['town1']);
});
it('identifies the precise requirement and rejects an impossible quorum before review', () => {
const plan = { ...defaultPlan(), scope: 'quorum' as const, quorum: 3, requirements: [{ ...requirement('matches', 'matches'), target: 1.5, minPlayers: 0 }] };
expect(betaPlanIssues(plan, 2).map(issue => issue.field)).toEqual(['quorum', 'matches-target', 'matches-minPlayers']);
expect(betaPlanIssues(plan, 2).find(issue => issue.field === 'matches-target')?.requirementId).toBe('matches');
});
});
describe('beta evidence explanations', () => {
const observation = { lastGameplayAt: new Date().toISOString(), lastEvent: 'round_change', players: 2, round: 12, matchStarted: true, gameEnded: false, healthy: true, healthCheckedAt: new Date().toISOString() };
it('distinguishes missing reports, insufficient population, and report-driven time', () => {
const r = { ...requirement('populated-minutes', 'play'), minPlayers: 3 };
expect(betaWaitingCondition(r)).toContain('No gameplay report');
expect(betaWaitingCondition(r, observation)).toContain('latest report has 2 players');
expect(betaWaitingCondition(r, { ...observation, players: 3 })).toContain('next report arrives');
});
it('summarizes telemetry while keeping arbitrary or legacy payloads readable', () => {
const event = { id: 1, server: 'town', received_at: '', kind: 'game_end', payload: JSON.stringify({ round: 25, players: [{ name: 'A' }], map: 'zm_transit' }) };
expect(betaEvidenceSummary(event)).toBe('Game ended at round 25 with 1 players on zm_transit.');
expect(betaEvidenceSummary({ ...event, kind: 'old-event', payload: 'invalid' })).toBe('old event');
});
});
+12 -1
View File
@@ -1,7 +1,7 @@
import { describe, expect, it } from 'vitest';
import { inspectConfig, type ConfigLayer } from '../../src/lib/server/fleet/config-inspect.js';
import { composeServerCfg } from '../../src/lib/server/fleet/config-compose.js';
import { changeConfigValue, configWarnings } from '../../src/lib/config-workbench.js';
import { addConfigSetting, changeConfigValue, configWarnings } from '../../src/lib/config-workbench.js';
const base = (text: string): ConfigLayer => ({ text, source: { id: 1, name: 'Base', version: 2, kind: 'baseline' } });
const overlay = (id: number, text: string): ConfigLayer => ({ text, source: { id, name: `Overlay ${id}`, version: 3, kind: 'config' } });
@@ -63,6 +63,17 @@ describe('config source inspection', () => {
});
describe('config settings editor', () => {
it('adds a structured value without altering existing content or line endings', () => {
expect(addConfigSetting('// comment\r\nset x 1', 'sv_hostname', 'Cherry lobby')).toBe('// comment\r\nset x 1\r\nset sv_hostname "Cherry lobby"\r\n');
expect(addConfigSetting('', 'g_inactivity', '0')).toBe('set g_inactivity 0\n');
});
it('rejects duplicate keys, missing values, and multiline assignments', () => {
expect(() => addConfigSetting('set FOO 1', 'foo', '2')).toThrow('already exists');
expect(() => addConfigSetting('', 'bad key', '1')).toThrow('setting name');
expect(() => addConfigSetting('', 'key', '// no value')).toThrow('value');
expect(() => addConfigSetting('', 'key', '1\nset other 2')).toThrow('one line');
});
it('changes only the selected assignment, preserving CRLF, spacing and raw commands', () => {
const text = '// note\r\n set a "1" // original\r\nexec other.cfg\r\nset a 5\n';
expect(changeConfigValue(text, 1, '"2" // changed')).toBe('// note\r\n set a "2" // changed\r\nexec other.cfg\r\nset a 5\n');
+1
View File
@@ -29,6 +29,7 @@ describe('config workbench API', () => {
expect(response.status).toBe(200);
const result = await response.json();
expect(result.inspection.settings.find((s: { key: string }) => s.key === 'g_inactivity')).toMatchObject({ value: '60', source: { id: overlay.script.id, draft: true } });
expect(result.promoted.settings.find((s: { key: string }) => s.key === 'g_inactivity').value).toBe('190');
expect(listVersions(overlay.script.id)).toHaveLength(1);
expect(getScript(baseline.script.id)?.current_version_id).toBe(baseline.version.id);
const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
+54
View File
@@ -48,6 +48,60 @@ afterEach(async () => {
vi.unstubAllEnvs();
});
describe('queued removal cancellation', () => {
it('lets cancellation win while the worker awaits a player count, and never removes the server', async () => {
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
await createServer(input);
let resolveCount!: (players: number) => void;
vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; }));
const queued = destroyServer(serverName);
const work = processDestructions();
await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function'));
expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(200);
resolveCount(0);
await work;
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', server_exists: 1, stage: 'Removal cancelled' });
expect(getDeployment(queued.deploymentId!)?.canCancelRemoval).toBe(false);
expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(409);
});
it('does not overwrite cancellation when the in-flight player check rejects', async () => {
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
const { cancelRemoval, getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
await createServer(input);
let rejectCount!: (error: Error) => void;
vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise((_resolve, reject) => { rejectCount = reject; }));
const queued = destroyServer(serverName);
const work = processDestructions();
await vi.waitFor(() => expect(rejectCount).toBeTypeOf('function'));
expect(cancelRemoval(queued.deploymentId!)).toBe(true);
rejectCount(new Error('RCON unavailable'));
await work;
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', error: null });
});
it('refuses cancellation after an atomic removal claim and for cleanup checkpoints', async () => {
const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
const { cancelRemoval, claimRemoval, canCancelRemoval, deploymentStage, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
const { db } = await import('../../src/lib/server/db/index.js');
await createServer(input);
const queued = destroyServer(serverName);
expect(claimRemoval(queued.deploymentId!)).toBe(true);
expect(cancelRemoval(queued.deploymentId!)).toBe(false);
expect(claimRemoval(queued.deploymentId!)).toBe(false);
deploymentStage(queued.deploymentId!, REMOVING_SERVER);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(queued.deploymentId);
expect(canCancelRemoval(queued.deploymentId!)).toBe(false);
expect(cancelRemoval(queued.deploymentId!)).toBe(false);
});
});
describe('destruction cleanup recovery', () => {
it('removes T6 servers with beta test runs and their dependent evidence', async () => {
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
+51
View File
@@ -21,6 +21,57 @@ afterEach(async () => {
await fixture.cleanup();
});
describe('reviewed drift plans', () => {
it('rejects changed file bytes before any deletion and returns a fresh review', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg');
await writeFile(path, 'original');
const plan = await previewDrift(['t5_zombie_theater_1']);
await writeFile(path, 'changed');
await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
expect(await readFile(path, 'utf8')).toBe('changed');
expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]);
const refreshed = await previewDrift(plan.servers);
expect(refreshed.fingerprint).not.toBe(plan.fingerprint);
expect((await runDriftDeployment({ servers: refreshed.servers, expectedFingerprint: refreshed.fingerprint })).totalRemoved).toBe(1);
});
it('rejects changed replacement bytes and preserves the reviewed existing file', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const { script, version } = createScript({ name: 'Managed config', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'dedicated_sp.cfg');
await writeFile(path, 'existing settings');
const plan = await previewDrift(['t5_zombie_theater_1']);
db().prepare('UPDATE script_versions SET content = ? WHERE id = ?').run(Buffer.from('set sv_voice "0"\n'), version.id);
await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
expect(await readFile(path, 'utf8')).toBe('existing settings');
});
it('keeps selected reconciliation within its reviewed server identities', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const second = join(fixture.t5, 't5_zombie_theater_2', 'configs');
await mkdir(second, { recursive: true });
await seedServerRegistry();
const firstPath = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg');
await writeFile(firstPath, 'one');
await writeFile(join(second, 'stray.cfg'), 'two');
const plan = await previewDrift(['t5_zombie_theater_1']);
await expect(runDriftDeployment({ expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
expect(existsSync(firstPath)).toBe(false);
expect(await readFile(join(second, 'stray.cfg'), 'utf8')).toBe('two');
});
});
describe('drift scanning', () => {
it('replaces a seeded config once instead of planning both removal and installation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
+35
View File
@@ -0,0 +1,35 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const server = 't5_zombie_theater_1';
beforeEach(async () => {
vi.resetModules(); fixture = await newFixture([server]);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => { await fixture.cleanup(); });
describe('scope impact preview', () => {
it('reports unscoping tracked script copies without changing scope or inventory', async () => {
const { createScript, getScript } = await import('../../src/lib/server/fleet/scripts.js');
const { recordScript, listServerScripts } = await import('../../src/lib/server/fleet/inventory.js');
const { GET } = await import('../../src/routes/api/scripts/[id]/scope/+server.js');
const script = createScript({ name: 'Example', scopeType: 'game', scopeValue: 't5', targetRelPath: 'example.gsc', content: Buffer.from('example') });
recordScript(server, 'example.gsc', script.version.id, script.version.sha256, script.version.size, 'test');
const response = await GET({ params: { id: String(script.script.id) }, url: new URL('http://localhost/?type=none') });
expect(await response.json()).toMatchObject({ before: [server], after: [], removed: [server], detached: [server], affected: [server], composed: false });
expect(getScript(script.script.id)?.scope_type).toBe('game');
expect(listServerScripts(server)).toHaveLength(1);
});
it('reports config composition impact separately from detached inventory', async () => {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { GET } = await import('../../src/routes/api/scripts/[id]/scope/+server.js');
const config = createScript({ name: 'Overlay', kind: 'config', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set g_inactivity 0') });
const response = await GET({ params: { id: String(config.script.id) }, url: new URL('http://localhost/?type=none') });
expect(await response.json()).toMatchObject({ composed: true, detached: [], affected: [server], removed: [server] });
const invalid = await GET({ params: { id: String(config.script.id) }, url: new URL('http://localhost/?type=invalid&value=t5') });
expect(invalid.status).toBe(400);
});
});
+49
View File
@@ -0,0 +1,49 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
vi.stubEnv('KOMODO_ADDRESS', '');
vi.stubEnv('KOMODO_API_KEY', '');
vi.stubEnv('KOMODO_API_SECRET', '');
vi.stubEnv('KOMODO_TARGET_SERVER', '');
vi.stubEnv('RCON_PASSWORD', '');
});
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); });
it('distinguishes missing setup from an unverifiable host volume without probing', async () => {
const probe = vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe');
const { GET } = await import('../../src/routes/api/system-status/+server.js');
const response = await GET({ url: new URL('http://localhost/api/system-status') });
const body = await response.json();
expect(response.headers.get('cache-control')).toBe('no-store');
expect(body.checks.find((check: { id: string }) => check.id === 'komodo').state).toBe('unconfigured');
expect(body.checks.find((check: { id: string }) => check.id === 'game-files').state).toBe('unverified');
expect(body.readyServers).toBe(0);
expect(probe).not.toHaveBeenCalled();
});
it('never exposes credentials or claims that network reachability validates them', async () => {
vi.stubEnv('KOMODO_ADDRESS', 'https://komodo.example.test');
vi.stubEnv('KOMODO_API_KEY', 'fixture-private-key');
vi.stubEnv('KOMODO_API_SECRET', 'fixture-private-secret');
vi.stubEnv('RCON_PASSWORD', 'fixture-rcon-secret');
vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe').mockResolvedValue({ label: 'Komodo', host: 'komodo.example.test', port: 443, resolvedIp: '192.0.2.1', outcome: 'connected', elapsedMs: 1 });
const { GET } = await import('../../src/routes/api/system-status/+server.js');
const text = await (await GET({ url: new URL('http://localhost/api/system-status?probe=1') })).text();
for (const secret of ['fixture-private-key', 'fixture-private-secret', 'fixture-rcon-secret']) expect(text).not.toContain(secret);
const connection = JSON.parse(text).checks.find((check: { id: string }) => check.id === 'komodo');
expect(connection.state).toBe('ready');
expect(connection.detail).toContain('does not validate API credentials');
});
it('reports a configured unreachable host as needing attention', async () => {
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.example.test:9120');
vi.stubEnv('KOMODO_API_KEY', 'key'); vi.stubEnv('KOMODO_API_SECRET', 'secret');
vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe').mockResolvedValue({ label: 'Komodo', host: 'komodo.example.test', port: 9120, resolvedIp: null, outcome: 'dns-fail', elapsedMs: 1 });
const { GET } = await import('../../src/routes/api/system-status/+server.js');
const body = await (await GET({ url: new URL('http://localhost/api/system-status?probe=1') })).json();
expect(body.checks.find((check: { id: string }) => check.id === 'komodo').state).toBe('attention');
});