josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add a unified Operations overview

Browse Source
This commit is contained in:
josh
2026-10-05 19:25:03 -04:00
parent 913dc99181
commit 2670f3b701
10 changed files with 562 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/auth-access.test.ts
+1 -1
View File
@@ -12,7 +12,7 @@ describe('management route permissions', () => {
'/leaderboard/players/[game]/[guid]', '/api/campaigns/[id]', '/campaigns/[id]',
'/api/provisioning/[id]', '/provisioning/[id]', '/api/provisioning/[id]/events',
'/api/betas/[id]', '/api/scripts/[id]', '/api/configs', '/api/plugins', '/api/hosts/[id]',
'/hosts/operations', '/api/hosts/operations',
'/hosts/operations', '/api/hosts/operations', '/operations', '/api/operations',
'/api/game-files/[id]', '/api/maintenance', '/api/events/servers', '/api/events/pipeline'
])('lets viewers read the safe summary route %s', route => {
expect(routePermission(route, 'GET')).toBe('viewer');
tests/unit/navigation.test.ts
+1
View File
@@ -7,6 +7,7 @@ describe('application navigation', () => {
['/collections/launches/42', 'fleet', '/collections'],
['/hosts/42', 'fleet', '/hosts'],
['/hosts/operations', 'operations', '/hosts/operations'],
['/operations', 'operations', '/operations'],
['/betas/new', 'operations', '/betas'],
['/campaigns/42', 'operations', '/campaigns'],
['/configs/42', 'library', '/configs'],
tests/unit/operations-overview.test.ts
+168
View File
@@ -0,0 +1,168 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const now = '2026-10-05T12:00:00.000Z';
const old = '2026-08-01T12:00:00.000Z';
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
async function setup() {
const { db } = await import('../../src/lib/server/db/index.js');
const { operationsOverview } = await import('../../src/lib/server/operations-overview.js');
const handle = db();
const overview = () => operationsOverview(Date.parse(now));
function job(id: string, state = 'done', server = id, timestamp = now) {
handle.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by,observation_json)
VALUES(?,?,'restart',?,?,?,?,'test','{"private":"observation-secret"}')`).run(id, server, state, state, timestamp, ['done','failed','warning'].includes(state) ? timestamp : null);
}
function campaign(id: string, state = 'pending', timestamp = now) {
handle.prepare("INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)").run(id, timestamp);
handle.prepare(`INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state,players,checked_at) VALUES(?,?,1,?,4,?)`).run(id, id, state, timestamp);
}
function incident(id: string, state = 'recovering', timestamp = now) {
handle.prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
VALUES(?,?,'health',?,'Recovery summary','{"private":"incident-secret"}',?,?)`).run(id, id, state, timestamp, timestamp);
}
return { handle, overview, job, campaign, incident };
}
describe('Operations overview', () => {
it('returns an empty summary and an uncached read-only API response', async () => {
const api = await setup();
expect(api.overview()).toMatchObject({ items: [], counts: { attention: 0, running: 0, recent: 0 }, recentDays: 7, recentLimit: 100 });
const { GET } = await import('../../src/routes/api/operations/+server.js');
const response = GET();
expect(response.headers.get('cache-control')).toBe('no-store');
expect((await response.json()).items).toEqual([]);
});
it('separates empty-lobby waits, latest failures, and completed results', async () => {
const api = await setup();
api.campaign('occupied');
api.job('waiting', 'waiting');
api.job('failed', 'failed');
api.job('completed');
const result = api.overview();
expect(result.counts).toEqual({ attention: 1, running: 2, recent: 1 });
expect(result.items.find(item => item.key === 'campaign:occupied')).toMatchObject({ view: 'running', status: 'Waiting for empty lobbies' });
expect(result.items.find(item => item.key === 'provisioning:failed')).toMatchObject({ view: 'attention' });
expect(JSON.stringify(result)).not.toContain('observation-secret');
});
it('groups collection attempts, retaining old unresolved launches until superseded', async () => {
const api = await setup();
api.job('one', 'failed', 'one', old);
api.handle.prepare('INSERT INTO server_launch_collections(id,name,created_at) VALUES(?,?,?)').run('launch', 'Evening launch', old);
api.handle.exec("INSERT INTO server_collection_members(collection_id,deployment_id,position) VALUES('launch','one',0)");
const first = api.overview();
expect(first.items).toHaveLength(1);
expect(first.items[0]).toMatchObject({ key: 'collection:launch', view: 'attention', active: false, children: [{ key: 'provisioning:one' }] });
api.job('retry', 'done', 'one');
expect(api.overview().items.map(item => item.key)).toEqual(['provisioning:retry']);
});
it('keeps all recovery attempts together without reviving resolved incidents', async () => {
const api = await setup();
api.incident('recovery', 'resolved');
api.job('attempt', 'failed', 'recovery');
api.campaign('repair', 'failed');
api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(now);
const result = api.overview();
expect(result.items).toHaveLength(1);
expect(result.items[0]).toMatchObject({ key: 'incident:recovery', view: 'recent', active: false });
expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']);
expect(JSON.stringify(result)).not.toContain('incident-secret');
});
it('retains older repair Campaign links within an active incident', async () => {
const api = await setup();
api.incident('recovery', 'attention');
api.job('attempt', 'done', 'recovery', old);
api.campaign('repair', 'done', old);
api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(old);
const result = api.overview();
expect(result.items).toHaveLength(1);
expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']);
});
it('does not resurface failed attempts from resolved incidents outside recent history', async () => {
const api = await setup();
api.incident('recovery', 'resolved', old);
api.job('attempt', 'failed', 'recovery', old);
api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,created_at) VALUES('a','recovery','restart','attempt',?)`).run(old);
expect(api.overview().items).toEqual([]);
});
it('keeps source-less attempts accessible when a recorded parent is unavailable', async () => {
const api = await setup();
api.job('orphan', 'failed');
api.handle.exec("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('orphan','campaign','missing','orphan')");
expect(api.overview().items).toMatchObject([{ key: 'provisioning:orphan', view: 'attention' }]);
});
it('shows Beta review and test failures as attention while empty-lobby waits keep running', async () => {
const api = await setup();
const script = api.handle.prepare("INSERT INTO scripts(name,target_rel_path) VALUES('script','scripts/test.gsc')").run().lastInsertRowid;
const version = api.handle.prepare("INSERT INTO script_versions(script_id,version,sha256,size,content) VALUES(?,1,'hash',1,X'31')").run(script).lastInsertRowid;
for (const [id, state, stage, outcome] of [[1,'running','awaiting-empty','active'],[2,'ready-promote','ready','passed'],[3,'running','ready','paused'],[4,'promoted','ready','passed']] as const) {
api.handle.prepare("INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at) VALUES(?,?,?,'health',?,?)").run(id,script,version,state,now);
api.handle.prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage,updated_at) VALUES(?,?,0,?,?)").run(id,`server-${id}`,stage,now);
api.handle.prepare("INSERT INTO beta_test_plans(beta_id,plan,outcome) VALUES(?,'{}',?)").run(id,outcome);
}
const result = api.overview();
expect(result.counts).toEqual({ attention: 2, running: 1, recent: 1 });
expect(result.items.find(item => item.key === 'beta:1')?.status).toBe('Waiting for empty lobbies');
api.campaign('promotion','done');
api.handle.exec("UPDATE deployments SET beta_id=4 WHERE id='promotion'");
expect(api.overview().items.find(item => item.key === 'campaign:promotion')?.related).toEqual([{ label: 'Beta #4', href: '/betas/4' }]);
});
it('omits healthy routine checks and superseded routine failures without loading private data', async () => {
const api = await setup();
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Test host' });
const first = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance', password: 'payload-secret' });
hosts.reportHostOperationStatus(host.id,first.id,{ state:'failed', error: 'Unreachable', result: { password: 'result-secret' } });
api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now);
expect(api.overview().items).toMatchObject([{ key: `host:${first.id}`, view:'attention' }]);
const next = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance' });
hosts.reportHostOperationStatus(host.id,next.id,{ state:'completed', result: { healthy:true } });
api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now);
expect(api.overview().items).toEqual([]);
const action = hosts.queueHostOperation(host.id,'instance.restart',{ password: 'payload-secret' });
api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now);
const result = api.overview();
expect(result.items).toMatchObject([{ key: `host:${action.id}`, view:'running' }]);
expect(JSON.stringify(result)).not.toContain('secret');
});
it('bounds finished history while retaining older active work and removing deleted resource links', async () => {
const api = await setup();
api.job('old-result', 'done', 'old-result', old);
api.job('old-active', 'waiting', 'old-active', old);
for (let index = 0; index < 120; index++) api.job(`result-${String(index).padStart(3,'0')}`);
api.handle.prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id=?').run(now,'result-119');
const result = api.overview();
expect(result.counts).toEqual({ attention:0,running:1,recent:100 });
expect(result.items.find(item => item.key === 'provisioning:old-active')?.view).toBe('running');
expect(result.items.find(item => item.key === 'provisioning:result-119')?.resources[0].href).toBeNull();
expect(result.items.some(item => item.key === 'provisioning:old-result')).toBe(false);
});
it('groups Maintenance and its provisioning and host work while keeping the workflow actionable', async () => {
const api = await setup();
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Test host' });
const operation = hosts.queueHostOperation(host.id, 'instance.restart', {});
api.job('restart', 'running');
api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','restart',?,'library','image',4976,?)`).run(host.id, now);
api.handle.prepare(`INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('restart','instance',1,?)`).run(operation.id);
api.handle.prepare(`INSERT INTO maintenance_jobs(server,state,requested_at,provisioning_id) VALUES('restart','restarting',?,'restart')`).run(now);
const result = api.overview();
expect(result.items).toHaveLength(1);
expect(result.items[0]).toMatchObject({ kind: 'maintenance', view: 'running', children: [{ key: 'provisioning:restart' }] });
expect(result.items[0].href).toContain('server=restart');
expect(result.items[0].resources).toContainEqual({ label: 'Test host', href: `/hosts/${host.id}` });
});
});