169 lines
12 KiB
TypeScript
169 lines
12 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
const now = '2026-10-05T12:00:00.000Z';
|
|
const old = '2026-08-01T12:00:00.000Z';
|
|
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
|
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
|
|
|
|
async function setup() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { operationsOverview } = await import('../../src/lib/server/operations-overview.js');
|
|
const handle = db();
|
|
const overview = () => operationsOverview(Date.parse(now));
|
|
function job(id: string, state = 'done', server = id, timestamp = now) {
|
|
handle.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by,observation_json)
|
|
VALUES(?,?,'restart',?,?,?,?,'test','{"private":"observation-secret"}')`).run(id, server, state, state, timestamp, ['done','failed','warning'].includes(state) ? timestamp : null);
|
|
}
|
|
function campaign(id: string, state = 'pending', timestamp = now) {
|
|
handle.prepare("INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)").run(id, timestamp);
|
|
handle.prepare(`INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state,players,checked_at) VALUES(?,?,1,?,4,?)`).run(id, id, state, timestamp);
|
|
}
|
|
function incident(id: string, state = 'recovering', timestamp = now) {
|
|
handle.prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
|
|
VALUES(?,?,'health',?,'Recovery summary','{"private":"incident-secret"}',?,?)`).run(id, id, state, timestamp, timestamp);
|
|
}
|
|
return { handle, overview, job, campaign, incident };
|
|
}
|
|
|
|
describe('Operations overview', () => {
|
|
it('returns an empty summary and an uncached read-only API response', async () => {
|
|
const api = await setup();
|
|
expect(api.overview()).toMatchObject({ items: [], counts: { attention: 0, running: 0, recent: 0 }, recentDays: 7, recentLimit: 100 });
|
|
const { GET } = await import('../../src/routes/api/operations/+server.js');
|
|
const response = GET();
|
|
expect(response.headers.get('cache-control')).toBe('no-store');
|
|
expect((await response.json()).items).toEqual([]);
|
|
});
|
|
|
|
it('separates empty-lobby waits, latest failures, and completed results', async () => {
|
|
const api = await setup();
|
|
api.campaign('occupied');
|
|
api.job('waiting', 'waiting');
|
|
api.job('failed', 'failed');
|
|
api.job('completed');
|
|
const result = api.overview();
|
|
expect(result.counts).toEqual({ attention: 1, running: 2, recent: 1 });
|
|
expect(result.items.find(item => item.key === 'campaign:occupied')).toMatchObject({ view: 'running', status: 'Waiting for empty lobbies' });
|
|
expect(result.items.find(item => item.key === 'provisioning:failed')).toMatchObject({ view: 'attention' });
|
|
expect(JSON.stringify(result)).not.toContain('observation-secret');
|
|
});
|
|
|
|
it('groups collection attempts, retaining old unresolved launches until superseded', async () => {
|
|
const api = await setup();
|
|
api.job('one', 'failed', 'one', old);
|
|
api.handle.prepare('INSERT INTO server_launch_collections(id,name,created_at) VALUES(?,?,?)').run('launch', 'Evening launch', old);
|
|
api.handle.exec("INSERT INTO server_collection_members(collection_id,deployment_id,position) VALUES('launch','one',0)");
|
|
const first = api.overview();
|
|
expect(first.items).toHaveLength(1);
|
|
expect(first.items[0]).toMatchObject({ key: 'collection:launch', view: 'attention', active: false, children: [{ key: 'provisioning:one' }] });
|
|
api.job('retry', 'done', 'one');
|
|
expect(api.overview().items.map(item => item.key)).toEqual(['provisioning:retry']);
|
|
});
|
|
|
|
it('keeps all recovery attempts together without reviving resolved incidents', async () => {
|
|
const api = await setup();
|
|
api.incident('recovery', 'resolved');
|
|
api.job('attempt', 'failed', 'recovery');
|
|
api.campaign('repair', 'failed');
|
|
api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(now);
|
|
const result = api.overview();
|
|
expect(result.items).toHaveLength(1);
|
|
expect(result.items[0]).toMatchObject({ key: 'incident:recovery', view: 'recent', active: false });
|
|
expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']);
|
|
expect(JSON.stringify(result)).not.toContain('incident-secret');
|
|
});
|
|
|
|
it('retains older repair Campaign links within an active incident', async () => {
|
|
const api = await setup();
|
|
api.incident('recovery', 'attention');
|
|
api.job('attempt', 'done', 'recovery', old);
|
|
api.campaign('repair', 'done', old);
|
|
api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(old);
|
|
const result = api.overview();
|
|
expect(result.items).toHaveLength(1);
|
|
expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']);
|
|
});
|
|
|
|
it('does not resurface failed attempts from resolved incidents outside recent history', async () => {
|
|
const api = await setup();
|
|
api.incident('recovery', 'resolved', old);
|
|
api.job('attempt', 'failed', 'recovery', old);
|
|
api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,created_at) VALUES('a','recovery','restart','attempt',?)`).run(old);
|
|
expect(api.overview().items).toEqual([]);
|
|
});
|
|
|
|
it('keeps source-less attempts accessible when a recorded parent is unavailable', async () => {
|
|
const api = await setup();
|
|
api.job('orphan', 'failed');
|
|
api.handle.exec("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('orphan','campaign','missing','orphan')");
|
|
expect(api.overview().items).toMatchObject([{ key: 'provisioning:orphan', view: 'attention' }]);
|
|
});
|
|
|
|
it('shows Beta review and test failures as attention while empty-lobby waits keep running', async () => {
|
|
const api = await setup();
|
|
const script = api.handle.prepare("INSERT INTO scripts(name,target_rel_path) VALUES('script','scripts/test.gsc')").run().lastInsertRowid;
|
|
const version = api.handle.prepare("INSERT INTO script_versions(script_id,version,sha256,size,content) VALUES(?,1,'hash',1,X'31')").run(script).lastInsertRowid;
|
|
for (const [id, state, stage, outcome] of [[1,'running','awaiting-empty','active'],[2,'ready-promote','ready','passed'],[3,'running','ready','paused'],[4,'promoted','ready','passed']] as const) {
|
|
api.handle.prepare("INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at) VALUES(?,?,?,'health',?,?)").run(id,script,version,state,now);
|
|
api.handle.prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage,updated_at) VALUES(?,?,0,?,?)").run(id,`server-${id}`,stage,now);
|
|
api.handle.prepare("INSERT INTO beta_test_plans(beta_id,plan,outcome) VALUES(?,'{}',?)").run(id,outcome);
|
|
}
|
|
const result = api.overview();
|
|
expect(result.counts).toEqual({ attention: 2, running: 1, recent: 1 });
|
|
expect(result.items.find(item => item.key === 'beta:1')?.status).toBe('Waiting for empty lobbies');
|
|
api.campaign('promotion','done');
|
|
api.handle.exec("UPDATE deployments SET beta_id=4 WHERE id='promotion'");
|
|
expect(api.overview().items.find(item => item.key === 'campaign:promotion')?.related).toEqual([{ label: 'Beta #4', href: '/betas/4' }]);
|
|
});
|
|
|
|
it('omits healthy routine checks and superseded routine failures without loading private data', async () => {
|
|
const api = await setup();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Test host' });
|
|
const first = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance', password: 'payload-secret' });
|
|
hosts.reportHostOperationStatus(host.id,first.id,{ state:'failed', error: 'Unreachable', result: { password: 'result-secret' } });
|
|
api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now);
|
|
expect(api.overview().items).toMatchObject([{ key: `host:${first.id}`, view:'attention' }]);
|
|
const next = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance' });
|
|
hosts.reportHostOperationStatus(host.id,next.id,{ state:'completed', result: { healthy:true } });
|
|
api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now);
|
|
expect(api.overview().items).toEqual([]);
|
|
const action = hosts.queueHostOperation(host.id,'instance.restart',{ password: 'payload-secret' });
|
|
api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now);
|
|
const result = api.overview();
|
|
expect(result.items).toMatchObject([{ key: `host:${action.id}`, view:'running' }]);
|
|
expect(JSON.stringify(result)).not.toContain('secret');
|
|
});
|
|
|
|
it('bounds finished history while retaining older active work and removing deleted resource links', async () => {
|
|
const api = await setup();
|
|
api.job('old-result', 'done', 'old-result', old);
|
|
api.job('old-active', 'waiting', 'old-active', old);
|
|
for (let index = 0; index < 120; index++) api.job(`result-${String(index).padStart(3,'0')}`);
|
|
api.handle.prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id=?').run(now,'result-119');
|
|
const result = api.overview();
|
|
expect(result.counts).toEqual({ attention:0,running:1,recent:100 });
|
|
expect(result.items.find(item => item.key === 'provisioning:old-active')?.view).toBe('running');
|
|
expect(result.items.find(item => item.key === 'provisioning:result-119')?.resources[0].href).toBeNull();
|
|
expect(result.items.some(item => item.key === 'provisioning:old-result')).toBe(false);
|
|
});
|
|
|
|
it('groups Maintenance and its provisioning and host work while keeping the workflow actionable', async () => {
|
|
const api = await setup();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const host = hosts.createHost({ name: 'Test host' });
|
|
const operation = hosts.queueHostOperation(host.id, 'instance.restart', {});
|
|
api.job('restart', 'running');
|
|
api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','restart',?,'library','image',4976,?)`).run(host.id, now);
|
|
api.handle.prepare(`INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('restart','instance',1,?)`).run(operation.id);
|
|
api.handle.prepare(`INSERT INTO maintenance_jobs(server,state,requested_at,provisioning_id) VALUES('restart','restarting',?,'restart')`).run(now);
|
|
const result = api.overview();
|
|
expect(result.items).toHaveLength(1);
|
|
expect(result.items[0]).toMatchObject({ kind: 'maintenance', view: 'running', children: [{ key: 'provisioning:restart' }] });
|
|
expect(result.items[0].href).toContain('server=restart');
|
|
expect(result.items[0].resources).toContainEqual({ label: 'Test host', href: `/hosts/${host.id}` });
|
|
});
|
|
});
|