import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; const now = '2026-10-05T12:00:00.000Z'; const old = '2026-08-01T12:00:00.000Z'; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); const { operationsOverview } = await import('../../src/lib/server/operations-overview.js'); const handle = db(); const overview = () => operationsOverview(Date.parse(now)); function job(id: string, state = 'done', server = id, timestamp = now) { handle.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by,observation_json) VALUES(?,?,'restart',?,?,?,?,'test','{"private":"observation-secret"}')`).run(id, server, state, state, timestamp, ['done','failed','warning'].includes(state) ? timestamp : null); } function campaign(id: string, state = 'pending', timestamp = now) { handle.prepare("INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)").run(id, timestamp); handle.prepare(`INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state,players,checked_at) VALUES(?,?,1,?,4,?)`).run(id, id, state, timestamp); } function incident(id: string, state = 'recovering', timestamp = now) { handle.prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at) VALUES(?,?,'health',?,'Recovery summary','{"private":"incident-secret"}',?,?)`).run(id, id, state, timestamp, timestamp); } return { handle, overview, job, campaign, incident }; } describe('Operations overview', () => { it('returns an empty summary and an uncached read-only API response', async () => { const api = await setup(); expect(api.overview()).toMatchObject({ items: [], counts: { attention: 0, running: 0, recent: 0 }, recentDays: 7, recentLimit: 100 }); const { GET } = await import('../../src/routes/api/operations/+server.js'); const response = GET(); expect(response.headers.get('cache-control')).toBe('no-store'); expect((await response.json()).items).toEqual([]); }); it('separates empty-lobby waits, latest failures, and completed results', async () => { const api = await setup(); api.campaign('occupied'); api.job('waiting', 'waiting'); api.job('failed', 'failed'); api.job('completed'); const result = api.overview(); expect(result.counts).toEqual({ attention: 1, running: 2, recent: 1 }); expect(result.items.find(item => item.key === 'campaign:occupied')).toMatchObject({ view: 'running', status: 'Waiting for empty lobbies' }); expect(result.items.find(item => item.key === 'provisioning:failed')).toMatchObject({ view: 'attention' }); expect(JSON.stringify(result)).not.toContain('observation-secret'); }); it('groups collection attempts, retaining old unresolved launches until superseded', async () => { const api = await setup(); api.job('one', 'failed', 'one', old); api.handle.prepare('INSERT INTO server_launch_collections(id,name,created_at) VALUES(?,?,?)').run('launch', 'Evening launch', old); api.handle.exec("INSERT INTO server_collection_members(collection_id,deployment_id,position) VALUES('launch','one',0)"); const first = api.overview(); expect(first.items).toHaveLength(1); expect(first.items[0]).toMatchObject({ key: 'collection:launch', view: 'attention', active: false, children: [{ key: 'provisioning:one' }] }); api.job('retry', 'done', 'one'); expect(api.overview().items.map(item => item.key)).toEqual(['provisioning:retry']); }); it('keeps all recovery attempts together without reviving resolved incidents', async () => { const api = await setup(); api.incident('recovery', 'resolved'); api.job('attempt', 'failed', 'recovery'); api.campaign('repair', 'failed'); api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(now); const result = api.overview(); expect(result.items).toHaveLength(1); expect(result.items[0]).toMatchObject({ key: 'incident:recovery', view: 'recent', active: false }); expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']); expect(JSON.stringify(result)).not.toContain('incident-secret'); }); it('retains older repair Campaign links within an active incident', async () => { const api = await setup(); api.incident('recovery', 'attention'); api.job('attempt', 'done', 'recovery', old); api.campaign('repair', 'done', old); api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(old); const result = api.overview(); expect(result.items).toHaveLength(1); expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']); }); it('does not resurface failed attempts from resolved incidents outside recent history', async () => { const api = await setup(); api.incident('recovery', 'resolved', old); api.job('attempt', 'failed', 'recovery', old); api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,created_at) VALUES('a','recovery','restart','attempt',?)`).run(old); expect(api.overview().items).toEqual([]); }); it('keeps source-less attempts accessible when a recorded parent is unavailable', async () => { const api = await setup(); api.job('orphan', 'failed'); api.handle.exec("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('orphan','campaign','missing','orphan')"); expect(api.overview().items).toMatchObject([{ key: 'provisioning:orphan', view: 'attention' }]); }); it('shows Beta review and test failures as attention while empty-lobby waits keep running', async () => { const api = await setup(); const script = api.handle.prepare("INSERT INTO scripts(name,target_rel_path) VALUES('script','scripts/test.gsc')").run().lastInsertRowid; const version = api.handle.prepare("INSERT INTO script_versions(script_id,version,sha256,size,content) VALUES(?,1,'hash',1,X'31')").run(script).lastInsertRowid; for (const [id, state, stage, outcome] of [[1,'running','awaiting-empty','active'],[2,'ready-promote','ready','passed'],[3,'running','ready','paused'],[4,'promoted','ready','passed']] as const) { api.handle.prepare("INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at) VALUES(?,?,?,'health',?,?)").run(id,script,version,state,now); api.handle.prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage,updated_at) VALUES(?,?,0,?,?)").run(id,`server-${id}`,stage,now); api.handle.prepare("INSERT INTO beta_test_plans(beta_id,plan,outcome) VALUES(?,'{}',?)").run(id,outcome); } const result = api.overview(); expect(result.counts).toEqual({ attention: 2, running: 1, recent: 1 }); expect(result.items.find(item => item.key === 'beta:1')?.status).toBe('Waiting for empty lobbies'); api.campaign('promotion','done'); api.handle.exec("UPDATE deployments SET beta_id=4 WHERE id='promotion'"); expect(api.overview().items.find(item => item.key === 'campaign:promotion')?.related).toEqual([{ label: 'Beta #4', href: '/betas/4' }]); }); it('omits healthy routine checks and superseded routine failures without loading private data', async () => { const api = await setup(); const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Test host' }); const first = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance', password: 'payload-secret' }); hosts.reportHostOperationStatus(host.id,first.id,{ state:'failed', error: 'Unreachable', result: { password: 'result-secret' } }); api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now); expect(api.overview().items).toMatchObject([{ key: `host:${first.id}`, view:'attention' }]); const next = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance' }); hosts.reportHostOperationStatus(host.id,next.id,{ state:'completed', result: { healthy:true } }); api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now); expect(api.overview().items).toEqual([]); const action = hosts.queueHostOperation(host.id,'instance.restart',{ password: 'payload-secret' }); api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now); const result = api.overview(); expect(result.items).toMatchObject([{ key: `host:${action.id}`, view:'running' }]); expect(JSON.stringify(result)).not.toContain('secret'); }); it('bounds finished history while retaining older active work and removing deleted resource links', async () => { const api = await setup(); api.job('old-result', 'done', 'old-result', old); api.job('old-active', 'waiting', 'old-active', old); for (let index = 0; index < 120; index++) api.job(`result-${String(index).padStart(3,'0')}`); api.handle.prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id=?').run(now,'result-119'); const result = api.overview(); expect(result.counts).toEqual({ attention:0,running:1,recent:100 }); expect(result.items.find(item => item.key === 'provisioning:old-active')?.view).toBe('running'); expect(result.items.find(item => item.key === 'provisioning:result-119')?.resources[0].href).toBeNull(); expect(result.items.some(item => item.key === 'provisioning:old-result')).toBe(false); }); it('groups Maintenance and its provisioning and host work while keeping the workflow actionable', async () => { const api = await setup(); const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Test host' }); const operation = hosts.queueHostOperation(host.id, 'instance.restart', {}); api.job('restart', 'running'); api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','restart',?,'library','image',4976,?)`).run(host.id, now); api.handle.prepare(`INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('restart','instance',1,?)`).run(operation.id); api.handle.prepare(`INSERT INTO maintenance_jobs(server,state,requested_at,provisioning_id) VALUES('restart','restarting',?,'restart')`).run(now); const result = api.overview(); expect(result.items).toHaveLength(1); expect(result.items[0]).toMatchObject({ kind: 'maintenance', view: 'running', children: [{ key: 'provisioning:restart' }] }); expect(result.items[0].href).toContain('server=restart'); expect(result.items[0].resources).toContainEqual({ label: 'Test host', href: `/hosts/${host.id}` }); }); });