From 2670f3b701f1b888db4a9895ce65bef0d636cc7f Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 5 Oct 2026 19:25:03 -0400 Subject: [PATCH] Add a unified Operations overview --- src/lib/navigation.ts | 1 + src/lib/operations-overview.ts | 28 ++++ src/lib/server/operations-overview.ts | 214 +++++++++++++++++++++++++ src/routes/api/operations/+server.ts | 6 + src/routes/maintenance/+page.svelte | 2 + src/routes/operations/+page.server.ts | 3 + src/routes/operations/+page.svelte | 138 ++++++++++++++++ tests/unit/auth-access.test.ts | 2 +- tests/unit/navigation.test.ts | 1 + tests/unit/operations-overview.test.ts | 168 +++++++++++++++++++ 10 files changed, 562 insertions(+), 1 deletion(-) create mode 100644 src/lib/operations-overview.ts create mode 100644 src/lib/server/operations-overview.ts create mode 100644 src/routes/api/operations/+server.ts create mode 100644 src/routes/operations/+page.server.ts create mode 100644 src/routes/operations/+page.svelte create mode 100644 tests/unit/operations-overview.test.ts diff --git a/src/lib/navigation.ts b/src/lib/navigation.ts index 7413c93..8f20e28 100644 --- a/src/lib/navigation.ts +++ b/src/lib/navigation.ts @@ -17,6 +17,7 @@ export const navigationGroups: NavigationGroup[] = [ { label: 'Game Files', href: '/game-files' } ] }, { id: 'operations', label: 'Operations', items: [ + { label: 'Overview', href: '/operations' }, { label: 'Provisioning', href: '/provisioning', shortcut: 'd', matches: path => /^\/(provisioning|deployments)(\/|$)/.test(path) }, { label: 'Betas', href: '/betas', shortcut: 'b' }, { label: 'Campaigns', href: '/campaigns', shortcut: 'a' }, diff --git a/src/lib/operations-overview.ts b/src/lib/operations-overview.ts new file mode 100644 index 0000000..a67eaa2 --- /dev/null +++ b/src/lib/operations-overview.ts @@ -0,0 +1,28 @@ +export type OperationView = 'attention' | 'running' | 'recent'; +export type OperationKind = 'collection' | 'provisioning' | 'beta' | 'campaign' | 'maintenance' | 'host'; +export interface OperationLink { key: string; title: string; href: string; status: string; detail: string } +export interface OperationItem extends OperationLink { + kind: OperationKind; + view: OperationView; + active: boolean; + updatedAt: string; + action: string; + progress: string | null; + resources: { label: string; href: string | null }[]; + related: { label: string; href: string }[]; + children: OperationLink[]; +} +export interface OperationsOverview { + items: OperationItem[]; + counts: Record; + generatedAt: string; + recentDays: number; + recentLimit: number; +} + +export const operationKindLabels: Record = { + collection: 'Collections', provisioning: 'Provisioning', beta: 'Betas', campaign: 'Campaigns', maintenance: 'Maintenance', host: 'Hosts' +}; +export function operationView(value: string | null): OperationView { + return value === 'running' || value === 'recent' ? value : 'attention'; +} diff --git a/src/lib/server/operations-overview.ts b/src/lib/server/operations-overview.ts new file mode 100644 index 0000000..8c7c72a --- /dev/null +++ b/src/lib/server/operations-overview.ts @@ -0,0 +1,214 @@ +import { db } from './db/index.js'; +import { ensureHostSchema } from './hosts/schema.js'; +import { importantOperationSql } from './hosts/observations.js'; +import { actionLabels, type ProvisioningAction } from '../provisioning.js'; +import { serverTitle } from '../ui/server-labels.js'; +import type { OperationItem, OperationsOverview, OperationView } from '../operations-overview.js'; + +const recentDays = 7; +const recentLimit = 100; +const inFlight = ['queued', 'waiting', 'running', 'awaiting-health']; +const failed = ['failed', 'interrupted', 'warning']; +interface Candidate { id: string; active: number; updatedAt: string } +interface Job extends Candidate { server: string; server_deleted_at: string | null; action: ProvisioningAction; state: string; stage: string; error: string | null; latest: number } +interface Member { parent: string; server: string; deleted: string | null; state: string; detail: string | null; players?: number | null } +interface ParentLink { parent: string; child: string } + +/** A summary-only read: never load file contents, agent payloads, or observation snapshots. */ +export function operationsOverview(now = Date.now()): OperationsOverview { + const handle = db(); + ensureHostSchema(handle); + return handle.transaction(() => { + const since = new Date(now - recentDays * 86_400_000).toISOString(); + // Active/unresolved work is retained regardless of age. Only finished history is capped. + function candidates(query: string): T[] { + return handle.prepare(`WITH summaries AS (${query}) SELECT * FROM summaries + WHERE active=1 OR id IN (SELECT id FROM summaries WHERE active=0 AND updatedAt>=@since + ORDER BY updatedAt DESC,id DESC LIMIT ${recentLimit}) + ORDER BY updatedAt DESC,id DESC`).all({ since }) as T[]; + } + const items = new Map(); + const keyList = (prefix: string) => JSON.stringify([...items.keys()].filter(key => key.startsWith(prefix + ':')).map(key => key.slice(prefix.length + 1))); + const serverResource = (server: string, deleted: string | null) => ({ label: serverTitle(server) + (deleted ? ' · deleted' : ''), href: deleted ? null : `/servers/${encodeURIComponent(server)}` }); + const make = (input: Omit & Partial>) => { + const item: OperationItem = { resources: [], children: [], related: [], progress: null, ...input }; + items.set(item.key, item); + return item; + }; + function addJob(job: Job) { + const active = !job.server_deleted_at && inFlight.includes(job.state); + const attention = !job.server_deleted_at && !!job.latest && failed.includes(job.state); + return make({ key: `provisioning:${job.id}`, kind: 'provisioning', title: `${actionLabels[job.action]} · ${serverTitle(job.server)}`, + href: `/provisioning/${encodeURIComponent(job.id)}`, active, view: attention ? 'attention' : active ? 'running' : 'recent', + status: attention ? 'Needs review' : active ? job.stage : job.state === 'done' ? 'Completed' : job.state, + detail: job.error || job.stage, updatedAt: job.updatedAt, action: attention ? 'Review operation' : active ? 'Follow progress' : 'View results', + resources: [serverResource(job.server, job.server_deleted_at)] }); + } + const jobColumns = `d.id,d.server,d.server_deleted_at,d.action,d.state,d.stage,d.error, + COALESCE(d.finished_at,d.observed_at,d.started_at) AS updatedAt, + NOT EXISTS(SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid) AS latest`; + const jobs = candidates(`SELECT ${jobColumns}, + (d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health') OR + (d.state IN ('failed','interrupted','warning') AND NOT EXISTS(SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)))) AS active + FROM server_deployments d`); + jobs.forEach(addJob); + + const membersSql = `SELECT collection_id,deployment_id FROM server_collection_members + UNION SELECT collection_id,deployment_id FROM server_collection_removals`; + const collections = candidates(`SELECT c.id,c.name,COUNT(d.id) AS total, + COALESCE(SUM(d.state IN ('done','warning')),0) AS done, + COALESCE(SUM(d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted','warning') + AND NOT EXISTS(SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)),0) AS errors, + COALESCE(SUM(d.state='cancelled'),0) AS cancelled, + COALESCE(MAX(d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health')),0) AS inFlight, + COALESCE(MAX(d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health') OR + (d.state IN ('failed','interrupted','warning') AND NOT EXISTS(SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)))),0) AS active, + MAX(COALESCE(d.finished_at,d.observed_at,d.started_at,c.created_at)) AS updatedAt + FROM server_launch_collections c LEFT JOIN (${membersSql}) m ON m.collection_id=c.id + LEFT JOIN server_deployments d ON d.id=m.deployment_id GROUP BY c.id`); + for (const row of collections) make({ key: `collection:${row.id}`, kind: 'collection', title: row.name, + href: `/collections/launches/${encodeURIComponent(row.id)}`, active: !!row.inFlight, + view: row.errors ? 'attention' : row.active ? 'running' : 'recent', status: row.errors ? 'Needs review' : row.active ? 'In progress' : row.cancelled ? 'Finished with cancellations' : 'Completed', + detail: row.errors ? `${row.errors} operations reported issues. Open the launch to review the results.` : row.active ? 'Servers may wait for safe lobbies, capacity, or startup slots.' : row.cancelled ? `${row.cancelled} operations were cancelled.` : 'Collection launch finished.', + progress: `${row.done} / ${row.total} operations completed`, updatedAt: row.updatedAt, action: row.errors ? 'Review launch' : row.active ? 'Follow launch' : 'View results' }); + + const campaigns = candidates(`SELECT d.id,d.action,d.beta_id,d.rolled_back_by,COUNT(s.server) AS total, + COALESCE(SUM(s.restart_state IN ('done','skipped','cancelled')),0) AS done, + COALESCE(SUM(s.server_deleted_at IS NULL AND s.restart_state IN ('failed','manual')),0) AS errors, + COALESCE(SUM(s.server_deleted_at IS NULL AND s.restart_state='pending' AND s.players>0),0) AS waiting, + COALESCE(SUM(s.server_deleted_at IS NULL AND s.restart_state='held'),0) AS held, + COALESCE(SUM(s.server_deleted_at IS NULL AND s.restart_state NOT IN ('done','skipped','cancelled','failed','manual')),0) AS inFlight, + (d.rolled_back_by IS NULL AND COALESCE(MAX(s.server_deleted_at IS NULL AND s.restart_state NOT IN ('done','skipped','cancelled')),0)) AS active, + MAX(COALESCE(s.checked_at,d.finished_at,d.started_at)) AS updatedAt + FROM deployments d LEFT JOIN deployment_servers s ON s.deployment_id=d.id GROUP BY d.id`); + for (const row of campaigns) make({ key: `campaign:${row.id}`, kind: 'campaign', title: `${row.action.charAt(0).toUpperCase() + row.action.slice(1)} campaign`, + href: `/campaigns/${encodeURIComponent(row.id)}`, active: !!row.active && row.inFlight > 0, + view: row.active ? row.errors ? 'attention' : 'running' : 'recent', + status: row.rolled_back_by ? 'Rolled back' : row.errors ? 'Needs attention' : row.held ? 'Held by a Beta' : row.waiting ? 'Waiting for empty lobbies' : row.active ? 'In progress' : 'Finished', + detail: row.errors ? 'Review failed or manually held activation before continuing.' : row.held ? 'A Beta holds one or more servers. Open the Campaign to review eligibility.' : row.waiting ? 'Occupied lobbies will finish before activation. No intervention is required.' : 'Files activate when servers are safe to restart.', + progress: `${row.done} / ${row.total} servers finished`, updatedAt: row.updatedAt, action: row.active && row.errors ? 'Review Campaign' : row.active ? 'Follow Campaign' : 'View results', + related: row.beta_id ? [{ label: `Beta #${row.beta_id}`, href: `/betas/${row.beta_id}` }] : [] }); + + const betas = candidates(`SELECT CAST(b.id AS TEXT) AS id,b.state,b.notes,p.outcome,p.reason,COUNT(s.server) AS total, + COALESCE(SUM(s.stage='ready'),0) AS ready,COALESCE(SUM(s.stage='failed' OR s.server_deleted_at IS NOT NULL),0) AS errors, + COALESCE(SUM(s.stage='awaiting-empty'),0) AS waiting, + (b.state IN ('running','ready-promote')) AS active, + MAX(COALESCE(b.closed_at,s.updated_at,b.opened_at)) AS updatedAt + FROM betas b LEFT JOIN beta_servers s ON s.beta_id=b.id LEFT JOIN beta_test_plans p ON p.beta_id=b.id GROUP BY b.id`); + for (const row of betas) { + const attention = !!row.active && (row.errors > 0 || row.state === 'ready-promote' || ['failed','paused','inconclusive'].includes(row.outcome ?? '')); + make({ key: `beta:${row.id}`, kind: 'beta', title: row.notes || `Beta #${row.id}`, href: `/betas/${row.id}`, active: !!row.active, + view: attention ? 'attention' : row.active ? 'running' : 'recent', + status: row.state === 'ready-promote' ? 'Ready for promotion review' : attention ? 'Needs attention' : row.waiting ? 'Waiting for empty lobbies' : row.active ? 'Testing' : row.state === 'promoted' ? 'Promoted' : 'Rolled back', + detail: row.reason || (row.state === 'ready-promote' ? 'Review the tested changes and promotion scope.' : row.errors ? 'Review the server results and recovery options.' : row.waiting ? 'Testing continues when the selected lobbies are empty.' : 'Open the Beta for its checks, files, and server results.'), + progress: `${row.ready} / ${row.total} servers ready`, updatedAt: row.updatedAt, action: attention ? 'Review Beta' : row.active ? 'Follow Beta' : 'View results' }); + } + + const incidents = candidates(`SELECT id,server,server_deleted_at,state,kind,summary, + (server_deleted_at IS NULL AND state IN ('open','recovering','attention')) AS active, + COALESCE(finished_at,last_seen_at,first_seen_at) AS updatedAt FROM watchdog_incidents`); + for (const row of incidents) make({ key: `incident:${row.id}`, kind: 'maintenance', title: row.kind === 'drift' ? 'Configuration drift recovery' : 'Server health recovery', + href: `/maintenance/${encodeURIComponent(row.id)}`, active: !!row.active, view: row.active ? row.state === 'attention' ? 'attention' : 'running' : 'recent', + status: row.state === 'attention' ? 'Needs attention' : row.state === 'open' ? 'Detected' : row.state, detail: row.summary, + updatedAt: row.updatedAt, action: row.state === 'attention' ? 'Review recovery' : row.active ? 'Follow recovery' : 'View results', resources: [serverResource(row.server, row.server_deleted_at)] }); + const maintenance = candidates(`SELECT CAST(m.id AS TEXT) AS id,m.server,m.server_deleted_at,m.state,m.kind,m.detail, + (m.server_deleted_at IS NULL AND (m.state IN ('queued','restarting') OR (m.state='failed' AND NOT EXISTS(SELECT 1 FROM maintenance_jobs n WHERE n.server=m.server AND n.kind=m.kind AND n.id>m.id)))) AS active, + COALESCE(m.finished_at,m.restarted_at,m.requested_at) AS updatedAt FROM maintenance_jobs m`); + for (const row of maintenance) make({ key: `maintenance:${row.id}`, kind: 'maintenance', title: row.kind === 'update' ? 'Plutonium update' : row.kind === 'schedule' ? 'Scheduled restart' : 'Maintenance restart', + href: `/maintenance?view=activity&server=${encodeURIComponent(row.server)}#job-${row.id}`, active: !!row.active && row.state !== 'failed', + view: row.active ? row.state === 'failed' ? 'attention' : 'running' : 'recent', status: row.state === 'failed' ? 'Needs attention' : row.state, + detail: row.detail || 'Open Maintenance for progress and verification.', updatedAt: row.updatedAt, action: row.state === 'failed' ? 'Review Maintenance' : 'View Maintenance', resources: [serverResource(row.server, row.server_deleted_at)] }); + + // Batch child membership; include older attempts for each selected parent without loading unrelated history. + const relationships = handle.prepare(` + SELECT 'collection:'||collection_id AS parent,'provisioning:'||deployment_id AS child FROM (${membersSql}) WHERE collection_id IN (SELECT value FROM json_each(@collections)) + UNION SELECT CASE kind WHEN 'watchdog' THEN 'incident:' ELSE kind||':' END||source_id,'provisioning:'||operation_id FROM provisioning_sources + WHERE CASE kind WHEN 'watchdog' THEN 'incident:' ELSE kind||':' END||source_id IN (SELECT value FROM json_each(@keys)) + UNION SELECT 'campaign:'||deployment_id,'provisioning:'||provisioning_id FROM deployment_servers WHERE deployment_id IN (SELECT value FROM json_each(@campaigns)) AND provisioning_id IS NOT NULL + UNION SELECT 'beta:'||beta_id,'provisioning:'||provisioning_id FROM beta_servers WHERE beta_id IN (SELECT value FROM json_each(@betas)) AND provisioning_id IS NOT NULL + UNION SELECT 'maintenance:'||id,'provisioning:'||provisioning_id FROM maintenance_jobs WHERE id IN (SELECT value FROM json_each(@maintenance)) AND provisioning_id IS NOT NULL + UNION SELECT 'incident:'||incident_id,'provisioning:'||operation_id FROM watchdog_actions WHERE incident_id IN (SELECT value FROM json_each(@incidents)) + UNION SELECT 'incident:'||incident_id,'campaign:'||campaign_id FROM watchdog_actions WHERE incident_id IN (SELECT value FROM json_each(@incidents)) AND campaign_id IS NOT NULL + UNION SELECT 'provisioning:'||creation_id,'provisioning:'||cleanup_id FROM native_creation_cancellations WHERE creation_id IN (SELECT value FROM json_each(@provisioning)) + `).all({ collections: keyList('collection'), campaigns: keyList('campaign'), betas: keyList('beta'), maintenance: keyList('maintenance'), incidents: keyList('incident'), provisioning: keyList('provisioning'), keys: JSON.stringify([...items.keys()]) }) as ParentLink[]; + const missingJobs = [...new Set(relationships.filter(link => link.child.startsWith('provisioning:') && !items.has(link.child)).map(link => link.child.slice(13)))]; + if (missingJobs.length) { + (handle.prepare(`SELECT ${jobColumns},0 AS active FROM server_deployments d WHERE d.id IN (SELECT value FROM json_each(?))`).all(JSON.stringify(missingJobs)) as Job[]).forEach(addJob); + } + // A recovery can outlive the recent-history window of its repair Campaign. + const missingCampaigns = [...new Set(relationships.filter(link => link.child.startsWith('campaign:') && !items.has(link.child)).map(link => link.child.slice(9)))]; + if (missingCampaigns.length) { + const rows = handle.prepare(`SELECT id,action,COALESCE(finished_at,started_at) AS updatedAt FROM deployments WHERE id IN (SELECT value FROM json_each(?))`).all(JSON.stringify(missingCampaigns)) as { id: string; action: string; updatedAt: string }[]; + for (const row of rows) make({ key: `campaign:${row.id}`, kind: 'campaign', title: `${row.action} campaign`, href: `/campaigns/${encodeURIComponent(row.id)}`, + view: 'recent', active: false, status: 'Finished', detail: 'Open the Campaign to review its server results.', updatedAt: row.updatedAt, action: 'View results' }); + } + const members = handle.prepare(`SELECT 'campaign:'||deployment_id AS parent,server,server_deleted_at AS deleted,restart_state AS state,error AS detail,players FROM deployment_servers + WHERE deployment_id IN (SELECT value FROM json_each(@campaigns)) + UNION ALL SELECT 'beta:'||beta_id,server,server_deleted_at,stage,error,NULL FROM beta_servers WHERE beta_id IN (SELECT value FROM json_each(@betas))`) + .all({ campaigns: keyList('campaign'), betas: keyList('beta') }) as Member[]; + for (const member of members) { + const parent = items.get(member.parent)!; + parent.resources.push(serverResource(member.server, member.deleted)); + if (member.detail && !member.deleted && (member.state === 'failed' || member.state === 'manual')) parent.detail = member.detail; + } + + const hostOperations = candidates(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.server_name AS server,o.kind,o.state,o.error,o.routine,o.observation_success AS observationSuccess, + (o.routine=0 AND o.state IN ('pending','accepted','running')) AS active,o.updated_at AS updatedAt + FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL WHERE ${importantOperationSql} + AND (o.routine=0 OR NOT EXISTS(SELECT 1 FROM ec_host_operations newer WHERE newer.host_id=o.host_id AND newer.kind=o.kind AND newer.instance_id IS o.instance_id AND newer.rowid>o.rowid))`); + const agentParents = handle.prepare(`SELECT operation_id AS id,'provisioning:'||deployment_id AS parent FROM native_provisioning WHERE operation_id IS NOT NULL + UNION SELECT stage_operation_id,'provisioning:'||deployment_id FROM native_provisioning WHERE stage_operation_id IS NOT NULL + UNION SELECT remove_operation_id,'provisioning:'||deployment_id FROM ec_host_migrations WHERE remove_operation_id IS NOT NULL`).all() as { id: string; parent: string }[]; + const represented = new Map(agentParents.map(row => [row.id, row.parent])); + for (const row of hostOperations) { + const attention = ['failed','unknown'].includes(row.state) || !!row.error || !!row.routine && row.state === 'completed' && !row.observationSuccess; + const item = make({ key: `host:${row.id}`, kind: 'host', title: `${row.kind} · ${row.hostName ?? 'Deleted host'}`, + href: `/hosts/operations?host=${encodeURIComponent(row.hostId)}`, view: attention ? 'attention' : row.active ? 'running' : 'recent', active: !!row.active, + status: attention ? 'Needs attention' : row.state, detail: row.error || (attention ? 'The latest host check reported a problem.' : 'Open host operation history for details.'), + updatedAt: row.updatedAt, action: 'Review host activity', resources: [{ label: row.hostName ?? 'Deleted host', href: row.hostName === null ? null : `/hosts/${encodeURIComponent(row.hostId)}` }] }); + const parent = represented.get(row.id); + if (parent && items.has(parent)) relationships.push({ parent, child: item.key }); + } + + // Preserve source semantics: an old failed attempt must not turn resolved recovery back into an alert. + const nested = new Set(); + // Completed parents outside the history window also own their failed attempts. + const historicalParents = handle.prepare(`WITH parents AS ( + SELECT 'provisioning:'||s.operation_id AS child FROM provisioning_sources s WHERE + (s.kind='watchdog' AND EXISTS(SELECT 1 FROM watchdog_incidents p WHERE p.id=s.source_id)) OR + (s.kind='campaign' AND EXISTS(SELECT 1 FROM deployments p WHERE p.id=s.source_id)) OR + (s.kind='beta' AND EXISTS(SELECT 1 FROM betas p WHERE p.id=s.source_id)) OR + (s.kind='maintenance' AND EXISTS(SELECT 1 FROM maintenance_jobs p WHERE p.id=s.source_id)) + UNION SELECT 'provisioning:'||operation_id FROM watchdog_actions + ) SELECT child FROM parents WHERE child IN (SELECT value FROM json_each(?)) + `).all(JSON.stringify([...items.keys()].filter(key => key.startsWith('provisioning:')))) as { child: string }[]; + const representedChildren = new Set(relationships.map(link => link.child)); + for (const { child: childKey } of historicalParents) { + const child = items.get(childKey); + if (child && !child.active && !representedChildren.has(childKey)) nested.add(childKey); + } + for (const { parent: parentKey, child: childKey } of relationships) { + const parent = items.get(parentKey), child = items.get(childKey); + if (!parent || !child || parent === child) continue; + if (!parent.children.some(item => item.key === child.key)) parent.children.push({ key: child.key, title: child.title, href: child.href, status: child.status, detail: child.detail }); + parent.resources.push(...child.resources); + if (parent.view === 'recent' && child.active) { parent.view = 'running'; parent.active = true; parent.status = 'Related work still running'; } + nested.add(child.key); + } + function relatedResources(item: OperationItem, seen = new Set()): OperationItem['resources'] { + if (seen.has(item.key)) return []; + seen.add(item.key); + return [...item.resources, ...item.children.flatMap(link => { + const child = items.get(link.key); + return child ? relatedResources(child, seen) : []; + })]; + } + const result = [...items.values()].filter(item => !nested.has(item.key)).map(item => ({ ...item, + resources: [...new Map(relatedResources(item).map(resource => [resource.label + resource.href, resource])).values()], + children: item.children.sort((a,b) => a.key.localeCompare(b.key)) + })).sort((a,b) => b.updatedAt.localeCompare(a.updatedAt) || a.key.localeCompare(b.key)); + const counts: Record = { attention: 0, running: 0, recent: 0 }; + for (const item of result) counts[item.view]++; + return { items: result, counts, generatedAt: new Date(now).toISOString(), recentDays, recentLimit }; + })(); +} diff --git a/src/routes/api/operations/+server.ts b/src/routes/api/operations/+server.ts new file mode 100644 index 0000000..b1f4de8 --- /dev/null +++ b/src/routes/api/operations/+server.ts @@ -0,0 +1,6 @@ +import { json } from '@sveltejs/kit'; +import { operationsOverview } from '$lib/server/operations-overview.js'; + +export function GET() { + return json(operationsOverview(), { headers: { 'Cache-Control': 'no-store' } }); +} diff --git a/src/routes/maintenance/+page.svelte b/src/routes/maintenance/+page.svelte index e8e7e8b..868e755 100644 --- a/src/routes/maintenance/+page.svelte +++ b/src/routes/maintenance/+page.svelte @@ -1,4 +1,5 @@ + +Operations · Electric Cherry +
+ + {#snippet actions()}{/snippet} + + saveContext(operationView(value), true)} ariaLabel="Operations views" idPrefix="operations-tabs" panelId="operations-panel" /> +
{ event.preventDefault(); saveContext(view, true); }}> + + + + {#if search || kind}{/if} +
+ {#if error}{/if} +
Updated {$timeFormat.dateTime(overview.generatedAt)} · Refreshes automatically
+
+ {#if !visible.length} + + {:else} +

{visible.length} {visible.length === 1 ? 'item' : 'items'} · Related operations are grouped together.

+ {#each visible.slice(0, limit) as item (item.key)} +
+
{operationKindLabels[item.kind]}

{item.title}

{item.status}
+

{item.detail}

+ {#if item.progress}{item.progress}{/if} +
{#each item.resources.slice(0, 4) as resource}{#if resource.href}{resource.label}{:else}{resource.label}{/if}{/each}{#if item.resources.length > 4}+{item.resources.length - 4} more below{/if}
+ {#if item.related.length}
{#each item.related as related}{related.label}{/each}
{/if} +
+ {#if item.children.length || item.resources.length > 4}
+ {item.children.length ? `${item.children.length} related operations` : `${item.resources.length} affected resources`} + {#if item.resources.length > 4}
{#each item.resources as resource}{#if resource.href}{resource.label}{:else}{resource.label}{/if}{/each}
{/if} +
    {#each item.children.slice(0, 25) as child (child.key)}
  • {child.title}{child.status}

    {child.detail}

    • {/each}
+ {#if item.children.length > 25}View all {item.children.length} related operations in the workflow →{/if} +
{/if} +
+ {/each} + saveContext()} label="items" /> + {/if} +
+

Running workflows stay visible. Recent activity includes up to {overview.recentLimit} finished records per workflow from the last {overview.recentDays} days. Full history remains on each workflow page. Routine successful agent checks are available in Host operation history.

+
+ + diff --git a/tests/unit/auth-access.test.ts b/tests/unit/auth-access.test.ts index cb88131..9c7783c 100644 --- a/tests/unit/auth-access.test.ts +++ b/tests/unit/auth-access.test.ts @@ -12,7 +12,7 @@ describe('management route permissions', () => { '/leaderboard/players/[game]/[guid]', '/api/campaigns/[id]', '/campaigns/[id]', '/api/provisioning/[id]', '/provisioning/[id]', '/api/provisioning/[id]/events', '/api/betas/[id]', '/api/scripts/[id]', '/api/configs', '/api/plugins', '/api/hosts/[id]', - '/hosts/operations', '/api/hosts/operations', + '/hosts/operations', '/api/hosts/operations', '/operations', '/api/operations', '/api/game-files/[id]', '/api/maintenance', '/api/events/servers', '/api/events/pipeline' ])('lets viewers read the safe summary route %s', route => { expect(routePermission(route, 'GET')).toBe('viewer'); diff --git a/tests/unit/navigation.test.ts b/tests/unit/navigation.test.ts index 7d54bdc..27ebdf6 100644 --- a/tests/unit/navigation.test.ts +++ b/tests/unit/navigation.test.ts @@ -7,6 +7,7 @@ describe('application navigation', () => { ['/collections/launches/42', 'fleet', '/collections'], ['/hosts/42', 'fleet', '/hosts'], ['/hosts/operations', 'operations', '/hosts/operations'], + ['/operations', 'operations', '/operations'], ['/betas/new', 'operations', '/betas'], ['/campaigns/42', 'operations', '/campaigns'], ['/configs/42', 'library', '/configs'], diff --git a/tests/unit/operations-overview.test.ts b/tests/unit/operations-overview.test.ts new file mode 100644 index 0000000..2896b91 --- /dev/null +++ b/tests/unit/operations-overview.test.ts @@ -0,0 +1,168 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +let fixture: Fixture; +const now = '2026-10-05T12:00:00.000Z'; +const old = '2026-08-01T12:00:00.000Z'; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); + +async function setup() { + const { db } = await import('../../src/lib/server/db/index.js'); + const { operationsOverview } = await import('../../src/lib/server/operations-overview.js'); + const handle = db(); + const overview = () => operationsOverview(Date.parse(now)); + function job(id: string, state = 'done', server = id, timestamp = now) { + handle.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by,observation_json) + VALUES(?,?,'restart',?,?,?,?,'test','{"private":"observation-secret"}')`).run(id, server, state, state, timestamp, ['done','failed','warning'].includes(state) ? timestamp : null); + } + function campaign(id: string, state = 'pending', timestamp = now) { + handle.prepare("INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)").run(id, timestamp); + handle.prepare(`INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state,players,checked_at) VALUES(?,?,1,?,4,?)`).run(id, id, state, timestamp); + } + function incident(id: string, state = 'recovering', timestamp = now) { + handle.prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at) + VALUES(?,?,'health',?,'Recovery summary','{"private":"incident-secret"}',?,?)`).run(id, id, state, timestamp, timestamp); + } + return { handle, overview, job, campaign, incident }; +} + +describe('Operations overview', () => { + it('returns an empty summary and an uncached read-only API response', async () => { + const api = await setup(); + expect(api.overview()).toMatchObject({ items: [], counts: { attention: 0, running: 0, recent: 0 }, recentDays: 7, recentLimit: 100 }); + const { GET } = await import('../../src/routes/api/operations/+server.js'); + const response = GET(); + expect(response.headers.get('cache-control')).toBe('no-store'); + expect((await response.json()).items).toEqual([]); + }); + + it('separates empty-lobby waits, latest failures, and completed results', async () => { + const api = await setup(); + api.campaign('occupied'); + api.job('waiting', 'waiting'); + api.job('failed', 'failed'); + api.job('completed'); + const result = api.overview(); + expect(result.counts).toEqual({ attention: 1, running: 2, recent: 1 }); + expect(result.items.find(item => item.key === 'campaign:occupied')).toMatchObject({ view: 'running', status: 'Waiting for empty lobbies' }); + expect(result.items.find(item => item.key === 'provisioning:failed')).toMatchObject({ view: 'attention' }); + expect(JSON.stringify(result)).not.toContain('observation-secret'); + }); + + it('groups collection attempts, retaining old unresolved launches until superseded', async () => { + const api = await setup(); + api.job('one', 'failed', 'one', old); + api.handle.prepare('INSERT INTO server_launch_collections(id,name,created_at) VALUES(?,?,?)').run('launch', 'Evening launch', old); + api.handle.exec("INSERT INTO server_collection_members(collection_id,deployment_id,position) VALUES('launch','one',0)"); + const first = api.overview(); + expect(first.items).toHaveLength(1); + expect(first.items[0]).toMatchObject({ key: 'collection:launch', view: 'attention', active: false, children: [{ key: 'provisioning:one' }] }); + api.job('retry', 'done', 'one'); + expect(api.overview().items.map(item => item.key)).toEqual(['provisioning:retry']); + }); + + it('keeps all recovery attempts together without reviving resolved incidents', async () => { + const api = await setup(); + api.incident('recovery', 'resolved'); + api.job('attempt', 'failed', 'recovery'); + api.campaign('repair', 'failed'); + api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(now); + const result = api.overview(); + expect(result.items).toHaveLength(1); + expect(result.items[0]).toMatchObject({ key: 'incident:recovery', view: 'recent', active: false }); + expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']); + expect(JSON.stringify(result)).not.toContain('incident-secret'); + }); + + it('retains older repair Campaign links within an active incident', async () => { + const api = await setup(); + api.incident('recovery', 'attention'); + api.job('attempt', 'done', 'recovery', old); + api.campaign('repair', 'done', old); + api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(old); + const result = api.overview(); + expect(result.items).toHaveLength(1); + expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']); + }); + + it('does not resurface failed attempts from resolved incidents outside recent history', async () => { + const api = await setup(); + api.incident('recovery', 'resolved', old); + api.job('attempt', 'failed', 'recovery', old); + api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,created_at) VALUES('a','recovery','restart','attempt',?)`).run(old); + expect(api.overview().items).toEqual([]); + }); + + it('keeps source-less attempts accessible when a recorded parent is unavailable', async () => { + const api = await setup(); + api.job('orphan', 'failed'); + api.handle.exec("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('orphan','campaign','missing','orphan')"); + expect(api.overview().items).toMatchObject([{ key: 'provisioning:orphan', view: 'attention' }]); + }); + + it('shows Beta review and test failures as attention while empty-lobby waits keep running', async () => { + const api = await setup(); + const script = api.handle.prepare("INSERT INTO scripts(name,target_rel_path) VALUES('script','scripts/test.gsc')").run().lastInsertRowid; + const version = api.handle.prepare("INSERT INTO script_versions(script_id,version,sha256,size,content) VALUES(?,1,'hash',1,X'31')").run(script).lastInsertRowid; + for (const [id, state, stage, outcome] of [[1,'running','awaiting-empty','active'],[2,'ready-promote','ready','passed'],[3,'running','ready','paused'],[4,'promoted','ready','passed']] as const) { + api.handle.prepare("INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at) VALUES(?,?,?,'health',?,?)").run(id,script,version,state,now); + api.handle.prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage,updated_at) VALUES(?,?,0,?,?)").run(id,`server-${id}`,stage,now); + api.handle.prepare("INSERT INTO beta_test_plans(beta_id,plan,outcome) VALUES(?,'{}',?)").run(id,outcome); + } + const result = api.overview(); + expect(result.counts).toEqual({ attention: 2, running: 1, recent: 1 }); + expect(result.items.find(item => item.key === 'beta:1')?.status).toBe('Waiting for empty lobbies'); + api.campaign('promotion','done'); + api.handle.exec("UPDATE deployments SET beta_id=4 WHERE id='promotion'"); + expect(api.overview().items.find(item => item.key === 'campaign:promotion')?.related).toEqual([{ label: 'Beta #4', href: '/betas/4' }]); + }); + + it('omits healthy routine checks and superseded routine failures without loading private data', async () => { + const api = await setup(); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const host = hosts.createHost({ name: 'Test host' }); + const first = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance', password: 'payload-secret' }); + hosts.reportHostOperationStatus(host.id,first.id,{ state:'failed', error: 'Unreachable', result: { password: 'result-secret' } }); + api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now); + expect(api.overview().items).toMatchObject([{ key: `host:${first.id}`, view:'attention' }]); + const next = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance' }); + hosts.reportHostOperationStatus(host.id,next.id,{ state:'completed', result: { healthy:true } }); + api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now); + expect(api.overview().items).toEqual([]); + const action = hosts.queueHostOperation(host.id,'instance.restart',{ password: 'payload-secret' }); + api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now); + const result = api.overview(); + expect(result.items).toMatchObject([{ key: `host:${action.id}`, view:'running' }]); + expect(JSON.stringify(result)).not.toContain('secret'); + }); + + it('bounds finished history while retaining older active work and removing deleted resource links', async () => { + const api = await setup(); + api.job('old-result', 'done', 'old-result', old); + api.job('old-active', 'waiting', 'old-active', old); + for (let index = 0; index < 120; index++) api.job(`result-${String(index).padStart(3,'0')}`); + api.handle.prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id=?').run(now,'result-119'); + const result = api.overview(); + expect(result.counts).toEqual({ attention:0,running:1,recent:100 }); + expect(result.items.find(item => item.key === 'provisioning:old-active')?.view).toBe('running'); + expect(result.items.find(item => item.key === 'provisioning:result-119')?.resources[0].href).toBeNull(); + expect(result.items.some(item => item.key === 'provisioning:old-result')).toBe(false); + }); + + it('groups Maintenance and its provisioning and host work while keeping the workflow actionable', async () => { + const api = await setup(); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const host = hosts.createHost({ name: 'Test host' }); + const operation = hosts.queueHostOperation(host.id, 'instance.restart', {}); + api.job('restart', 'running'); + api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','restart',?,'library','image',4976,?)`).run(host.id, now); + api.handle.prepare(`INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('restart','instance',1,?)`).run(operation.id); + api.handle.prepare(`INSERT INTO maintenance_jobs(server,state,requested_at,provisioning_id) VALUES('restart','restarting',?,'restart')`).run(now); + const result = api.overview(); + expect(result.items).toHaveLength(1); + expect(result.items[0]).toMatchObject({ kind: 'maintenance', view: 'running', children: [{ key: 'provisioning:restart' }] }); + expect(result.items[0].href).toContain('server=restart'); + expect(result.items[0].resources).toContainEqual({ label: 'Test host', href: `/hosts/${host.id}` }); + }); +});