Add a unified Operations overview
This commit is contained in:
@@ -17,6 +17,7 @@ export const navigationGroups: NavigationGroup[] = [
|
||||
{ label: 'Game Files', href: '/game-files' }
|
||||
] },
|
||||
{ id: 'operations', label: 'Operations', items: [
|
||||
{ label: 'Overview', href: '/operations' },
|
||||
{ label: 'Provisioning', href: '/provisioning', shortcut: 'd', matches: path => /^\/(provisioning|deployments)(\/|$)/.test(path) },
|
||||
{ label: 'Betas', href: '/betas', shortcut: 'b' },
|
||||
{ label: 'Campaigns', href: '/campaigns', shortcut: 'a' },
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
export type OperationView = 'attention' | 'running' | 'recent';
|
||||
export type OperationKind = 'collection' | 'provisioning' | 'beta' | 'campaign' | 'maintenance' | 'host';
|
||||
export interface OperationLink { key: string; title: string; href: string; status: string; detail: string }
|
||||
export interface OperationItem extends OperationLink {
|
||||
kind: OperationKind;
|
||||
view: OperationView;
|
||||
active: boolean;
|
||||
updatedAt: string;
|
||||
action: string;
|
||||
progress: string | null;
|
||||
resources: { label: string; href: string | null }[];
|
||||
related: { label: string; href: string }[];
|
||||
children: OperationLink[];
|
||||
}
|
||||
export interface OperationsOverview {
|
||||
items: OperationItem[];
|
||||
counts: Record<OperationView, number>;
|
||||
generatedAt: string;
|
||||
recentDays: number;
|
||||
recentLimit: number;
|
||||
}
|
||||
|
||||
export const operationKindLabels: Record<OperationKind, string> = {
|
||||
collection: 'Collections', provisioning: 'Provisioning', beta: 'Betas', campaign: 'Campaigns', maintenance: 'Maintenance', host: 'Hosts'
|
||||
};
|
||||
export function operationView(value: string | null): OperationView {
|
||||
return value === 'running' || value === 'recent' ? value : 'attention';
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
import { db } from './db/index.js';
|
||||
import { ensureHostSchema } from './hosts/schema.js';
|
||||
import { importantOperationSql } from './hosts/observations.js';
|
||||
import { actionLabels, type ProvisioningAction } from '../provisioning.js';
|
||||
import { serverTitle } from '../ui/server-labels.js';
|
||||
import type { OperationItem, OperationsOverview, OperationView } from '../operations-overview.js';
|
||||
|
||||
const recentDays = 7;
|
||||
const recentLimit = 100;
|
||||
const inFlight = ['queued', 'waiting', 'running', 'awaiting-health'];
|
||||
const failed = ['failed', 'interrupted', 'warning'];
|
||||
interface Candidate { id: string; active: number; updatedAt: string }
|
||||
interface Job extends Candidate { server: string; server_deleted_at: string | null; action: ProvisioningAction; state: string; stage: string; error: string | null; latest: number }
|
||||
interface Member { parent: string; server: string; deleted: string | null; state: string; detail: string | null; players?: number | null }
|
||||
interface ParentLink { parent: string; child: string }
|
||||
|
||||
/** A summary-only read: never load file contents, agent payloads, or observation snapshots. */
|
||||
export function operationsOverview(now = Date.now()): OperationsOverview {
|
||||
const handle = db();
|
||||
ensureHostSchema(handle);
|
||||
return handle.transaction(() => {
|
||||
const since = new Date(now - recentDays * 86_400_000).toISOString();
|
||||
// Active/unresolved work is retained regardless of age. Only finished history is capped.
|
||||
function candidates<T extends Candidate>(query: string): T[] {
|
||||
return handle.prepare(`WITH summaries AS (${query}) SELECT * FROM summaries
|
||||
WHERE active=1 OR id IN (SELECT id FROM summaries WHERE active=0 AND updatedAt>=@since
|
||||
ORDER BY updatedAt DESC,id DESC LIMIT ${recentLimit})
|
||||
ORDER BY updatedAt DESC,id DESC`).all({ since }) as T[];
|
||||
}
|
||||
const items = new Map<string, OperationItem>();
|
||||
const keyList = (prefix: string) => JSON.stringify([...items.keys()].filter(key => key.startsWith(prefix + ':')).map(key => key.slice(prefix.length + 1)));
|
||||
const serverResource = (server: string, deleted: string | null) => ({ label: serverTitle(server) + (deleted ? ' · deleted' : ''), href: deleted ? null : `/servers/${encodeURIComponent(server)}` });
|
||||
const make = (input: Omit<OperationItem, 'resources' | 'children' | 'related' | 'progress'> & Partial<Pick<OperationItem, 'resources' | 'children' | 'related' | 'progress'>>) => {
|
||||
const item: OperationItem = { resources: [], children: [], related: [], progress: null, ...input };
|
||||
items.set(item.key, item);
|
||||
return item;
|
||||
};
|
||||
function addJob(job: Job) {
|
||||
const active = !job.server_deleted_at && inFlight.includes(job.state);
|
||||
const attention = !job.server_deleted_at && !!job.latest && failed.includes(job.state);
|
||||
return make({ key: `provisioning:${job.id}`, kind: 'provisioning', title: `${actionLabels[job.action]} · ${serverTitle(job.server)}`,
|
||||
href: `/provisioning/${encodeURIComponent(job.id)}`, active, view: attention ? 'attention' : active ? 'running' : 'recent',
|
||||
status: attention ? 'Needs review' : active ? job.stage : job.state === 'done' ? 'Completed' : job.state,
|
||||
detail: job.error || job.stage, updatedAt: job.updatedAt, action: attention ? 'Review operation' : active ? 'Follow progress' : 'View results',
|
||||
resources: [serverResource(job.server, job.server_deleted_at)] });
|
||||
}
|
||||
const jobColumns = `d.id,d.server,d.server_deleted_at,d.action,d.state,d.stage,d.error,
|
||||
COALESCE(d.finished_at,d.observed_at,d.started_at) AS updatedAt,
|
||||
NOT EXISTS(SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid) AS latest`;
|
||||
const jobs = candidates<Job>(`SELECT ${jobColumns},
|
||||
(d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health') OR
|
||||
(d.state IN ('failed','interrupted','warning') AND NOT EXISTS(SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)))) AS active
|
||||
FROM server_deployments d`);
|
||||
jobs.forEach(addJob);
|
||||
|
||||
const membersSql = `SELECT collection_id,deployment_id FROM server_collection_members
|
||||
UNION SELECT collection_id,deployment_id FROM server_collection_removals`;
|
||||
const collections = candidates<Candidate & { name: string; total: number; done: number; errors: number; cancelled: number; inFlight: number }>(`SELECT c.id,c.name,COUNT(d.id) AS total,
|
||||
COALESCE(SUM(d.state IN ('done','warning')),0) AS done,
|
||||
COALESCE(SUM(d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted','warning')
|
||||
AND NOT EXISTS(SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)),0) AS errors,
|
||||
COALESCE(SUM(d.state='cancelled'),0) AS cancelled,
|
||||
COALESCE(MAX(d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health')),0) AS inFlight,
|
||||
COALESCE(MAX(d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health') OR
|
||||
(d.state IN ('failed','interrupted','warning') AND NOT EXISTS(SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)))),0) AS active,
|
||||
MAX(COALESCE(d.finished_at,d.observed_at,d.started_at,c.created_at)) AS updatedAt
|
||||
FROM server_launch_collections c LEFT JOIN (${membersSql}) m ON m.collection_id=c.id
|
||||
LEFT JOIN server_deployments d ON d.id=m.deployment_id GROUP BY c.id`);
|
||||
for (const row of collections) make({ key: `collection:${row.id}`, kind: 'collection', title: row.name,
|
||||
href: `/collections/launches/${encodeURIComponent(row.id)}`, active: !!row.inFlight,
|
||||
view: row.errors ? 'attention' : row.active ? 'running' : 'recent', status: row.errors ? 'Needs review' : row.active ? 'In progress' : row.cancelled ? 'Finished with cancellations' : 'Completed',
|
||||
detail: row.errors ? `${row.errors} operations reported issues. Open the launch to review the results.` : row.active ? 'Servers may wait for safe lobbies, capacity, or startup slots.' : row.cancelled ? `${row.cancelled} operations were cancelled.` : 'Collection launch finished.',
|
||||
progress: `${row.done} / ${row.total} operations completed`, updatedAt: row.updatedAt, action: row.errors ? 'Review launch' : row.active ? 'Follow launch' : 'View results' });
|
||||
|
||||
const campaigns = candidates<Candidate & { action: string; beta_id: number | null; rolled_back_by: string | null; total: number; done: number; errors: number; waiting: number; held: number; inFlight: number }>(`SELECT d.id,d.action,d.beta_id,d.rolled_back_by,COUNT(s.server) AS total,
|
||||
COALESCE(SUM(s.restart_state IN ('done','skipped','cancelled')),0) AS done,
|
||||
COALESCE(SUM(s.server_deleted_at IS NULL AND s.restart_state IN ('failed','manual')),0) AS errors,
|
||||
COALESCE(SUM(s.server_deleted_at IS NULL AND s.restart_state='pending' AND s.players>0),0) AS waiting,
|
||||
COALESCE(SUM(s.server_deleted_at IS NULL AND s.restart_state='held'),0) AS held,
|
||||
COALESCE(SUM(s.server_deleted_at IS NULL AND s.restart_state NOT IN ('done','skipped','cancelled','failed','manual')),0) AS inFlight,
|
||||
(d.rolled_back_by IS NULL AND COALESCE(MAX(s.server_deleted_at IS NULL AND s.restart_state NOT IN ('done','skipped','cancelled')),0)) AS active,
|
||||
MAX(COALESCE(s.checked_at,d.finished_at,d.started_at)) AS updatedAt
|
||||
FROM deployments d LEFT JOIN deployment_servers s ON s.deployment_id=d.id GROUP BY d.id`);
|
||||
for (const row of campaigns) make({ key: `campaign:${row.id}`, kind: 'campaign', title: `${row.action.charAt(0).toUpperCase() + row.action.slice(1)} campaign`,
|
||||
href: `/campaigns/${encodeURIComponent(row.id)}`, active: !!row.active && row.inFlight > 0,
|
||||
view: row.active ? row.errors ? 'attention' : 'running' : 'recent',
|
||||
status: row.rolled_back_by ? 'Rolled back' : row.errors ? 'Needs attention' : row.held ? 'Held by a Beta' : row.waiting ? 'Waiting for empty lobbies' : row.active ? 'In progress' : 'Finished',
|
||||
detail: row.errors ? 'Review failed or manually held activation before continuing.' : row.held ? 'A Beta holds one or more servers. Open the Campaign to review eligibility.' : row.waiting ? 'Occupied lobbies will finish before activation. No intervention is required.' : 'Files activate when servers are safe to restart.',
|
||||
progress: `${row.done} / ${row.total} servers finished`, updatedAt: row.updatedAt, action: row.active && row.errors ? 'Review Campaign' : row.active ? 'Follow Campaign' : 'View results',
|
||||
related: row.beta_id ? [{ label: `Beta #${row.beta_id}`, href: `/betas/${row.beta_id}` }] : [] });
|
||||
|
||||
const betas = candidates<Candidate & { state: string; notes: string | null; outcome: string | null; reason: string | null; total: number; ready: number; errors: number; waiting: number }>(`SELECT CAST(b.id AS TEXT) AS id,b.state,b.notes,p.outcome,p.reason,COUNT(s.server) AS total,
|
||||
COALESCE(SUM(s.stage='ready'),0) AS ready,COALESCE(SUM(s.stage='failed' OR s.server_deleted_at IS NOT NULL),0) AS errors,
|
||||
COALESCE(SUM(s.stage='awaiting-empty'),0) AS waiting,
|
||||
(b.state IN ('running','ready-promote')) AS active,
|
||||
MAX(COALESCE(b.closed_at,s.updated_at,b.opened_at)) AS updatedAt
|
||||
FROM betas b LEFT JOIN beta_servers s ON s.beta_id=b.id LEFT JOIN beta_test_plans p ON p.beta_id=b.id GROUP BY b.id`);
|
||||
for (const row of betas) {
|
||||
const attention = !!row.active && (row.errors > 0 || row.state === 'ready-promote' || ['failed','paused','inconclusive'].includes(row.outcome ?? ''));
|
||||
make({ key: `beta:${row.id}`, kind: 'beta', title: row.notes || `Beta #${row.id}`, href: `/betas/${row.id}`, active: !!row.active,
|
||||
view: attention ? 'attention' : row.active ? 'running' : 'recent',
|
||||
status: row.state === 'ready-promote' ? 'Ready for promotion review' : attention ? 'Needs attention' : row.waiting ? 'Waiting for empty lobbies' : row.active ? 'Testing' : row.state === 'promoted' ? 'Promoted' : 'Rolled back',
|
||||
detail: row.reason || (row.state === 'ready-promote' ? 'Review the tested changes and promotion scope.' : row.errors ? 'Review the server results and recovery options.' : row.waiting ? 'Testing continues when the selected lobbies are empty.' : 'Open the Beta for its checks, files, and server results.'),
|
||||
progress: `${row.ready} / ${row.total} servers ready`, updatedAt: row.updatedAt, action: attention ? 'Review Beta' : row.active ? 'Follow Beta' : 'View results' });
|
||||
}
|
||||
|
||||
const incidents = candidates<Candidate & { server: string; server_deleted_at: string | null; state: string; kind: string; summary: string }>(`SELECT id,server,server_deleted_at,state,kind,summary,
|
||||
(server_deleted_at IS NULL AND state IN ('open','recovering','attention')) AS active,
|
||||
COALESCE(finished_at,last_seen_at,first_seen_at) AS updatedAt FROM watchdog_incidents`);
|
||||
for (const row of incidents) make({ key: `incident:${row.id}`, kind: 'maintenance', title: row.kind === 'drift' ? 'Configuration drift recovery' : 'Server health recovery',
|
||||
href: `/maintenance/${encodeURIComponent(row.id)}`, active: !!row.active, view: row.active ? row.state === 'attention' ? 'attention' : 'running' : 'recent',
|
||||
status: row.state === 'attention' ? 'Needs attention' : row.state === 'open' ? 'Detected' : row.state, detail: row.summary,
|
||||
updatedAt: row.updatedAt, action: row.state === 'attention' ? 'Review recovery' : row.active ? 'Follow recovery' : 'View results', resources: [serverResource(row.server, row.server_deleted_at)] });
|
||||
const maintenance = candidates<Candidate & { server: string; server_deleted_at: string | null; state: string; kind: string; detail: string | null }>(`SELECT CAST(m.id AS TEXT) AS id,m.server,m.server_deleted_at,m.state,m.kind,m.detail,
|
||||
(m.server_deleted_at IS NULL AND (m.state IN ('queued','restarting') OR (m.state='failed' AND NOT EXISTS(SELECT 1 FROM maintenance_jobs n WHERE n.server=m.server AND n.kind=m.kind AND n.id>m.id)))) AS active,
|
||||
COALESCE(m.finished_at,m.restarted_at,m.requested_at) AS updatedAt FROM maintenance_jobs m`);
|
||||
for (const row of maintenance) make({ key: `maintenance:${row.id}`, kind: 'maintenance', title: row.kind === 'update' ? 'Plutonium update' : row.kind === 'schedule' ? 'Scheduled restart' : 'Maintenance restart',
|
||||
href: `/maintenance?view=activity&server=${encodeURIComponent(row.server)}#job-${row.id}`, active: !!row.active && row.state !== 'failed',
|
||||
view: row.active ? row.state === 'failed' ? 'attention' : 'running' : 'recent', status: row.state === 'failed' ? 'Needs attention' : row.state,
|
||||
detail: row.detail || 'Open Maintenance for progress and verification.', updatedAt: row.updatedAt, action: row.state === 'failed' ? 'Review Maintenance' : 'View Maintenance', resources: [serverResource(row.server, row.server_deleted_at)] });
|
||||
|
||||
// Batch child membership; include older attempts for each selected parent without loading unrelated history.
|
||||
const relationships = handle.prepare(`
|
||||
SELECT 'collection:'||collection_id AS parent,'provisioning:'||deployment_id AS child FROM (${membersSql}) WHERE collection_id IN (SELECT value FROM json_each(@collections))
|
||||
UNION SELECT CASE kind WHEN 'watchdog' THEN 'incident:' ELSE kind||':' END||source_id,'provisioning:'||operation_id FROM provisioning_sources
|
||||
WHERE CASE kind WHEN 'watchdog' THEN 'incident:' ELSE kind||':' END||source_id IN (SELECT value FROM json_each(@keys))
|
||||
UNION SELECT 'campaign:'||deployment_id,'provisioning:'||provisioning_id FROM deployment_servers WHERE deployment_id IN (SELECT value FROM json_each(@campaigns)) AND provisioning_id IS NOT NULL
|
||||
UNION SELECT 'beta:'||beta_id,'provisioning:'||provisioning_id FROM beta_servers WHERE beta_id IN (SELECT value FROM json_each(@betas)) AND provisioning_id IS NOT NULL
|
||||
UNION SELECT 'maintenance:'||id,'provisioning:'||provisioning_id FROM maintenance_jobs WHERE id IN (SELECT value FROM json_each(@maintenance)) AND provisioning_id IS NOT NULL
|
||||
UNION SELECT 'incident:'||incident_id,'provisioning:'||operation_id FROM watchdog_actions WHERE incident_id IN (SELECT value FROM json_each(@incidents))
|
||||
UNION SELECT 'incident:'||incident_id,'campaign:'||campaign_id FROM watchdog_actions WHERE incident_id IN (SELECT value FROM json_each(@incidents)) AND campaign_id IS NOT NULL
|
||||
UNION SELECT 'provisioning:'||creation_id,'provisioning:'||cleanup_id FROM native_creation_cancellations WHERE creation_id IN (SELECT value FROM json_each(@provisioning))
|
||||
`).all({ collections: keyList('collection'), campaigns: keyList('campaign'), betas: keyList('beta'), maintenance: keyList('maintenance'), incidents: keyList('incident'), provisioning: keyList('provisioning'), keys: JSON.stringify([...items.keys()]) }) as ParentLink[];
|
||||
const missingJobs = [...new Set(relationships.filter(link => link.child.startsWith('provisioning:') && !items.has(link.child)).map(link => link.child.slice(13)))];
|
||||
if (missingJobs.length) {
|
||||
(handle.prepare(`SELECT ${jobColumns},0 AS active FROM server_deployments d WHERE d.id IN (SELECT value FROM json_each(?))`).all(JSON.stringify(missingJobs)) as Job[]).forEach(addJob);
|
||||
}
|
||||
// A recovery can outlive the recent-history window of its repair Campaign.
|
||||
const missingCampaigns = [...new Set(relationships.filter(link => link.child.startsWith('campaign:') && !items.has(link.child)).map(link => link.child.slice(9)))];
|
||||
if (missingCampaigns.length) {
|
||||
const rows = handle.prepare(`SELECT id,action,COALESCE(finished_at,started_at) AS updatedAt FROM deployments WHERE id IN (SELECT value FROM json_each(?))`).all(JSON.stringify(missingCampaigns)) as { id: string; action: string; updatedAt: string }[];
|
||||
for (const row of rows) make({ key: `campaign:${row.id}`, kind: 'campaign', title: `${row.action} campaign`, href: `/campaigns/${encodeURIComponent(row.id)}`,
|
||||
view: 'recent', active: false, status: 'Finished', detail: 'Open the Campaign to review its server results.', updatedAt: row.updatedAt, action: 'View results' });
|
||||
}
|
||||
const members = handle.prepare(`SELECT 'campaign:'||deployment_id AS parent,server,server_deleted_at AS deleted,restart_state AS state,error AS detail,players FROM deployment_servers
|
||||
WHERE deployment_id IN (SELECT value FROM json_each(@campaigns))
|
||||
UNION ALL SELECT 'beta:'||beta_id,server,server_deleted_at,stage,error,NULL FROM beta_servers WHERE beta_id IN (SELECT value FROM json_each(@betas))`)
|
||||
.all({ campaigns: keyList('campaign'), betas: keyList('beta') }) as Member[];
|
||||
for (const member of members) {
|
||||
const parent = items.get(member.parent)!;
|
||||
parent.resources.push(serverResource(member.server, member.deleted));
|
||||
if (member.detail && !member.deleted && (member.state === 'failed' || member.state === 'manual')) parent.detail = member.detail;
|
||||
}
|
||||
|
||||
const hostOperations = candidates<Candidate & { hostId: string; hostName: string | null; server: string | null; kind: string; state: string; error: string | null; routine: number; observationSuccess: number }>(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.server_name AS server,o.kind,o.state,o.error,o.routine,o.observation_success AS observationSuccess,
|
||||
(o.routine=0 AND o.state IN ('pending','accepted','running')) AS active,o.updated_at AS updatedAt
|
||||
FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL WHERE ${importantOperationSql}
|
||||
AND (o.routine=0 OR NOT EXISTS(SELECT 1 FROM ec_host_operations newer WHERE newer.host_id=o.host_id AND newer.kind=o.kind AND newer.instance_id IS o.instance_id AND newer.rowid>o.rowid))`);
|
||||
const agentParents = handle.prepare(`SELECT operation_id AS id,'provisioning:'||deployment_id AS parent FROM native_provisioning WHERE operation_id IS NOT NULL
|
||||
UNION SELECT stage_operation_id,'provisioning:'||deployment_id FROM native_provisioning WHERE stage_operation_id IS NOT NULL
|
||||
UNION SELECT remove_operation_id,'provisioning:'||deployment_id FROM ec_host_migrations WHERE remove_operation_id IS NOT NULL`).all() as { id: string; parent: string }[];
|
||||
const represented = new Map(agentParents.map(row => [row.id, row.parent]));
|
||||
for (const row of hostOperations) {
|
||||
const attention = ['failed','unknown'].includes(row.state) || !!row.error || !!row.routine && row.state === 'completed' && !row.observationSuccess;
|
||||
const item = make({ key: `host:${row.id}`, kind: 'host', title: `${row.kind} · ${row.hostName ?? 'Deleted host'}`,
|
||||
href: `/hosts/operations?host=${encodeURIComponent(row.hostId)}`, view: attention ? 'attention' : row.active ? 'running' : 'recent', active: !!row.active,
|
||||
status: attention ? 'Needs attention' : row.state, detail: row.error || (attention ? 'The latest host check reported a problem.' : 'Open host operation history for details.'),
|
||||
updatedAt: row.updatedAt, action: 'Review host activity', resources: [{ label: row.hostName ?? 'Deleted host', href: row.hostName === null ? null : `/hosts/${encodeURIComponent(row.hostId)}` }] });
|
||||
const parent = represented.get(row.id);
|
||||
if (parent && items.has(parent)) relationships.push({ parent, child: item.key });
|
||||
}
|
||||
|
||||
// Preserve source semantics: an old failed attempt must not turn resolved recovery back into an alert.
|
||||
const nested = new Set<string>();
|
||||
// Completed parents outside the history window also own their failed attempts.
|
||||
const historicalParents = handle.prepare(`WITH parents AS (
|
||||
SELECT 'provisioning:'||s.operation_id AS child FROM provisioning_sources s WHERE
|
||||
(s.kind='watchdog' AND EXISTS(SELECT 1 FROM watchdog_incidents p WHERE p.id=s.source_id)) OR
|
||||
(s.kind='campaign' AND EXISTS(SELECT 1 FROM deployments p WHERE p.id=s.source_id)) OR
|
||||
(s.kind='beta' AND EXISTS(SELECT 1 FROM betas p WHERE p.id=s.source_id)) OR
|
||||
(s.kind='maintenance' AND EXISTS(SELECT 1 FROM maintenance_jobs p WHERE p.id=s.source_id))
|
||||
UNION SELECT 'provisioning:'||operation_id FROM watchdog_actions
|
||||
) SELECT child FROM parents WHERE child IN (SELECT value FROM json_each(?))
|
||||
`).all(JSON.stringify([...items.keys()].filter(key => key.startsWith('provisioning:')))) as { child: string }[];
|
||||
const representedChildren = new Set(relationships.map(link => link.child));
|
||||
for (const { child: childKey } of historicalParents) {
|
||||
const child = items.get(childKey);
|
||||
if (child && !child.active && !representedChildren.has(childKey)) nested.add(childKey);
|
||||
}
|
||||
for (const { parent: parentKey, child: childKey } of relationships) {
|
||||
const parent = items.get(parentKey), child = items.get(childKey);
|
||||
if (!parent || !child || parent === child) continue;
|
||||
if (!parent.children.some(item => item.key === child.key)) parent.children.push({ key: child.key, title: child.title, href: child.href, status: child.status, detail: child.detail });
|
||||
parent.resources.push(...child.resources);
|
||||
if (parent.view === 'recent' && child.active) { parent.view = 'running'; parent.active = true; parent.status = 'Related work still running'; }
|
||||
nested.add(child.key);
|
||||
}
|
||||
function relatedResources(item: OperationItem, seen = new Set<string>()): OperationItem['resources'] {
|
||||
if (seen.has(item.key)) return [];
|
||||
seen.add(item.key);
|
||||
return [...item.resources, ...item.children.flatMap(link => {
|
||||
const child = items.get(link.key);
|
||||
return child ? relatedResources(child, seen) : [];
|
||||
})];
|
||||
}
|
||||
const result = [...items.values()].filter(item => !nested.has(item.key)).map(item => ({ ...item,
|
||||
resources: [...new Map(relatedResources(item).map(resource => [resource.label + resource.href, resource])).values()],
|
||||
children: item.children.sort((a,b) => a.key.localeCompare(b.key))
|
||||
})).sort((a,b) => b.updatedAt.localeCompare(a.updatedAt) || a.key.localeCompare(b.key));
|
||||
const counts: Record<OperationView, number> = { attention: 0, running: 0, recent: 0 };
|
||||
for (const item of result) counts[item.view]++;
|
||||
return { items: result, counts, generatedAt: new Date(now).toISOString(), recentDays, recentLimit };
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { operationsOverview } from '$lib/server/operations-overview.js';
|
||||
|
||||
export function GET() {
|
||||
return json(operationsOverview(), { headers: { 'Cache-Control': 'no-store' } });
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
<script lang="ts">
|
||||
import BackLink from '$ui/components/BackLink.svelte';
|
||||
import { contextHref } from '$lib/navigation-context.js';
|
||||
import { onMount } from 'svelte';
|
||||
import { page } from '$app/stores';
|
||||
@@ -130,6 +131,7 @@
|
||||
{#if error}<p class="ui-error" role="alert">{error} {data ? 'Showing the last successful refresh.' : ''} <Button size="sm" onclick={load} disabled={busy}>Retry refresh</Button></p>{/if}
|
||||
{#if notice}<p role="status">{notice}</p>{/if}
|
||||
{#if data?.policies.paused}<div class="paused"><strong>All maintenance is paused.</strong> Checks and verification continue. New actions, scheduled runs, and queued maintenance are held. Submitted operations finish recovery. Campaigns, betas, and manual Provisioning actions continue.</div>{/if}
|
||||
{#if $page.url.searchParams.has('returnTo')}<BackLink fallback="/operations" />{/if}
|
||||
<Tabs {tabs} bind:value={view} onchange={() => saveContext()} ariaLabel="Maintenance views" idPrefix="maintenance-tabs" panelId="maintenance-panel" />
|
||||
<div id="maintenance-panel" role="tabpanel" aria-labelledby={`maintenance-tabs-${view}`} class="panels">
|
||||
{#if data}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
import { operationsOverview } from '$lib/server/operations-overview.js';
|
||||
|
||||
export function load() { return { overview: operationsOverview() }; }
|
||||
@@ -0,0 +1,138 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { beforeNavigate, goto } from '$app/navigation';
|
||||
import { page, navigating } from '$app/stores';
|
||||
import type { PageData } from './$types';
|
||||
import { operationKindLabels, operationView, type OperationsOverview } from '$lib/operations-overview.js';
|
||||
import { contextHref } from '$lib/navigation-context.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { timeFormat } from '$ui/stores/time-zone.js';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import Tabs from '$ui/components/primitives/Tabs.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import Badge from '$ui/components/primitives/Badge.svelte';
|
||||
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
|
||||
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
let snapshot = $state<{ source: PageData; overview: OperationsOverview } | null>(null);
|
||||
let refreshing = $state(false);
|
||||
let error = $state('');
|
||||
let request: AbortController | null = null;
|
||||
let sequence = 0;
|
||||
let search = $state('');
|
||||
let kind = $state('');
|
||||
let limit = $state(25);
|
||||
const overview = $derived(snapshot?.source === data ? snapshot.overview : data.overview);
|
||||
const view = $derived(operationView($page.url.searchParams.get('view')));
|
||||
const tabs = $derived([
|
||||
{ id: 'attention', label: 'Needs attention', badge: overview.counts.attention },
|
||||
{ id: 'running', label: 'Running', badge: overview.counts.running },
|
||||
{ id: 'recent', label: 'Recent', badge: overview.counts.recent }
|
||||
]);
|
||||
const visible = $derived(overview.items.filter(item => item.view === view && (!kind || item.kind === kind)
|
||||
&& [item.title, item.detail, item.status, ...item.resources.map(resource => resource.label)].join(' ').toLowerCase().includes(search.trim().toLowerCase())));
|
||||
$effect(() => {
|
||||
search = $page.url.searchParams.get('q') ?? '';
|
||||
const value = $page.url.searchParams.get('kind') ?? '';
|
||||
kind = Object.hasOwn(operationKindLabels, value) ? value : '';
|
||||
limit = Math.max(25, Math.min(10000, Number($page.url.searchParams.get('limit')) || 25));
|
||||
});
|
||||
function contextUrl(nextView = view) {
|
||||
const url = new URL($page.url);
|
||||
for (const [key, value] of Object.entries({ view: nextView, q: search, kind, limit: limit > 25 ? String(limit) : '' })) {
|
||||
if (value) url.searchParams.set(key, value); else url.searchParams.delete(key);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
const origin = $derived(contextUrl());
|
||||
function saveContext(nextView = view, reset = false) {
|
||||
if (reset) limit = 25;
|
||||
void goto(contextUrl(nextView), { replaceState: nextView === view, noScroll: true, keepFocus: true });
|
||||
}
|
||||
function cancel() { sequence++; request?.abort(); request = null; refreshing = false; }
|
||||
async function refresh(signal?: AbortSignal) {
|
||||
if (refreshing || $navigating) return;
|
||||
const current = ++sequence, source = data;
|
||||
const controller = new AbortController(); request = controller; refreshing = true;
|
||||
try {
|
||||
const response = await fetch('/api/operations', { cache: 'no-store', signal: AbortSignal.any([controller.signal, AbortSignal.timeout(15000), ...(signal ? [signal] : [])]) });
|
||||
if (!response.ok) throw new Error('Could not refresh Operations. Showing the last successful update.');
|
||||
const result: OperationsOverview = await response.json();
|
||||
if (current === sequence && source === data && !signal?.aborted) { snapshot = { source, overview: result }; error = ''; }
|
||||
} catch (cause) { if (current === sequence && !signal?.aborted && !controller.signal.aborted) error = (cause as Error).message; }
|
||||
finally { if (current === sequence) { request = null; refreshing = false; } }
|
||||
}
|
||||
beforeNavigate(cancel);
|
||||
onMount(() => { const stop = pollVisible(refresh, 10000, false); return () => { stop(); cancel(); }; });
|
||||
</script>
|
||||
|
||||
<svelte:head><title>Operations · Electric Cherry</title></svelte:head>
|
||||
<section class="page">
|
||||
<SectionHeader title="Operations" subtitle="Follow fleet changes, review blockers, and return to the work that needs you.">
|
||||
{#snippet actions()}<Button disabled={refreshing || !!$navigating} onclick={() => void refresh()}>{refreshing ? 'Refreshing…' : 'Refresh'}</Button>{/snippet}
|
||||
</SectionHeader>
|
||||
<Tabs {tabs} value={view} onchange={value => saveContext(operationView(value), true)} ariaLabel="Operations views" idPrefix="operations-tabs" panelId="operations-panel" />
|
||||
<form class="filters" onsubmit={event => { event.preventDefault(); saveContext(view, true); }}>
|
||||
<label>Find work<input type="search" bind:value={search} placeholder="Search servers, hosts, or work…" /></label>
|
||||
<label>Workflow<select bind:value={kind} onchange={event => { kind = event.currentTarget.value; saveContext(view, true); }}><option value="">All workflows</option>{#each Object.entries(operationKindLabels) as [value, label]}<option {value}>{label}</option>{/each}</select></label>
|
||||
<Button type="submit">Search</Button>
|
||||
{#if search || kind}<Button variant="ghost" onclick={() => { search = ''; kind = ''; saveContext(view, true); }}>Clear filters</Button>{/if}
|
||||
</form>
|
||||
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}
|
||||
<div class="freshness">Updated {$timeFormat.dateTime(overview.generatedAt)} · Refreshes automatically</div>
|
||||
<div id="operations-panel" role="tabpanel" aria-labelledby={`operations-tabs-${view}`} class="rows">
|
||||
{#if !visible.length}
|
||||
<EmptyState title={search || kind ? 'No matching work' : view === 'attention' ? 'Nothing needs attention' : view === 'running' ? 'No work running' : 'No recent results'} description={view === 'attention' && !search && !kind ? 'Queued work and normal waits appear under Running.' : 'Choose another view or workflow to explore its activity.'} />
|
||||
{:else}
|
||||
<p class="result-count" role="status">{visible.length} {visible.length === 1 ? 'item' : 'items'} · Related operations are grouped together.</p>
|
||||
{#each visible.slice(0, limit) as item (item.key)}
|
||||
<article class:attention={item.view === 'attention'}>
|
||||
<div class="row-heading"><div><span class="eyebrow">{operationKindLabels[item.kind]}</span><h2><a href={contextHref(item.href, origin)}>{item.title}</a></h2></div><Badge variant={item.view === 'attention' ? 'warning' : item.view === 'running' ? 'info' : 'neutral'}>{item.status}</Badge></div>
|
||||
<p>{item.detail}</p>
|
||||
{#if item.progress}<strong class="progress">{item.progress}</strong>{/if}
|
||||
<div class="resources">{#each item.resources.slice(0, 4) as resource}{#if resource.href}<a href={contextHref(resource.href, origin)}>{resource.label}</a>{:else}<span>{resource.label}</span>{/if}{/each}{#if item.resources.length > 4}<span>+{item.resources.length - 4} more below</span>{/if}</div>
|
||||
{#if item.related.length}<div class="resources">{#each item.related as related}<a href={contextHref(related.href, origin)}>{related.label}</a>{/each}</div>{/if}
|
||||
<div class="row-footer"><time datetime={item.updatedAt}>{$timeFormat.dateTime(item.updatedAt)}</time><Button size="sm" href={contextHref(item.href, origin)}>{item.action} →</Button></div>
|
||||
{#if item.children.length || item.resources.length > 4}<details>
|
||||
<summary>{item.children.length ? `${item.children.length} related operations` : `${item.resources.length} affected resources`}</summary>
|
||||
{#if item.resources.length > 4}<div class="resources">{#each item.resources as resource}{#if resource.href}<a href={contextHref(resource.href, origin)}>{resource.label}</a>{:else}<span>{resource.label}</span>{/if}{/each}</div>{/if}
|
||||
<ul>{#each item.children.slice(0, 25) as child (child.key)}<li><a href={contextHref(child.href, origin)}>{child.title}</a><span>{child.status}</span><p>{child.detail}</p></li>{/each}</ul>
|
||||
{#if item.children.length > 25}<a href={contextHref(item.href, origin)}>View all {item.children.length} related operations in the workflow →</a>{/if}
|
||||
</details>{/if}
|
||||
</article>
|
||||
{/each}
|
||||
<ListFooter total={visible.length} bind:limit onchange={() => saveContext()} label="items" />
|
||||
{/if}
|
||||
</div>
|
||||
<p class="history-note">Running workflows stay visible. Recent activity includes up to {overview.recentLimit} finished records per workflow from the last {overview.recentDays} days. Full history remains on each workflow page. Routine successful agent checks are available in <a href="/hosts/operations?filter=routine">Host operation history</a>.</p>
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.page,.rows { display:grid; gap:20px; min-width:0; }
|
||||
.filters { display:flex; flex-wrap:wrap; align-items:end; gap:12px; }
|
||||
label { display:grid; gap:8px; font-size:var(--text-sm); color:var(--text-secondary); }
|
||||
label:first-child { flex:1; min-width:min(250px,100%); }
|
||||
input,select { width:100%; min-height:42px; padding:10px 12px; background:var(--surface); border:1px solid var(--border-control); border-radius:var(--radius-sm); color:var(--text-primary); font:inherit; }
|
||||
.freshness,.result-count,.history-note,time { font-size:var(--text-xs); color:var(--text-secondary); }
|
||||
.result-count,.history-note { margin:0; line-height:1.7; }
|
||||
article { min-width:0; border:1px solid var(--border-subtle); border-radius:var(--radius-md); background:var(--surface); padding:20px; }
|
||||
article.attention { border-left:3px solid var(--state-warning); }
|
||||
.row-heading,.row-footer { display:flex; justify-content:space-between; align-items:center; flex-wrap:wrap; gap:12px; }
|
||||
.row-heading { align-items:flex-start; }
|
||||
.eyebrow { font-size:var(--text-xs); color:var(--text-secondary); }
|
||||
h2 { margin:6px 0 0; font-size:var(--text-lg); }
|
||||
h2 a { color:var(--text-primary); text-decoration:none; }
|
||||
article p { font-size:var(--text-sm); color:var(--text-secondary); line-height:1.65; }
|
||||
.progress { font-size:var(--text-sm); }
|
||||
.resources { display:flex; flex-wrap:wrap; gap:8px 16px; margin-top:12px; font-size:var(--text-sm); }
|
||||
.resources span { color:var(--text-secondary); }
|
||||
.row-footer { margin-top:16px; }
|
||||
details { margin-top:16px; padding-top:16px; border-top:1px solid var(--border-subtle); font-size:var(--text-sm); }
|
||||
summary { cursor:pointer; }
|
||||
ul { list-style:none; padding:0; margin:16px 0 0; display:grid; gap:16px; }
|
||||
li { min-width:0; padding:12px; background:var(--surface-sunken); border-radius:var(--radius-sm); }
|
||||
li > span { display:block; margin-top:6px; color:var(--text-secondary); font-size:var(--text-xs); }
|
||||
li p { margin-bottom:0; }
|
||||
@media (max-width:600px) { article { padding:16px; } .filters label { flex:1 1 100%; } .row-footer { align-items:flex-start; } }
|
||||
</style>
|
||||
@@ -12,7 +12,7 @@ describe('management route permissions', () => {
|
||||
'/leaderboard/players/[game]/[guid]', '/api/campaigns/[id]', '/campaigns/[id]',
|
||||
'/api/provisioning/[id]', '/provisioning/[id]', '/api/provisioning/[id]/events',
|
||||
'/api/betas/[id]', '/api/scripts/[id]', '/api/configs', '/api/plugins', '/api/hosts/[id]',
|
||||
'/hosts/operations', '/api/hosts/operations',
|
||||
'/hosts/operations', '/api/hosts/operations', '/operations', '/api/operations',
|
||||
'/api/game-files/[id]', '/api/maintenance', '/api/events/servers', '/api/events/pipeline'
|
||||
])('lets viewers read the safe summary route %s', route => {
|
||||
expect(routePermission(route, 'GET')).toBe('viewer');
|
||||
|
||||
@@ -7,6 +7,7 @@ describe('application navigation', () => {
|
||||
['/collections/launches/42', 'fleet', '/collections'],
|
||||
['/hosts/42', 'fleet', '/hosts'],
|
||||
['/hosts/operations', 'operations', '/hosts/operations'],
|
||||
['/operations', 'operations', '/operations'],
|
||||
['/betas/new', 'operations', '/betas'],
|
||||
['/campaigns/42', 'operations', '/campaigns'],
|
||||
['/configs/42', 'library', '/configs'],
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
const now = '2026-10-05T12:00:00.000Z';
|
||||
const old = '2026-08-01T12:00:00.000Z';
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
||||
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
|
||||
|
||||
async function setup() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { operationsOverview } = await import('../../src/lib/server/operations-overview.js');
|
||||
const handle = db();
|
||||
const overview = () => operationsOverview(Date.parse(now));
|
||||
function job(id: string, state = 'done', server = id, timestamp = now) {
|
||||
handle.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by,observation_json)
|
||||
VALUES(?,?,'restart',?,?,?,?,'test','{"private":"observation-secret"}')`).run(id, server, state, state, timestamp, ['done','failed','warning'].includes(state) ? timestamp : null);
|
||||
}
|
||||
function campaign(id: string, state = 'pending', timestamp = now) {
|
||||
handle.prepare("INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)").run(id, timestamp);
|
||||
handle.prepare(`INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state,players,checked_at) VALUES(?,?,1,?,4,?)`).run(id, id, state, timestamp);
|
||||
}
|
||||
function incident(id: string, state = 'recovering', timestamp = now) {
|
||||
handle.prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
|
||||
VALUES(?,?,'health',?,'Recovery summary','{"private":"incident-secret"}',?,?)`).run(id, id, state, timestamp, timestamp);
|
||||
}
|
||||
return { handle, overview, job, campaign, incident };
|
||||
}
|
||||
|
||||
describe('Operations overview', () => {
|
||||
it('returns an empty summary and an uncached read-only API response', async () => {
|
||||
const api = await setup();
|
||||
expect(api.overview()).toMatchObject({ items: [], counts: { attention: 0, running: 0, recent: 0 }, recentDays: 7, recentLimit: 100 });
|
||||
const { GET } = await import('../../src/routes/api/operations/+server.js');
|
||||
const response = GET();
|
||||
expect(response.headers.get('cache-control')).toBe('no-store');
|
||||
expect((await response.json()).items).toEqual([]);
|
||||
});
|
||||
|
||||
it('separates empty-lobby waits, latest failures, and completed results', async () => {
|
||||
const api = await setup();
|
||||
api.campaign('occupied');
|
||||
api.job('waiting', 'waiting');
|
||||
api.job('failed', 'failed');
|
||||
api.job('completed');
|
||||
const result = api.overview();
|
||||
expect(result.counts).toEqual({ attention: 1, running: 2, recent: 1 });
|
||||
expect(result.items.find(item => item.key === 'campaign:occupied')).toMatchObject({ view: 'running', status: 'Waiting for empty lobbies' });
|
||||
expect(result.items.find(item => item.key === 'provisioning:failed')).toMatchObject({ view: 'attention' });
|
||||
expect(JSON.stringify(result)).not.toContain('observation-secret');
|
||||
});
|
||||
|
||||
it('groups collection attempts, retaining old unresolved launches until superseded', async () => {
|
||||
const api = await setup();
|
||||
api.job('one', 'failed', 'one', old);
|
||||
api.handle.prepare('INSERT INTO server_launch_collections(id,name,created_at) VALUES(?,?,?)').run('launch', 'Evening launch', old);
|
||||
api.handle.exec("INSERT INTO server_collection_members(collection_id,deployment_id,position) VALUES('launch','one',0)");
|
||||
const first = api.overview();
|
||||
expect(first.items).toHaveLength(1);
|
||||
expect(first.items[0]).toMatchObject({ key: 'collection:launch', view: 'attention', active: false, children: [{ key: 'provisioning:one' }] });
|
||||
api.job('retry', 'done', 'one');
|
||||
expect(api.overview().items.map(item => item.key)).toEqual(['provisioning:retry']);
|
||||
});
|
||||
|
||||
it('keeps all recovery attempts together without reviving resolved incidents', async () => {
|
||||
const api = await setup();
|
||||
api.incident('recovery', 'resolved');
|
||||
api.job('attempt', 'failed', 'recovery');
|
||||
api.campaign('repair', 'failed');
|
||||
api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(now);
|
||||
const result = api.overview();
|
||||
expect(result.items).toHaveLength(1);
|
||||
expect(result.items[0]).toMatchObject({ key: 'incident:recovery', view: 'recent', active: false });
|
||||
expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']);
|
||||
expect(JSON.stringify(result)).not.toContain('incident-secret');
|
||||
});
|
||||
|
||||
it('retains older repair Campaign links within an active incident', async () => {
|
||||
const api = await setup();
|
||||
api.incident('recovery', 'attention');
|
||||
api.job('attempt', 'done', 'recovery', old);
|
||||
api.campaign('repair', 'done', old);
|
||||
api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at) VALUES('a','recovery','rebuild','attempt','repair',?)`).run(old);
|
||||
const result = api.overview();
|
||||
expect(result.items).toHaveLength(1);
|
||||
expect(result.items[0].children.map(item => item.key)).toEqual(['campaign:repair', 'provisioning:attempt']);
|
||||
});
|
||||
|
||||
it('does not resurface failed attempts from resolved incidents outside recent history', async () => {
|
||||
const api = await setup();
|
||||
api.incident('recovery', 'resolved', old);
|
||||
api.job('attempt', 'failed', 'recovery', old);
|
||||
api.handle.prepare(`INSERT INTO watchdog_actions(id,incident_id,action,operation_id,created_at) VALUES('a','recovery','restart','attempt',?)`).run(old);
|
||||
expect(api.overview().items).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps source-less attempts accessible when a recorded parent is unavailable', async () => {
|
||||
const api = await setup();
|
||||
api.job('orphan', 'failed');
|
||||
api.handle.exec("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('orphan','campaign','missing','orphan')");
|
||||
expect(api.overview().items).toMatchObject([{ key: 'provisioning:orphan', view: 'attention' }]);
|
||||
});
|
||||
|
||||
it('shows Beta review and test failures as attention while empty-lobby waits keep running', async () => {
|
||||
const api = await setup();
|
||||
const script = api.handle.prepare("INSERT INTO scripts(name,target_rel_path) VALUES('script','scripts/test.gsc')").run().lastInsertRowid;
|
||||
const version = api.handle.prepare("INSERT INTO script_versions(script_id,version,sha256,size,content) VALUES(?,1,'hash',1,X'31')").run(script).lastInsertRowid;
|
||||
for (const [id, state, stage, outcome] of [[1,'running','awaiting-empty','active'],[2,'ready-promote','ready','passed'],[3,'running','ready','paused'],[4,'promoted','ready','passed']] as const) {
|
||||
api.handle.prepare("INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at) VALUES(?,?,?,'health',?,?)").run(id,script,version,state,now);
|
||||
api.handle.prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage,updated_at) VALUES(?,?,0,?,?)").run(id,`server-${id}`,stage,now);
|
||||
api.handle.prepare("INSERT INTO beta_test_plans(beta_id,plan,outcome) VALUES(?,'{}',?)").run(id,outcome);
|
||||
}
|
||||
const result = api.overview();
|
||||
expect(result.counts).toEqual({ attention: 2, running: 1, recent: 1 });
|
||||
expect(result.items.find(item => item.key === 'beta:1')?.status).toBe('Waiting for empty lobbies');
|
||||
api.campaign('promotion','done');
|
||||
api.handle.exec("UPDATE deployments SET beta_id=4 WHERE id='promotion'");
|
||||
expect(api.overview().items.find(item => item.key === 'campaign:promotion')?.related).toEqual([{ label: 'Beta #4', href: '/betas/4' }]);
|
||||
});
|
||||
|
||||
it('omits healthy routine checks and superseded routine failures without loading private data', async () => {
|
||||
const api = await setup();
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.createHost({ name: 'Test host' });
|
||||
const first = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance', password: 'payload-secret' });
|
||||
hosts.reportHostOperationStatus(host.id,first.id,{ state:'failed', error: 'Unreachable', result: { password: 'result-secret' } });
|
||||
api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now);
|
||||
expect(api.overview().items).toMatchObject([{ key: `host:${first.id}`, view:'attention' }]);
|
||||
const next = hosts.queueHostOperation(host.id,'instance.inspect',{ instanceId: 'instance' });
|
||||
hosts.reportHostOperationStatus(host.id,next.id,{ state:'completed', result: { healthy:true } });
|
||||
api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now);
|
||||
expect(api.overview().items).toEqual([]);
|
||||
const action = hosts.queueHostOperation(host.id,'instance.restart',{ password: 'payload-secret' });
|
||||
api.handle.prepare('UPDATE ec_host_operations SET updated_at=?').run(now);
|
||||
const result = api.overview();
|
||||
expect(result.items).toMatchObject([{ key: `host:${action.id}`, view:'running' }]);
|
||||
expect(JSON.stringify(result)).not.toContain('secret');
|
||||
});
|
||||
|
||||
it('bounds finished history while retaining older active work and removing deleted resource links', async () => {
|
||||
const api = await setup();
|
||||
api.job('old-result', 'done', 'old-result', old);
|
||||
api.job('old-active', 'waiting', 'old-active', old);
|
||||
for (let index = 0; index < 120; index++) api.job(`result-${String(index).padStart(3,'0')}`);
|
||||
api.handle.prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id=?').run(now,'result-119');
|
||||
const result = api.overview();
|
||||
expect(result.counts).toEqual({ attention:0,running:1,recent:100 });
|
||||
expect(result.items.find(item => item.key === 'provisioning:old-active')?.view).toBe('running');
|
||||
expect(result.items.find(item => item.key === 'provisioning:result-119')?.resources[0].href).toBeNull();
|
||||
expect(result.items.some(item => item.key === 'provisioning:old-result')).toBe(false);
|
||||
});
|
||||
|
||||
it('groups Maintenance and its provisioning and host work while keeping the workflow actionable', async () => {
|
||||
const api = await setup();
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.createHost({ name: 'Test host' });
|
||||
const operation = hosts.queueHostOperation(host.id, 'instance.restart', {});
|
||||
api.job('restart', 'running');
|
||||
api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','restart',?,'library','image',4976,?)`).run(host.id, now);
|
||||
api.handle.prepare(`INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('restart','instance',1,?)`).run(operation.id);
|
||||
api.handle.prepare(`INSERT INTO maintenance_jobs(server,state,requested_at,provisioning_id) VALUES('restart','restarting',?,'restart')`).run(now);
|
||||
const result = api.overview();
|
||||
expect(result.items).toHaveLength(1);
|
||||
expect(result.items[0]).toMatchObject({ kind: 'maintenance', view: 'running', children: [{ key: 'provisioning:restart' }] });
|
||||
expect(result.items[0].href).toContain('server=restart');
|
||||
expect(result.items[0].resources).toContainEqual({ label: 'Test host', href: `/hosts/${host.id}` });
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user