From a6c40cde9bc03cf9f52cc38f1ce9a18755ce74ce Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 5 Oct 2026 13:23:11 -0400 Subject: [PATCH] Add filters and context to host operation history --- docs/native-hosts.md | 29 ++- src/hooks.server.ts | 4 +- src/lib/hosts.ts | 10 +- src/lib/server/hosts/index.ts | 32 +++- src/lib/server/hosts/observations.ts | 26 ++- src/lib/server/hosts/operation-context.ts | 12 ++ src/lib/server/hosts/schema.ts | 18 +- src/lib/server/maintenance/recovery.ts | 4 +- src/lib/server/pipeline/activity.ts | 3 +- src/lib/server/pipeline/worker.ts | 3 +- src/routes/api/hosts/operations/+server.ts | 8 +- src/routes/hosts/operations/+page.server.ts | 13 +- src/routes/hosts/operations/+page.svelte | 56 +++++- tests/unit/host-observation-retention.test.ts | 1 + tests/unit/host-operation-history.test.ts | 180 ++++++++++++++++-- tests/unit/operation-context.test.ts | 64 +++++++ 16 files changed, 417 insertions(+), 46 deletions(-) create mode 100644 src/lib/server/hosts/operation-context.ts create mode 100644 tests/unit/operation-context.test.ts diff --git a/docs/native-hosts.md b/docs/native-hosts.md index d866dce..c18a519 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -60,10 +60,20 @@ or enrolling another host needs no manual propagation. Setup waits for an active healthy host with sufficient storage. Failed operations retry automatically with backoff from one minute to fifteen minutes; their errors remain visible in the host's setup panel and **Hosts → Operation history**. This shared history lists -operations across hosts, newest first, with a host column and 25 rows per page. +operations across hosts, newest first, with 25 rows per page and host, target, +and source details. It defaults to **Actions & errors**; **Routine checks** and +**All operations** include successful monitoring reads. Older records may lack +source details. Disconnected hosts resume when they reconnect. Draining and disabled hosts do not receive new automatic setup work. +**Settings → Routine operation history retention (days)** controls cleanup of +successful monitoring and file-read records. The default is one day; zero keeps +them indefinitely. Actions, failures, negative checks, and unresolved operations +are retained. Agent 1.4.9 also cleans up successful routine journal records after +the controller confirms their final receipt. Older unconfirmed journal entries +remain until a replay supplies that confirmation. Cleanup runs in bounded batches. + An automatic download that has not been delivered to the agent can be replaced when a newer upload becomes available. A delivered operation finishes safely before the host prepares the latest files. Failed or incomplete uploads never @@ -287,8 +297,11 @@ explicit installer `--reenroll` when local credentials already exist. API outline: - `GET /api/hosts/:id`: host details and deletion blockers. -- `GET /api/hosts/operations?page=1`: operation history across hosts, newest first, - with 25 summaries per page and total/page counts. Payloads and results are omitted. +- `GET /api/hosts/operations?page=1&filter=important`: operation history across + hosts, newest first, with 25 summaries per page and filtered total/page counts. + Filters are `important` (default), `routine`, and `all`. Summaries include the + server, safe target path or status command, and initiating source when recorded. + Payloads and results are omitted. - `DELETE /api/hosts/:id`: administrator requests cleanup of an unused host; returns `202` while remote removal is pending, `200` for an unenrolled host, or `409` when the agent needs updating or unresolved ownership remains. @@ -551,8 +564,14 @@ of which task delayed a previous deployment. Campaign, beta and maintenance observers run independently of provisioning. New requests and completed agent commands wake provisioning; an arrival during a pass schedules another pass. Routine file drift scans run every five minutes; -manual checks bypass that interval. Overlapping identical native observations -share one operation until it completes, without caching results for later checks. +manual checks bypass that interval. Agent 1.4.9 supports one hash manifest per +drift scan, avoiding separate list/stat/read operations for unchanged files. +Older agents use the existing file checks. Overlapping identical native +observations share one operation until it completes. Routine status monitoring +and maintenance additionally share successful health results for up to 30 seconds +from the request start, preserving the original observation time. Changed server +identity or lifecycle intent invalidates reuse. Manual checks and disruption +safety checks continue to request fresh evidence. Agent 1.4.5 reserves two observation workers for instance health and read-only RCON, and one for files, logs, and host inventory. Background reads arrive in batches of up to four. File receipt uploads release the instance lock before contacting the diff --git a/src/hooks.server.ts b/src/hooks.server.ts index 8f5e151..7d7deaf 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -5,6 +5,7 @@ import { interruptDeployments } from '$lib/server/provisioning/history.js'; import { authenticationHandle } from '$lib/server/auth/guard.js'; import { emit } from '$lib/server/pipeline/bus.js'; import { startPerformanceMonitor } from '$lib/server/performance.js'; +import { withOperationSource } from '$lib/server/hosts/operation-context.js'; let booted = false; function boot(): void { @@ -22,7 +23,8 @@ function boot(): void { export const handle: Handle = async (input) => { boot(); const started = performance.now(); - const response = await authenticationHandle(input); + const response = await withOperationSource(`${input.event.request.method} ${input.event.route.id ?? 'Web request'}`, + () => authenticationHandle(input)); const elapsed = performance.now() - started; response.headers.append('Server-Timing', `app;dur=${elapsed.toFixed(1)}`); if (process.env.EC_PERF_LOG === 'true' && elapsed > 250 && !input.event.route.id?.startsWith('/api/agent/')) { diff --git a/src/lib/hosts.ts b/src/lib/hosts.ts index 5f2c1d3..ebe039c 100644 --- a/src/lib/hosts.ts +++ b/src/lib/hosts.ts @@ -102,9 +102,17 @@ export interface HostOperation { } export interface HostOperationSummary extends Omit { hostName: string | null; + serverName: string | null; + targetPath: string | null; + source: string | null; + routine: boolean; +} +export type HostOperationFilter = 'important' | 'routine' | 'all'; +export function isHostOperationFilter(value: unknown): value is HostOperationFilter { + return typeof value === 'string' && ['important', 'routine', 'all'].includes(value); } export interface HostOperationHistoryPage { - operations: HostOperationSummary[]; page: number; pageSize: 25; total: number; totalPages: number; + operations: HostOperationSummary[]; page: number; pageSize: 25; total: number; totalPages: number; filter: HostOperationFilter; } export interface HostPreparationItem { diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index d1b32be..0c8288f 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -4,11 +4,12 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt import { isIP } from 'node:net'; import { db } from '../db/index.js'; import { ensureHostSchema } from './schema.js'; -import { successfulObservation } from './observations.js'; +import { successfulObservation, routineObservationSql, importantOperationSql, operationTargetSql, safeOperationTarget } from './observations.js'; +import { currentOperationSource } from './operation-context.js'; import { fleetSettings } from '../settings.js'; import { recordInstanceResources, validateInstanceResources } from './calibration.js'; import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js'; -import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationHistoryPage, HostOperationState } from '../../hosts.js'; +import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationFilter, HostOperationHistoryPage, HostOperationState } from '../../hosts.js'; export const AGENT_PROTOCOL_VERSION = 1; export const HEARTBEAT_MAX_AGE_MS = 90_000; @@ -133,7 +134,8 @@ export function deleteHost(id: string): { deleted: boolean; operationId?: string const operationId = randomUUID(); const token = 'ec_cleanup_' + randomBytes(32).toString('base64url'); const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` }; - handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp); + handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,source) VALUES(?,?,'host.decommission',?,'pending',?,?,?)") + .run(operationId, id, JSON.stringify(payload), timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200)); handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp); handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?) ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`) @@ -359,19 +361,26 @@ export function listHostOperations(hostId: string, limit = 100, summary = false) .all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); } /** Browse retained history without loading command payloads or credential-bearing results. */ -export function listHostOperationHistory(page = 1): HostOperationHistoryPage { +export function listHostOperationHistory(page = 1, filter: HostOperationFilter = 'important'): HostOperationHistoryPage { const handle = database(); const pageSize = 25; + const where = filter === 'all' ? '' : `WHERE ${filter === 'routine' ? 'o.routine=1' : importantOperationSql}`; return handle.transaction((): HostOperationHistoryPage => { - const { total } = handle.prepare('SELECT COUNT(*) AS total FROM ec_host_operations').get() as { total: number }; + const { total } = handle.prepare(`SELECT COUNT(*) AS total FROM ec_host_operations o ${where}`).get() as { total: number }; const totalPages = Math.max(1, Math.ceil(total / pageSize)); const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1); const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind, - o.instance_id AS instanceId,o.generation,o.state,o.error,o.created_at AS createdAt,o.updated_at AS updatedAt + o.instance_id AS instanceId,o.generation,o.state, + CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0 + THEN COALESCE(NULLIF(o.error,''),'Observation reported a problem') ELSE o.error END AS error, + o.created_at AS createdAt,o.updated_at AS updatedAt,o.routine,o.server_name AS serverName,o.target_path AS targetPath,o.source FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL + ${where} ORDER BY o.created_at DESC,o.rowid DESC LIMIT ? OFFSET ?`) .all(pageSize, (currentPage - 1) * pageSize) as HostOperationHistoryPage['operations']; - return { operations, page: currentPage, pageSize, total, totalPages }; + return { operations: operations.map(operation => ({ ...operation, routine: !!operation.routine, + targetPath: safeOperationTarget(operation.kind, operation.targetPath) })), filter, + page: currentPage, pageSize, total, totalPages }; })(); } @@ -428,8 +437,15 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu if (['instance.create','create'].includes(kind)) requireReadyHost(hostId); if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.'); const id = options.id ?? randomUUID(); const timestamp = now(); - database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, routingPayload ?? payload, timestamp, timestamp); + const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?') + .get(options.instanceId) as { server: string } | undefined)?.server ?? null : null; + database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name) + VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?)`) + .run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, + routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName); if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload); + database().prepare(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END, + target_path=${operationTargetSql} WHERE o.id=?`).run(id); database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp); return getHostOperationStatus(id)!; })(); diff --git a/src/lib/server/hosts/observations.ts b/src/lib/server/hosts/observations.ts index 6466a95..2cb19c1 100644 --- a/src/lib/server/hosts/observations.ts +++ b/src/lib/server/hosts/observations.ts @@ -1,6 +1,13 @@ export const observationKinds = ['host.inspect', 'instance.inspect', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read']; const commands = ['status', 'getstatus', 'version']; +export function safeOperationTarget(kind: string, value: string | null): string | null { + if (!value || value.length > 1024) return null; + const paths = kind === 'files.manifest' ? value.split(', ') : [value]; + return paths.every(path => path === '.' || !/[\\\x00-\x1f\x7f]/.test(path) + && path.split('/').every(part => part !== '' && part !== '.' && part !== '..')) ? value : null; +} + export function successfulObservation(kind: string, payload: Record, result: unknown): boolean { const routine = observationKinds.includes(kind) || kind === 'rcon.execute' && commands.includes(String(payload.command)); const value = result && typeof result === 'object' ? result as Record : {}; @@ -8,11 +15,24 @@ export function successfulObservation(kind: string, payload: Record `'${kind}'`).join(',')}) - OR (o.kind='rcon.execute' AND json_extract(COALESCE( - (SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')))`; +export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')}) THEN 1 + WHEN o.kind='rcon.execute' THEN json_extract(COALESCE( + (SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version') + ELSE 0 END)`; export const successfulObservationSql = `(o.state='completed' AND COALESCE(o.error,'')='' AND ${routineObservationSql} AND CASE WHEN o.result IS NULL THEN 1 WHEN json_valid(o.result) THEN json_extract(o.result,'$.ok') IS NOT 0 AND json_extract(o.result,'$.healthy') IS NOT 0 AND COALESCE(json_extract(o.result,'$.error'),'')='' ELSE 0 END)`; + +export const importantOperationSql = `(o.routine=0 OR o.state IN ('failed','unknown') + OR (o.state='completed' AND o.observation_success=0) OR COALESCE(o.error,'')!='')`; + +const payloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'; +/** Deliberately exclude arbitrary RCON commands and all file contents/credentials. */ +export const operationTargetSql = `CASE + WHEN o.kind IN ('files.read','files.stat','files.list','logs.read') THEN + CASE WHEN json_type(${payloadSql},'$.path')='text' THEN substr(json_extract(${payloadSql},'$.path'),1,1024) END + WHEN o.kind='files.manifest' THEN substr((SELECT group_concat(value,', ') FROM json_each(${payloadSql},'$.roots') WHERE type='text'),1,1024) + WHEN o.kind='rcon.execute' AND ${routineObservationSql} THEN json_extract(${payloadSql},'$.command') + ELSE NULL END`; diff --git a/src/lib/server/hosts/operation-context.ts b/src/lib/server/hosts/operation-context.ts new file mode 100644 index 0000000..96dbe5f --- /dev/null +++ b/src/lib/server/hosts/operation-context.ts @@ -0,0 +1,12 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; + +const operationSource = new AsyncLocalStorage(); + +/** Carry the initiating request or background task through asynchronous work. */ +export function withOperationSource(source: string, work: () => T): T { + return operationSource.run(source, work); +} + +export function currentOperationSource(): string | null { + return operationSource.getStore() ?? null; +} diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts index 6c2ae47..b64f60b 100644 --- a/src/lib/server/hosts/schema.ts +++ b/src/lib/server/hosts/schema.ts @@ -1,5 +1,5 @@ import type Database from 'better-sqlite3'; -import { successfulObservationSql } from './observations.js'; +import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js'; const initialized = new WeakSet(); export function ensureHostSchema(handle: Database.Database): void { @@ -30,6 +30,7 @@ export function ensureHostSchema(handle: Database.Database): void { CHECK (state IN ('pending','accepted','running','completed','failed','unknown')), result TEXT, error TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, last_delivered_at TEXT, observation_success INTEGER NOT NULL DEFAULT 0, + routine INTEGER NOT NULL DEFAULT 0, server_name TEXT, target_path TEXT, source TEXT, UNIQUE(host_id, idempotency_key) ); CREATE INDEX IF NOT EXISTS ec_host_operations_queue ON ec_host_operations(host_id, state, created_at); @@ -93,6 +94,21 @@ export function ensureHostSchema(handle: Database.Database): void { } handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_observation_retention ON ec_host_operations(updated_at) WHERE observation_success=1 AND state='completed'`); + if (!columns.some(column => column.name === 'routine')) { + handle.transaction(() => { + handle.exec(`ALTER TABLE ec_host_operations ADD COLUMN routine INTEGER NOT NULL DEFAULT 0; + ALTER TABLE ec_host_operations ADD COLUMN server_name TEXT; + ALTER TABLE ec_host_operations ADD COLUMN target_path TEXT; + ALTER TABLE ec_host_operations ADD COLUMN source TEXT;`); + const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as { name: string }[]; + const nativeInstances = ['id', 'server'].every(name => nativeColumns.some(column => column.name === name)); + handle.exec(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END, + target_path=${operationTargetSql}, server_name=${nativeInstances ? '(SELECT n.server FROM native_instances n WHERE n.id=o.instance_id)' : 'NULL'}`); + })(); + } + handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_routine_history ON ec_host_operations(routine,created_at); + CREATE INDEX IF NOT EXISTS ec_host_operations_important_history ON ec_host_operations(created_at) + WHERE ${importantOperationSql.replaceAll('o.', '')}`); const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[]; if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT'); const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[]; diff --git a/src/lib/server/maintenance/recovery.ts b/src/lib/server/maintenance/recovery.ts index 4c0abfb..1eeb1be 100644 --- a/src/lib/server/maintenance/recovery.ts +++ b/src/lib/server/maintenance/recovery.ts @@ -10,6 +10,7 @@ import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from ' import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js'; import { deploymentStage, finishDeployment } from '../provisioning/history.js'; import { recordProgress } from '../provisioning/progress.js'; +import { withOperationSource } from '../hosts/operation-context.js'; import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js'; import { maintenanceAllowed, setMaintenancePolicy } from './policies.js'; @@ -309,7 +310,8 @@ async function scanServer(server: string, forceScan: boolean): Promise { let inFlight: Promise | null = null; export function watchdogTick(forceScan = false): Promise { if (inFlight) return inFlight; - inFlight = run(forceScan).finally(() => { inFlight = null; }); + inFlight = withOperationSource(forceScan ? 'Manual maintenance check' : 'Maintenance watchdog', () => run(forceScan)) + .finally(() => { inFlight = null; }); return inFlight; } async function run(forceScan: boolean): Promise { diff --git a/src/lib/server/pipeline/activity.ts b/src/lib/server/pipeline/activity.ts index f569906..80fec85 100644 --- a/src/lib/server/pipeline/activity.ts +++ b/src/lib/server/pipeline/activity.ts @@ -1,4 +1,5 @@ import type { WorkerActivity } from '../../provisioning-detail.js'; +import { currentOperationSource, withOperationSource } from '../hosts/operation-context.js'; const active = new Map(); let sequence = 0; @@ -11,6 +12,6 @@ export function workerActivity(): WorkerActivity[] { export async function trackActivity(key: string, label: string, work: () => Promise): Promise { if (active.has(key)) key = `${key}:${++sequence}`; active.set(key, { key, label, startedAt: new Date().toISOString() }); - try { return await work(); } + try { return await withOperationSource(currentOperationSource() ?? label, work); } finally { active.delete(key); } } diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index dd5862e..c5cb477 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -17,6 +17,7 @@ import { selectedZombieMap } from '../../map-aliases.js'; import { runMaintenance } from '../maintenance/index.js'; import { cleanupControllerLibraries } from '../game-files/index.js'; import { trackActivity } from './activity.js'; +import { withOperationSource } from '../hosts/operation-context.js'; const TICK_MS = 15_000; // Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the @@ -104,7 +105,7 @@ function queueStatus(server: string): Promise { statusQueue.push(async () => { let capacity: number | null = null; try { - capacity = await refreshServerStatus(server); + capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server)); // Publish this server's latest evidence without waiting for the fleet. const current = getServer(server); if (current) publishStatus(statusEntry(current)); diff --git a/src/routes/api/hosts/operations/+server.ts b/src/routes/api/hosts/operations/+server.ts index fc6642d..25c8121 100644 --- a/src/routes/api/hosts/operations/+server.ts +++ b/src/routes/api/hosts/operations/+server.ts @@ -1,10 +1,14 @@ import { error, json } from '@sveltejs/kit'; import { listHostOperationHistory } from '$lib/server/hosts/index.js'; import { requireManagementOrigin } from '$lib/server/hosts/http.js'; +import { isHostOperationFilter } from '$lib/hosts.js'; export function GET({ request }: { request: Request }) { requireManagementOrigin(request); - const page = Number(new URL(request.url).searchParams.get('page') ?? '1'); + const params = new URL(request.url).searchParams; + const page = Number(params.get('page') ?? '1'); + const filter = params.get('filter') ?? 'important'; if (!Number.isSafeInteger(page) || page < 1) error(400, 'Invalid operation history page.'); - return json(listHostOperationHistory(page), { headers: { 'Cache-Control': 'no-store' } }); + if (!isHostOperationFilter(filter)) error(400, 'Invalid operation history filter.'); + return json(listHostOperationHistory(page, filter), { headers: { 'Cache-Control': 'no-store' } }); } diff --git a/src/routes/hosts/operations/+page.server.ts b/src/routes/hosts/operations/+page.server.ts index 4dac42e..b4c6a8f 100644 --- a/src/routes/hosts/operations/+page.server.ts +++ b/src/routes/hosts/operations/+page.server.ts @@ -1,12 +1,19 @@ import { redirect } from '@sveltejs/kit'; import { listHostOperationHistory } from '$lib/server/hosts/index.js'; +import { isHostOperationFilter } from '$lib/hosts.js'; import type { PageServerLoad } from './$types'; export const load: PageServerLoad = ({ url }) => { const requestedPage = url.searchParams.get('page'); - const history = listHostOperationHistory(Number(requestedPage ?? 1)); - if (requestedPage !== null && requestedPage !== String(history.page)) { - redirect(307, history.page === 1 ? '/hosts/operations' : `/hosts/operations?page=${history.page}`); + const requestedFilter = url.searchParams.get('filter'); + const filter = isHostOperationFilter(requestedFilter) ? requestedFilter : 'important'; + const history = listHostOperationHistory(Number(requestedPage ?? 1), filter); + if ((requestedPage !== null && requestedPage !== String(history.page)) + || (requestedFilter !== null && requestedFilter !== filter)) { + const params = new URLSearchParams(); + if (requestedFilter !== null) params.set('filter', filter); + if (history.page !== 1) params.set('page', String(history.page)); + redirect(307, `/hosts/operations${params.size ? `?${params}` : ''}`); } return { history }; }; diff --git a/src/routes/hosts/operations/+page.svelte b/src/routes/hosts/operations/+page.svelte index 6036e6f..06bd280 100644 --- a/src/routes/hosts/operations/+page.svelte +++ b/src/routes/hosts/operations/+page.svelte @@ -2,7 +2,7 @@ import { onMount, untrack } from 'svelte'; import { beforeNavigate } from '$app/navigation'; import { navigating } from '$app/stores'; - import type { HostOperationHistoryPage } from '$lib/hosts.js'; + import type { HostOperationFilter, HostOperationHistoryPage } from '$lib/hosts.js'; import type { PageData } from './$types'; import { pollVisible } from '$ui/poll.js'; import { timeFormat } from '$ui/stores/time-zone.js'; @@ -19,7 +19,17 @@ const changingPage = $derived($navigating?.to?.url.pathname === '/hosts/operations'); const firstOperation = $derived((history.page - 1) * history.pageSize + 1); const numbers = new Intl.NumberFormat('en-US'); - function pageHref(page: number) { return page === 1 ? '/hosts/operations' : `/hosts/operations?page=${page}`; } + const filters: Array<{ id: HostOperationFilter; label: string; description: string; empty: string }> = [ + { id: 'important', label: 'Actions & errors', description: 'Actions, failures, and checks that need attention.', empty: 'No actions or checks needing attention recorded.' }, + { id: 'routine', label: 'Routine checks', description: 'Monitoring and file reads across all states.', empty: 'No routine checks recorded.' }, + { id: 'all', label: 'All operations', description: 'Every recorded remote operation.', empty: 'No remote operations yet.' } + ]; + const selectedFilter = $derived(filters.find(filter => filter.id === history.filter)!); + function pageHref(page: number, filter: HostOperationFilter = history.filter) { + const params = new URLSearchParams({ filter }); + if (page !== 1) params.set('page', String(page)); + return `/hosts/operations?${params}`; + } function cancel() { sequence++; request?.abort(); request = null; refreshing = false; } async function refresh(signal?: AbortSignal) { @@ -30,7 +40,8 @@ request = controller; refreshing = true; try { - const response = await fetch(`/api/hosts/operations?page=${history.page}`, { + const params = new URLSearchParams({ page: String(history.page), filter: history.filter }); + const response = await fetch(`/api/hosts/operations?${params}`, { signal: AbortSignal.any([controller.signal, AbortSignal.timeout(15000), ...(signal ? [signal] : [])]) }); const body = await response.json(); @@ -55,18 +66,32 @@ {#snippet actions()}{/snippet} +
+ +

{selectedFilter.description}

+
{#if refreshError}{/if}
- {#if !history.operations.length}

No remote operations yet.

+ {#if !history.operations.length}

{selectedFilter.empty}

{:else}
- + {#each history.operations as operation (operation.id)} - + + + @@ -88,14 +113,25 @@ diff --git a/tests/unit/host-observation-retention.test.ts b/tests/unit/host-observation-retention.test.ts index 42e4ae0..1e89975 100644 --- a/tests/unit/host-observation-retention.test.ts +++ b/tests/unit/host-observation-retention.test.ts @@ -123,6 +123,7 @@ describe('routine host observation retention', () => { api.handle.prepare('UPDATE ec_host_operations SET result=? WHERE id=?').run('{not-json', malformed); retained.push(malformed); api.handle.exec('DROP INDEX ec_host_operations_observation_retention'); + api.handle.exec('DROP INDEX ec_host_operations_important_history'); api.handle.exec('ALTER TABLE ec_host_operations DROP COLUMN observation_success'); api.closeDb(); diff --git a/tests/unit/host-operation-history.test.ts b/tests/unit/host-operation-history.test.ts index 073aa90..88c6355 100644 --- a/tests/unit/host-operation-history.test.ts +++ b/tests/unit/host-operation-history.test.ts @@ -30,27 +30,43 @@ async function seedHistory(count = 53) { return { ...hosts, handle, ids, first, second, deleted }; } +async function operationHistory() { + const api = await seedHistory(0); + function add(kind: string, options: { payload?: Record; state?: string; result?: unknown; error?: string } = {}) { + const operation = api.queueHostOperation(api.first.id, kind, options.payload ?? {}); + if (options.state !== 'pending') api.reportHostOperationStatus(api.first.id, operation.id, { + state: options.state ?? 'completed', result: options.result, error: options.error + }); + return operation.id; + } + return { ...api, add }; +} + describe('host operation history', () => { it('pages all hosts in stable newest-first order, retaining deleted-host records', async () => { const api = await seedHistory(); - const pages = [1, 2, 3].map(page => api.listHostOperationHistory(page)); + const pages = [1, 2, 3].map(page => api.listHostOperationHistory(page, 'all')); expect(pages.map(page => page.operations.length)).toEqual([25, 25, 3]); - pages.forEach((page, index) => expect(page).toMatchObject({ page: index + 1, pageSize: 25, total: 53, totalPages: 3 })); + pages.forEach((page, index) => expect(page).toMatchObject({ page: index + 1, pageSize: 25, total: 53, totalPages: 3, filter: 'all' })); const operations = pages.flatMap(page => page.operations); expect(operations.map(operation => operation.id)).toEqual([...api.ids].reverse()); expect(operations.find(operation => operation.hostId === api.first.id)?.hostName).toBe('Alpha host'); expect(operations.find(operation => operation.hostId === api.second.id)?.hostName).toBe('Beta host'); expect(operations.filter(operation => operation.hostId === api.deleted.id).length).toBeGreaterThan(0); expect(operations.filter(operation => operation.hostId === api.deleted.id).every(operation => operation.hostName === null)).toBe(true); - expect(api.listHostOperationHistory(999)).toEqual(pages[2]); + expect(api.listHostOperationHistory(999, 'all')).toEqual(pages[2]); }); it('bounds reads in SQLite using the history index and omits private payloads and results', async () => { const api = await seedHistory(1003); const prepare = vi.spyOn(api.handle, 'prepare'); - const history = api.listHostOperationHistory(2); + const history = api.listHostOperationHistory(2, 'all'); + api.listHostOperationHistory(1, 'important'); + api.listHostOperationHistory(1, 'routine'); + const statements = prepare.mock.calls.map(([sql]) => sql); const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o LEFT JOIN'))!; prepare.mockRestore(); + expect(statements.every(sql => !/\b(?:payload|result|json_extract|ec_host_operation_payloads)\b/.test(sql))).toBe(true); const projection = query.split('FROM ec_host_operations')[0]; expect(projection).not.toContain('payload'); expect(projection).not.toContain('result'); @@ -65,28 +81,167 @@ describe('host operation history', () => { expect(JSON.stringify(history)).not.toContain('secret'); }); + it('seeks through the selected history index without sorting or reading private payloads', async () => { + const api = await seedHistory(1003); + for (const filter of ['important', 'routine'] as const) { + const prepare = vi.spyOn(api.handle, 'prepare'); + api.listHostOperationHistory(2, filter); + const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o LEFT JOIN'))!; + prepare.mockRestore(); + const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(25, 25) as { detail: string }[]; + expect(plan.some(row => row.detail.includes(`USING INDEX ec_host_operations_${filter}_history`))).toBe(true); + expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false); + expect(query).not.toMatch(/\b(?:payload|result|json_extract)\b/); + } + }); + it('returns an empty first page and normalizes invalid internal page numbers', async () => { const api = await seedHistory(0); - const empty = { operations: [], page: 1, pageSize: 25, total: 0, totalPages: 1 }; + const empty = { operations: [], page: 1, pageSize: 25, total: 0, totalPages: 1, filter: 'important' }; expect(api.listHostOperationHistory()).toEqual(empty); for (const page of [0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, 999]) { expect(api.listHostOperationHistory(page)).toEqual(empty); } }); + it('defaults to actions and errors, with independent counts and pagination for every filter', async () => { + const api = await operationHistory(); + const actions = Array.from({ length: 26 }, () => api.add('instance.restart')); + actions.push(api.add('rcon.execute', { payload: { command: 'map_restart' } })); + const successful = Array.from({ length: 26 }, () => api.add('files.manifest', { result: { files: [] } })); + const pending = ['pending', 'accepted', 'running'].map(state => api.add('files.read', { state })); + const failures = ['failed', 'unknown'].map(state => api.add('files.read', { state, error: 'Observation unavailable' })); + const negative = [ + api.add('rcon.execute', { payload: { command: 'status' }, result: { ok: false, reason: 'timeout' } }), + api.add('instance.inspect', { result: { healthy: false, state: 'stopped' } }), + api.add('files.read', { result: { error: 'Read was incomplete' } }), + api.add('files.stat', { error: 'Could not inspect the file' }) + ]; + const ids = [...actions, ...successful, ...pending, ...failures, ...negative]; + for (const [filter, matching] of [ + ['important', [...actions, ...failures, ...negative]], + ['routine', [...successful, ...pending, ...failures, ...negative]], + ['all', ids] + ] as const) { + const pages = Array.from({ length: Math.ceil(matching.length / 25) }, (_, index) => api.listHostOperationHistory(index + 1, filter)); + expect(pages[0]).toMatchObject({ filter, total: matching.length, totalPages: pages.length, pageSize: 25 }); + expect(pages.flatMap(page => page.operations.map(operation => operation.id))).toEqual([...matching].reverse()); + expect(api.listHostOperationHistory(999, filter)).toEqual(pages.at(-1)); + } + expect(api.listHostOperationHistory()).toEqual(api.listHostOperationHistory(1, 'important')); + const routine = api.listHostOperationHistory(1, 'routine').operations; + expect(routine.every(operation => operation.routine)).toBe(true); + expect(api.listHostOperationHistory(2, 'important').operations.every(operation => !operation.routine)).toBe(true); + }); + + it('keeps manual RCON commands visible and only exposes safe read command names', async () => { + const api = await operationHistory(); + const reads = ['status', 'getstatus', 'version'].map(command => api.add('rcon.execute', { payload: { command } })); + const manual = api.add('rcon.execute', { payload: { command: 'say credential-secret' } }); + const combined = api.add('rcon.execute', { payload: { command: 'status; map_restart' } }); + const important = api.listHostOperationHistory(); + expect(important.operations.map(operation => operation.id)).toEqual([combined, manual]); + expect(important.operations.every(operation => !operation.routine && operation.targetPath === null)).toBe(true); + const routine = api.listHostOperationHistory(1, 'routine'); + expect(routine.operations.map(operation => operation.id)).toEqual([...reads].reverse()); + expect(routine.operations.map(operation => operation.targetPath)).toEqual(['version', 'getstatus', 'status']); + expect(JSON.stringify(api.listHostOperationHistory(1, 'all'))).not.toContain('credential-secret'); + }); + + it('persists server, target, and source summaries without depending on current instance records', async () => { + const api = await operationHistory(); + const { withOperationSource } = await import('../../src/lib/server/hosts/operation-context.js'); + api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('history-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)`) + .run(api.first.id, new Date().toISOString()); + const read = withOperationSource('Routine status', () => api.queueHostOperation(api.first.id, 'files.read', { + path: 'configs/server.cfg', credential: 'payload-secret' + }, { instanceId: 'history-instance', generation: 1 })); + const manifest = withOperationSource('Watchdog drift scan', () => api.queueHostOperation(api.first.id, 'files.manifest', { + roots: ['scripts', 'configs', 'plugins'], ignoreNames: [], paths: ['configs/server.cfg'] + }, { instanceId: 'history-instance', generation: 1 })); + api.reportHostOperationStatus(api.first.id, read.id, { state: 'completed', result: { contentBase64: 'result-secret' } }); + api.handle.prepare('DELETE FROM native_instances WHERE id=?').run('history-instance'); + const history = api.listHostOperationHistory(1, 'all'); + expect(history.operations.find(operation => operation.id === read.id)).toMatchObject({ + serverName: 't5_zombie_theater_1', targetPath: 'configs/server.cfg', source: 'Routine status', routine: true + }); + expect(history.operations.find(operation => operation.id === manifest.id)).toMatchObject({ + serverName: 't5_zombie_theater_1', targetPath: 'scripts, configs, plugins', source: 'Watchdog drift scan', routine: true + }); + expect(JSON.stringify(history)).not.toContain('secret'); + }); + + it('refuses unsafe target summaries and never derives labels from arbitrary payload fields', async () => { + const api = await operationHistory(); + const unsafe = ['/etc/private-secret', '../private-secret', 'configs/../../private-secret', 'configs/line\nprivate-secret', 'configs\\private-secret']; + const ids = unsafe.map(path => api.add('files.read', { payload: { path, command: 'command-secret', name: 'name-secret' } })); + ids.push(api.add('files.manifest', { payload: { roots: ['scripts', '../private-secret'] } })); + ids.push(api.add('instance.restart', { payload: { path: 'configs/private-secret', command: 'command-secret' } })); + const history = api.listHostOperationHistory(1, 'all'); + expect(history.operations).toHaveLength(ids.length); + expect(history.operations.every(operation => operation.targetPath === null && operation.serverName === null)).toBe(true); + expect(JSON.stringify(history)).not.toContain('secret'); + }); + + it('backfills safe legacy summaries from immutable payloads while leaving source unknown', async () => { + const api = await operationHistory(); + const { closeDb } = await import('../../src/lib/server/db/index.js'); + api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('legacy-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)`) + .run(api.first.id, new Date().toISOString()); + const read = api.queueHostOperationSerialized(api.first.id, 'files.read', Buffer.from(JSON.stringify({ + path: 'configs/server.cfg', credential: 'payload-secret' + })), { instanceId: 'legacy-instance', generation: 1 }, { path: 'configs/routing-secret' }); + api.reportHostOperationStatus(api.first.id, read.id, { state: 'completed', result: { credential: 'result-secret' } }); + const command = api.queueHostOperationSerialized(api.first.id, 'rcon.execute', Buffer.from('{"command":"status"}'), {}, { command: 'command-secret' }); + const manual = api.add('rcon.execute', { payload: { command: 'say command-secret' } }); + const unsafe = api.add('files.read', { payload: { path: '../private-secret' } }); + api.handle.exec(`DROP INDEX ec_host_operations_routine_history; + DROP INDEX ec_host_operations_important_history; + ALTER TABLE ec_host_operations DROP COLUMN routine; + ALTER TABLE ec_host_operations DROP COLUMN server_name; + ALTER TABLE ec_host_operations DROP COLUMN target_path; + ALTER TABLE ec_host_operations DROP COLUMN source;`); + closeDb(); + const history = api.listHostOperationHistory(1, 'all'); + expect(history.operations).toHaveLength(4); + expect(history.operations.find(operation => operation.id === read.id)).toMatchObject({ + routine: true, serverName: 't5_zombie_theater_1', targetPath: 'configs/server.cfg', source: null + }); + expect(history.operations.find(operation => operation.id === command.id)).toMatchObject({ routine: true, targetPath: 'status', source: null }); + expect(history.operations.find(operation => operation.id === manual)).toMatchObject({ routine: false, targetPath: null, source: null }); + expect(history.operations.find(operation => operation.id === unsafe)).toMatchObject({ routine: true, targetPath: null, source: null }); + expect(JSON.stringify(history)).not.toContain('secret'); + closeDb(); + expect(api.listHostOperationHistory(1, 'all')).toEqual(history); + }); + it('serves fixed-size API pages without caching or exposing private operation data', async () => { const api = await seedHistory(); const { GET } = await import('../../src/routes/api/hosts/operations/+server.js'); - const first = GET({ request: new Request('https://ec.test/api/hosts/operations') }); + const first = GET({ request: new Request('https://ec.test/api/hosts/operations?filter=all') }); expect(first.headers.get('cache-control')).toBe('no-store'); - expect(await first.json()).toEqual(api.listHostOperationHistory(1)); - const second = GET({ request: new Request('https://ec.test/api/hosts/operations?page=2&pageSize=500') }); + expect(await first.json()).toEqual(api.listHostOperationHistory(1, 'all')); + const second = GET({ request: new Request('https://ec.test/api/hosts/operations?page=2&pageSize=500&filter=all') }); const body = await second.json(); - expect(body).toEqual(api.listHostOperationHistory(2)); + expect(body).toEqual(api.listHostOperationHistory(2, 'all')); expect(body.operations).toHaveLength(25); expect(JSON.stringify(body)).not.toContain('secret'); - const last = GET({ request: new Request('https://ec.test/api/hosts/operations?page=999') }); - expect(await last.json()).toEqual(api.listHostOperationHistory(3)); + const last = GET({ request: new Request('https://ec.test/api/hosts/operations?page=999&filter=all') }); + expect(await last.json()).toEqual(api.listHostOperationHistory(3, 'all')); + }); + + it('serves the selected filter and defaults the API to actions and errors', async () => { + const api = await operationHistory(); + const action = api.add('instance.restart'); + const read = api.add('instance.inspect', { result: { healthy: true } }); + const { GET } = await import('../../src/routes/api/hosts/operations/+server.js'); + const request = (query = '') => GET({ request: new Request(`https://ec.test/api/hosts/operations${query}`) }).json(); + expect(await request()).toEqual(api.listHostOperationHistory()); + expect((await request()).operations.map((operation: { id: string }) => operation.id)).toEqual([action]); + expect((await request('?filter=routine')).operations.map((operation: { id: string }) => operation.id)).toEqual([read]); + expect(await request('?filter=all')).toEqual(api.listHostOperationHistory(1, 'all')); }); it('rejects invalid API pages and keeps host management access restrictions', async () => { @@ -95,6 +250,9 @@ describe('host operation history', () => { for (const page of ['', '0', '-1', '1.5', 'nope', 'Infinity', '9007199254740992']) { expect(() => GET({ request: new Request(`${url}?page=${page}`) })).toThrow(expect.objectContaining({ status: 400 })); } + for (const filter of ['', 'unknown', 'ALL', 'important%27%20OR%201=1']) { + expect(() => GET({ request: new Request(`${url}?filter=${filter}`) })).toThrow(expect.objectContaining({ status: 400 })); + } const blockedHeaders: HeadersInit[] = [ { origin: 'https://other.test' }, { 'sec-fetch-site': 'cross-site' }, diff --git a/tests/unit/operation-context.test.ts b/tests/unit/operation-context.test.ts new file mode 100644 index 0000000..4f18afa --- /dev/null +++ b/tests/unit/operation-context.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest'; +import { currentOperationSource, withOperationSource } from '../../src/lib/server/hosts/operation-context.js'; +import { trackActivity, workerActivity } from '../../src/lib/server/pipeline/activity.js'; + +describe('host operation source context', () => { + it('isolates overlapping asynchronous sources and restores the caller', async () => { + let release!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const first = withOperationSource('GET /servers/[name]', async () => { + await barrier; + return currentOperationSource(); + }); + const second = withOperationSource('Status monitor', async () => { + release(); + await Promise.resolve(); + return currentOperationSource(); + }); + expect(currentOperationSource()).toBeNull(); + expect(await Promise.all([first, second])).toEqual(['GET /servers/[name]', 'Status monitor']); + expect(currentOperationSource()).toBeNull(); + }); + + it('lets explicit background work override incidental request context without leaking it', async () => { + await withOperationSource('POST /api/servers/[name]/refresh', async () => { + await withOperationSource('Status monitor', async () => { + await Promise.resolve(); + expect(currentOperationSource()).toBe('Status monitor'); + }); + expect(currentOperationSource()).toBe('POST /api/servers/[name]/refresh'); + expect(() => withOperationSource('Failing task', () => { throw new Error('failed'); })).toThrow('failed'); + expect(currentOperationSource()).toBe('POST /api/servers/[name]/refresh'); + }); + expect(currentOperationSource()).toBeNull(); + }); + + it('gives background activity a source while preserving its initiating source through nested activity', async () => { + await trackActivity('context-test', 'Reconciling campaigns', async () => { + await Promise.resolve(); + expect(currentOperationSource()).toBe('Reconciling campaigns'); + await trackActivity('context-test-child', 'Waiting for files.read', async () => { + expect(currentOperationSource()).toBe('Reconciling campaigns'); + }); + }); + await withOperationSource('POST /api/servers/[name]/restart', () => + trackActivity('context-test-request', 'Processing provisioning requests', async () => { + expect(currentOperationSource()).toBe('POST /api/servers/[name]/restart'); + })); + expect(currentOperationSource()).toBeNull(); + expect(workerActivity()).toEqual([]); + }); + + it('restores context and clears activity when asynchronous work rejects', async () => { + await withOperationSource('Manual maintenance check', async () => { + await expect(trackActivity('context-test-error', 'Waiting for files.read', async () => { + await Promise.resolve(); + expect(currentOperationSource()).toBe('Manual maintenance check'); + throw new Error('unavailable'); + })).rejects.toThrow('unavailable'); + expect(currentOperationSource()).toBe('Manual maintenance check'); + }); + expect(currentOperationSource()).toBeNull(); + expect(workerActivity()).toEqual([]); + }); +});
OperationHostStateUpdatedResult
OperationHostTargetSourceStateUpdatedResult
{operation.kind}{operation.id}{operation.kind}{operation.id} {#if operation.hostName !== null}{operation.hostName}{:else}Deleted host{operation.hostId}{/if} + {operation.serverName ?? operation.instanceId ?? 'Host-level'} + {#if operation.targetPath}{operation.targetPath}{/if} + {operation.source ?? 'Not recorded'} {operation.state} {$timeFormat.dateTime(operation.updatedAt)} {operation.error ?? '—'}