Add filters and context to host operation history

src/lib/server/hosts/index.ts

@@ -4,11 +4,12 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt
 import { isIP } from 'node:net';
 import { db } from '../db/index.js';
 import { ensureHostSchema } from './schema.js';
-import { successfulObservation } from './observations.js';
+import { successfulObservation, routineObservationSql, importantOperationSql, operationTargetSql, safeOperationTarget } from './observations.js';
+import { currentOperationSource } from './operation-context.js';
 import { fleetSettings } from '../settings.js';
 import { recordInstanceResources, validateInstanceResources } from './calibration.js';
 import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
-import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationHistoryPage, HostOperationState } from '../../hosts.js';
+import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationFilter, HostOperationHistoryPage, HostOperationState } from '../../hosts.js';
 
 export const AGENT_PROTOCOL_VERSION = 1;
 export const HEARTBEAT_MAX_AGE_MS = 90_000;
@@ -133,7 +134,8 @@ export function deleteHost(id: string): { deleted: boolean; operationId?: string
         const operationId = randomUUID();
         const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
         const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` };
-        handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp);
+        handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,source) VALUES(?,?,'host.decommission',?,'pending',?,?,?)")
+            .run(operationId, id, JSON.stringify(payload), timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200));
         handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp);
         handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
             ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
@@ -359,19 +361,26 @@ export function listHostOperations(hostId: string, limit = 100, summary = false)
         .all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow);
 }
 /** Browse retained history without loading command payloads or credential-bearing results. */
-export function listHostOperationHistory(page = 1): HostOperationHistoryPage {
+export function listHostOperationHistory(page = 1, filter: HostOperationFilter = 'important'): HostOperationHistoryPage {
     const handle = database();
     const pageSize = 25;
+    const where = filter === 'all' ? '' : `WHERE ${filter === 'routine' ? 'o.routine=1' : importantOperationSql}`;
     return handle.transaction((): HostOperationHistoryPage => {
-        const { total } = handle.prepare('SELECT COUNT(*) AS total FROM ec_host_operations').get() as { total: number };
+        const { total } = handle.prepare(`SELECT COUNT(*) AS total FROM ec_host_operations o ${where}`).get() as { total: number };
         const totalPages = Math.max(1, Math.ceil(total / pageSize));
         const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1);
         const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind,
-            o.instance_id AS instanceId,o.generation,o.state,o.error,o.created_at AS createdAt,o.updated_at AS updatedAt
+            o.instance_id AS instanceId,o.generation,o.state,
+            CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0
+                THEN COALESCE(NULLIF(o.error,''),'Observation reported a problem') ELSE o.error END AS error,
+            o.created_at AS createdAt,o.updated_at AS updatedAt,o.routine,o.server_name AS serverName,o.target_path AS targetPath,o.source
             FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL
+            ${where}
             ORDER BY o.created_at DESC,o.rowid DESC LIMIT ? OFFSET ?`)
             .all(pageSize, (currentPage - 1) * pageSize) as HostOperationHistoryPage['operations'];
-        return { operations, page: currentPage, pageSize, total, totalPages };
+        return { operations: operations.map(operation => ({ ...operation, routine: !!operation.routine,
+            targetPath: safeOperationTarget(operation.kind, operation.targetPath) })), filter,
+            page: currentPage, pageSize, total, totalPages };
     })();
 }
 
@@ -428,8 +437,15 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
         if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
         if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
         const id = options.id ?? randomUUID(); const timestamp = now();
-        database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, routingPayload ?? payload, timestamp, timestamp);
+        const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?')
+            .get(options.instanceId) as { server: string } | undefined)?.server ?? null : null;
+        database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name)
+            VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?)`)
+            .run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null,
+                routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName);
         if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload);
+        database().prepare(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
+            target_path=${operationTargetSql} WHERE o.id=?`).run(id);
         database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
         return getHostOperationStatus(id)!;
     })();

src/lib/server/hosts/observations.ts

@@ -1,6 +1,13 @@
 export const observationKinds = ['host.inspect', 'instance.inspect', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read'];
 const commands = ['status', 'getstatus', 'version'];
 
+export function safeOperationTarget(kind: string, value: string | null): string | null {
+    if (!value || value.length > 1024) return null;
+    const paths = kind === 'files.manifest' ? value.split(', ') : [value];
+    return paths.every(path => path === '.' || !/[\\\x00-\x1f\x7f]/.test(path)
+        && path.split('/').every(part => part !== '' && part !== '.' && part !== '..')) ? value : null;
+}
+
 export function successfulObservation(kind: string, payload: Record<string, unknown>, result: unknown): boolean {
     const routine = observationKinds.includes(kind) || kind === 'rcon.execute' && commands.includes(String(payload.command));
     const value = result && typeof result === 'object' ? result as Record<string, unknown> : {};
@@ -8,11 +15,24 @@ export function successfulObservation(kind: string, payload: Record<string, unkn
 }
 
 /** Uses the operation alias o; serialized payloads may live outside the routing row. */
-export const routineObservationSql = `(o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')})
-    OR (o.kind='rcon.execute' AND json_extract(COALESCE(
-        (SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')))`;
+export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')}) THEN 1
+    WHEN o.kind='rcon.execute' THEN json_extract(COALESCE(
+        (SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')
+    ELSE 0 END)`;
 
 export const successfulObservationSql = `(o.state='completed' AND COALESCE(o.error,'')='' AND ${routineObservationSql}
     AND CASE WHEN o.result IS NULL THEN 1 WHEN json_valid(o.result)
         THEN json_extract(o.result,'$.ok') IS NOT 0 AND json_extract(o.result,'$.healthy') IS NOT 0
             AND COALESCE(json_extract(o.result,'$.error'),'')='' ELSE 0 END)`;
+
+export const importantOperationSql = `(o.routine=0 OR o.state IN ('failed','unknown')
+    OR (o.state='completed' AND o.observation_success=0) OR COALESCE(o.error,'')!='')`;
+
+const payloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
+/** Deliberately exclude arbitrary RCON commands and all file contents/credentials. */
+export const operationTargetSql = `CASE
+    WHEN o.kind IN ('files.read','files.stat','files.list','logs.read') THEN
+        CASE WHEN json_type(${payloadSql},'$.path')='text' THEN substr(json_extract(${payloadSql},'$.path'),1,1024) END
+    WHEN o.kind='files.manifest' THEN substr((SELECT group_concat(value,', ') FROM json_each(${payloadSql},'$.roots') WHERE type='text'),1,1024)
+    WHEN o.kind='rcon.execute' AND ${routineObservationSql} THEN json_extract(${payloadSql},'$.command')
+    ELSE NULL END`;

src/lib/server/hosts/operation-context.ts

@@ -0,0 +1,12 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+const operationSource = new AsyncLocalStorage<string>();
+
+/** Carry the initiating request or background task through asynchronous work. */
+export function withOperationSource<T>(source: string, work: () => T): T {
+    return operationSource.run(source, work);
+}
+
+export function currentOperationSource(): string | null {
+    return operationSource.getStore() ?? null;
+}

src/lib/server/hosts/schema.ts

@@ -1,5 +1,5 @@
 import type Database from 'better-sqlite3';
-import { successfulObservationSql } from './observations.js';
+import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
 
 const initialized = new WeakSet<Database.Database>();
 export function ensureHostSchema(handle: Database.Database): void {
@@ -30,6 +30,7 @@ export function ensureHostSchema(handle: Database.Database): void {
                 CHECK (state IN ('pending','accepted','running','completed','failed','unknown')),
             result TEXT, error TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, last_delivered_at TEXT,
             observation_success INTEGER NOT NULL DEFAULT 0,
+            routine INTEGER NOT NULL DEFAULT 0, server_name TEXT, target_path TEXT, source TEXT,
             UNIQUE(host_id, idempotency_key)
         );
         CREATE INDEX IF NOT EXISTS ec_host_operations_queue ON ec_host_operations(host_id, state, created_at);
@@ -93,6 +94,21 @@ export function ensureHostSchema(handle: Database.Database): void {
     }
     handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_observation_retention ON ec_host_operations(updated_at)
         WHERE observation_success=1 AND state='completed'`);
+    if (!columns.some(column => column.name === 'routine')) {
+        handle.transaction(() => {
+            handle.exec(`ALTER TABLE ec_host_operations ADD COLUMN routine INTEGER NOT NULL DEFAULT 0;
+                ALTER TABLE ec_host_operations ADD COLUMN server_name TEXT;
+                ALTER TABLE ec_host_operations ADD COLUMN target_path TEXT;
+                ALTER TABLE ec_host_operations ADD COLUMN source TEXT;`);
+            const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as { name: string }[];
+            const nativeInstances = ['id', 'server'].every(name => nativeColumns.some(column => column.name === name));
+            handle.exec(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
+                target_path=${operationTargetSql}, server_name=${nativeInstances ? '(SELECT n.server FROM native_instances n WHERE n.id=o.instance_id)' : 'NULL'}`);
+        })();
+    }
+    handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_routine_history ON ec_host_operations(routine,created_at);
+        CREATE INDEX IF NOT EXISTS ec_host_operations_important_history ON ec_host_operations(created_at)
+            WHERE ${importantOperationSql.replaceAll('o.', '')}`);
     const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
     if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
     const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[];

src/lib/server/maintenance/recovery.ts

@@ -10,6 +10,7 @@ import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '
 import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
 import { deploymentStage, finishDeployment } from '../provisioning/history.js';
 import { recordProgress } from '../provisioning/progress.js';
+import { withOperationSource } from '../hosts/operation-context.js';
 import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js';
 
 import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
@@ -309,7 +310,8 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
 let inFlight: Promise<void> | null = null;
 export function watchdogTick(forceScan = false): Promise<void> {
     if (inFlight) return inFlight;
-    inFlight = run(forceScan).finally(() => { inFlight = null; });
+    inFlight = withOperationSource(forceScan ? 'Manual maintenance check' : 'Maintenance watchdog', () => run(forceScan))
+        .finally(() => { inFlight = null; });
     return inFlight;
 }
 async function run(forceScan: boolean): Promise<void> {

src/lib/server/pipeline/activity.ts

@@ -1,4 +1,5 @@
 import type { WorkerActivity } from '../../provisioning-detail.js';
+import { currentOperationSource, withOperationSource } from '../hosts/operation-context.js';
 
 const active = new Map<string, WorkerActivity>();
 let sequence = 0;
@@ -11,6 +12,6 @@ export function workerActivity(): WorkerActivity[] {
 export async function trackActivity<T>(key: string, label: string, work: () => Promise<T>): Promise<T> {
     if (active.has(key)) key = `${key}:${++sequence}`;
     active.set(key, { key, label, startedAt: new Date().toISOString() });
-    try { return await work(); }
+    try { return await withOperationSource(currentOperationSource() ?? label, work); }
     finally { active.delete(key); }
 }

src/lib/server/pipeline/worker.ts

@@ -17,6 +17,7 @@ import { selectedZombieMap } from '../../map-aliases.js';
 import { runMaintenance } from '../maintenance/index.js';
 import { cleanupControllerLibraries } from '../game-files/index.js';
 import { trackActivity } from './activity.js';
+import { withOperationSource } from '../hosts/operation-context.js';
 
 const TICK_MS = 15_000;
 // Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
@@ -104,7 +105,7 @@ function queueStatus(server: string): Promise<number | null> {
     statusQueue.push(async () => {
         let capacity: number | null = null;
         try {
-            capacity = await refreshServerStatus(server);
+            capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server));
             // Publish this server's latest evidence without waiting for the fleet.
             const current = getServer(server);
             if (current) publishStatus(statusEntry(current));