josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add filters and context to host operation history

Browse Source
This commit is contained in:
josh
2026-10-05 13:23:11 -04:00
parent e6912ceb57
commit a6c40cde9b
16 changed files with 417 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/index.ts
+24 -8
View File
@@ -4,11 +4,12 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt
import { isIP } from 'node:net';
import { db } from '../db/index.js';
import { ensureHostSchema } from './schema.js';
import { successfulObservation } from './observations.js';
import { successfulObservation, routineObservationSql, importantOperationSql, operationTargetSql, safeOperationTarget } from './observations.js';
import { currentOperationSource } from './operation-context.js';
import { fleetSettings } from '../settings.js';
import { recordInstanceResources, validateInstanceResources } from './calibration.js';
import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationHistoryPage, HostOperationState } from '../../hosts.js';
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationFilter, HostOperationHistoryPage, HostOperationState } from '../../hosts.js';
export const AGENT_PROTOCOL_VERSION = 1;
export const HEARTBEAT_MAX_AGE_MS = 90_000;
@@ -133,7 +134,8 @@ export function deleteHost(id: string): { deleted: boolean; operationId?: string
const operationId = randomUUID();
const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` };
handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp);
handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,source) VALUES(?,?,'host.decommission',?,'pending',?,?,?)")
.run(operationId, id, JSON.stringify(payload), timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200));
handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp);
handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
@@ -359,19 +361,26 @@ export function listHostOperations(hostId: string, limit = 100, summary = false)
.all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow);
}
/** Browse retained history without loading command payloads or credential-bearing results. */
export function listHostOperationHistory(page = 1): HostOperationHistoryPage {
export function listHostOperationHistory(page = 1, filter: HostOperationFilter = 'important'): HostOperationHistoryPage {
const handle = database();
const pageSize = 25;
const where = filter === 'all' ? '' : `WHERE ${filter === 'routine' ? 'o.routine=1' : importantOperationSql}`;
return handle.transaction((): HostOperationHistoryPage => {
const { total } = handle.prepare('SELECT COUNT(*) AS total FROM ec_host_operations').get() as { total: number };
const { total } = handle.prepare(`SELECT COUNT(*) AS total FROM ec_host_operations o ${where}`).get() as { total: number };
const totalPages = Math.max(1, Math.ceil(total / pageSize));
const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1);
const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind,
o.instance_id AS instanceId,o.generation,o.state,o.error,o.created_at AS createdAt,o.updated_at AS updatedAt
o.instance_id AS instanceId,o.generation,o.state,
CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0
THEN COALESCE(NULLIF(o.error,''),'Observation reported a problem') ELSE o.error END AS error,
o.created_at AS createdAt,o.updated_at AS updatedAt,o.routine,o.server_name AS serverName,o.target_path AS targetPath,o.source
FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL
${where}
ORDER BY o.created_at DESC,o.rowid DESC LIMIT ? OFFSET ?`)
.all(pageSize, (currentPage - 1) * pageSize) as HostOperationHistoryPage['operations'];
return { operations, page: currentPage, pageSize, total, totalPages };
return { operations: operations.map(operation => ({ ...operation, routine: !!operation.routine,
targetPath: safeOperationTarget(operation.kind, operation.targetPath) })), filter,
page: currentPage, pageSize, total, totalPages };
})();
}
@@ -428,8 +437,15 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
const id = options.id ?? randomUUID(); const timestamp = now();
database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, routingPayload ?? payload, timestamp, timestamp);
const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?')
.get(options.instanceId) as { server: string } | undefined)?.server ?? null : null;
database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name)
VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?)`)
.run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null,
routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName);
if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload);
database().prepare(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
target_path=${operationTargetSql} WHERE o.id=?`).run(id);
database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
return getHostOperationStatus(id)!;
})();
src/lib/server/hosts/observations.ts
+23 -3
View File
@@ -1,6 +1,13 @@
export const observationKinds = ['host.inspect', 'instance.inspect', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read'];
const commands = ['status', 'getstatus', 'version'];
export function safeOperationTarget(kind: string, value: string | null): string | null {
if (!value || value.length > 1024) return null;
const paths = kind === 'files.manifest' ? value.split(', ') : [value];
return paths.every(path => path === '.' || !/[\\\x00-\x1f\x7f]/.test(path)
&& path.split('/').every(part => part !== '' && part !== '.' && part !== '..')) ? value : null;
}
export function successfulObservation(kind: string, payload: Record<string, unknown>, result: unknown): boolean {
const routine = observationKinds.includes(kind) || kind === 'rcon.execute' && commands.includes(String(payload.command));
const value = result && typeof result === 'object' ? result as Record<string, unknown> : {};
@@ -8,11 +15,24 @@ export function successfulObservation(kind: string, payload: Record<string, unkn
}
/** Uses the operation alias o; serialized payloads may live outside the routing row. */
export const routineObservationSql = `(o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')})
OR (o.kind='rcon.execute' AND json_extract(COALESCE(
(SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')))`;
export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')}) THEN 1
WHEN o.kind='rcon.execute' THEN json_extract(COALESCE(
(SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')
ELSE 0 END)`;
export const successfulObservationSql = `(o.state='completed' AND COALESCE(o.error,'')='' AND ${routineObservationSql}
AND CASE WHEN o.result IS NULL THEN 1 WHEN json_valid(o.result)
THEN json_extract(o.result,'$.ok') IS NOT 0 AND json_extract(o.result,'$.healthy') IS NOT 0
AND COALESCE(json_extract(o.result,'$.error'),'')='' ELSE 0 END)`;
export const importantOperationSql = `(o.routine=0 OR o.state IN ('failed','unknown')
OR (o.state='completed' AND o.observation_success=0) OR COALESCE(o.error,'')!='')`;
const payloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
/** Deliberately exclude arbitrary RCON commands and all file contents/credentials. */
export const operationTargetSql = `CASE
WHEN o.kind IN ('files.read','files.stat','files.list','logs.read') THEN
CASE WHEN json_type(${payloadSql},'$.path')='text' THEN substr(json_extract(${payloadSql},'$.path'),1,1024) END
WHEN o.kind='files.manifest' THEN substr((SELECT group_concat(value,', ') FROM json_each(${payloadSql},'$.roots') WHERE type='text'),1,1024)
WHEN o.kind='rcon.execute' AND ${routineObservationSql} THEN json_extract(${payloadSql},'$.command')
ELSE NULL END`;
src/lib/server/hosts/operation-context.ts
+12
View File
@@ -0,0 +1,12 @@
import { AsyncLocalStorage } from 'node:async_hooks';
const operationSource = new AsyncLocalStorage<string>();
/** Carry the initiating request or background task through asynchronous work. */
export function withOperationSource<T>(source: string, work: () => T): T {
return operationSource.run(source, work);
}
export function currentOperationSource(): string | null {
return operationSource.getStore() ?? null;
}
src/lib/server/hosts/schema.ts
+17 -1
View File
@@ -1,5 +1,5 @@
import type Database from 'better-sqlite3';
import { successfulObservationSql } from './observations.js';
import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
const initialized = new WeakSet<Database.Database>();
export function ensureHostSchema(handle: Database.Database): void {
@@ -30,6 +30,7 @@ export function ensureHostSchema(handle: Database.Database): void {
CHECK (state IN ('pending','accepted','running','completed','failed','unknown')),
result TEXT, error TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, last_delivered_at TEXT,
observation_success INTEGER NOT NULL DEFAULT 0,
routine INTEGER NOT NULL DEFAULT 0, server_name TEXT, target_path TEXT, source TEXT,
UNIQUE(host_id, idempotency_key)
);
CREATE INDEX IF NOT EXISTS ec_host_operations_queue ON ec_host_operations(host_id, state, created_at);
@@ -93,6 +94,21 @@ export function ensureHostSchema(handle: Database.Database): void {
}
handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_observation_retention ON ec_host_operations(updated_at)
WHERE observation_success=1 AND state='completed'`);
if (!columns.some(column => column.name === 'routine')) {
handle.transaction(() => {
handle.exec(`ALTER TABLE ec_host_operations ADD COLUMN routine INTEGER NOT NULL DEFAULT 0;
ALTER TABLE ec_host_operations ADD COLUMN server_name TEXT;
ALTER TABLE ec_host_operations ADD COLUMN target_path TEXT;
ALTER TABLE ec_host_operations ADD COLUMN source TEXT;`);
const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as { name: string }[];
const nativeInstances = ['id', 'server'].every(name => nativeColumns.some(column => column.name === name));
handle.exec(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
target_path=${operationTargetSql}, server_name=${nativeInstances ? '(SELECT n.server FROM native_instances n WHERE n.id=o.instance_id)' : 'NULL'}`);
})();
}
handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_routine_history ON ec_host_operations(routine,created_at);
CREATE INDEX IF NOT EXISTS ec_host_operations_important_history ON ec_host_operations(created_at)
WHERE ${importantOperationSql.replaceAll('o.', '')}`);
const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[];