Improve application responsiveness and background processing

src/lib/server/hosts/index.ts

@@ -1,4 +1,5 @@
 import { recordAgentReceipt } from '../provisioning/agent-events.js';
+import { EventEmitter } from 'node:events';
 import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
 import { isIP } from 'node:net';
 import { db } from '../db/index.js';
@@ -8,6 +9,16 @@ import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, Hos
 export const AGENT_PROTOCOL_VERSION = 1;
 export const HEARTBEAT_MAX_AGE_MS = 90_000;
 const now = () => new Date().toISOString();
+const workEvents = new EventEmitter();
+workEvents.setMaxListeners(0);
+export function waitForHostWork(hostId: string, signal: AbortSignal, timeoutMs = 20_000): Promise<void> {
+    return new Promise(resolve => {
+        const done = () => { clearTimeout(timer); workEvents.off(hostId, done); signal.removeEventListener('abort', done); resolve(); };
+        const timer = setTimeout(done, timeoutMs);
+        workEvents.on(hostId, done); signal.addEventListener('abort', done, { once: true });
+        if (signal.aborted) done();
+    });
+}
 const digest = (token: string) => createHash('sha256').update(token).digest('hex');
 function database() { const handle = db(); ensureHostSchema(handle); return handle; }
 type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
@@ -113,6 +124,7 @@ export function deleteHost(id: string): { deleted: boolean; operationId?: string
             ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
             .run(id, operationId, digest(token), timestamp, timestamp);
         handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
+        queueMicrotask(() => workEvents.emit(id));
         return { deleted: false, operationId };
     }).immediate();
 }
@@ -287,7 +299,7 @@ export function requireReadyHost(hostId: string, options: { libraryId?: string;
 type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string };
 function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
 export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; }
-export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
+export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] { return (database().prepare(`SELECT ${summary ? `id,host_id,kind,instance_id,generation,state,error,created_at,updated_at,'{}' AS payload,NULL AS result` : '*'} FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?`).all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
 /** Heartbeats can tell an executing create to stop at its next safe command boundary. */
 export function pendingCreationCancellations(hostId: string): string[] {
     return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
@@ -300,7 +312,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
     if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
     if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
     if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
-    return database().transaction(() => {
+    const operation = database().transaction(() => {
         if (options.idempotencyKey) {
             const existing = database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? AND idempotency_key=?').get(hostId, options.idempotencyKey) as OperationRow | undefined;
             if (existing) {
@@ -315,8 +327,10 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
         database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
         return getHostOperation(id)!;
     })();
+    queueMicrotask(() => workEvents.emit(hostId));
+    return operation;
 }
-export function pendingHostOperations(hostId: string): HostOperation[] {
+export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'write' = 'all'): HostOperation[] {
     const host = requireHost(hostId);
     if (host.mode === 'disabled' || !host.connected) return [];
     // Older agents hold activity.lock across the entire delivered batch, including
@@ -328,8 +342,12 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
         WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
     // Accepted/running/unknown commands keep the same immutable ID. The agent
     // consults its durable journal before acknowledging or executing a replay.
+    const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
+        OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
+    const laneFilter = lane === 'all' ? '' : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}`;
     return database().transaction(() => {
         const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
+            ${laneFilter}
             AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
             AND (kind!='instance.cancel-create' OR ?)
             AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
@@ -342,7 +360,7 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
     })();
 }
 export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
-    return database().transaction(() => {
+    const operation = database().transaction(() => {
         requireHost(hostId);
         const operation = getHostOperation(id);
         if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
@@ -367,6 +385,8 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
         }
         return getHostOperation(id)!;
     })();
+    queueMicrotask(() => workEvents.emit(`operation:${id}`));
+    return operation;
 }
 export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
     const deadline = Date.now() + timeoutMs;
@@ -375,7 +395,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise
         if (!operation) throw new Error('Host operation was not found.');
         if (['completed','failed','unknown'].includes(operation.state)) return operation;
         if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`);
-        await new Promise(resolve => setTimeout(resolve, 200));
+        await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now())));
     }
 }