Read chat logs incrementally and drain retention (Refs #62)

src/lib/server/hosts/index.ts

@@ -8,6 +8,7 @@ import { db } from '../db/index.js';
 import { ensureHostSchema } from './schema.js';
 import { successfulObservation, routineObservationSql, importantOperationSql, operationTargetSql, safeOperationTarget } from './observations.js';
 import { currentOperationSource } from './operation-context.js';
+import { logResultMetadata } from './log-results.js';
 import { parseHostInventory } from './request-context.js';
 import { fleetSettings } from '../settings.js';
 import { recordInstanceResources, validateInstanceResources } from './calibration.js';
@@ -19,6 +20,8 @@ export const AGENT_PROTOCOL_VERSION = 1;
 export const HEARTBEAT_MAX_AGE_MS = 90_000;
 const now = () => new Date().toISOString();
 const workEvents = new EventEmitter();
+// Hold reply bodies only while a reader is awaiting this operation. No TTL cache or disk copy.
+const logReaders = new Map<string, Set<{ received: boolean; result?: unknown }>>();
 workEvents.setMaxListeners(0);
 export function waitForHostWork(hostId: string, signal: AbortSignal, timeoutMs = 20_000): Promise<void> {
     return new Promise(resolve => {
@@ -599,6 +602,7 @@ export function getHostOperationPayload(hostId: string, id: string): Buffer {
     return Buffer.from(JSON.stringify(unpackFileJson(JSON.parse(row.payload.toString()))));
 }
 export function reportHostOperationStatus(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperationStatus {
+    let liveLogResult: { value: unknown } | undefined;
     const operation = database().transaction(() => {
         requireHost(hostId);
         const operation = getHostOperationStatus(id);
@@ -612,7 +616,7 @@ export function reportHostOperationStatus(hostId: string, id: string, input: { s
         let result = input.result === undefined ? null : JSON.stringify(input.result);
         if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
         if (result !== null) {
-            const packed = packFileJson(input.result);
+            const packed = packFileJson(operation.kind === 'logs.read' ? logResultMetadata(input.result) : input.result);
             result = JSON.stringify(packed.value);
             referenceBlobs('operation-result', id, packed.hashes);
         }
@@ -620,6 +624,7 @@ export function reportHostOperationStatus(hostId: string, id: string, input: { s
             operation.kind === 'rcon.execute' ? JSON.parse(getHostOperationPayload(hostId, id).toString('utf8')) : {}, input.result);
         database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=?,observation_success=? WHERE id=?')
             .run(state, result, error, now(), Number(observation), id);
+        if (operation.kind === 'logs.read' && state === 'completed') liveLogResult = { value: input.result ?? null };
         if (operation.kind === 'host.decommission') {
             if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id);
             else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id);
@@ -632,6 +637,10 @@ export function reportHostOperationStatus(hostId: string, id: string, input: { s
         }
         return getHostOperationStatus(id)!;
     })();
+    if (liveLogResult) for (const reader of logReaders.get(id) ?? []) {
+        reader.received = true;
+        reader.result = liveLogResult.value;
+    }
     queueMicrotask(() => workEvents.emit(`operation:${id}`));
     return operation;
 }
@@ -640,6 +649,21 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
     return getHostOperation(id)!;
 }
 export async function waitHostOperationStatus(id: string, timeoutMs = 30_000): Promise<HostOperationStatus> {
+    if (getHostOperationStatus(id)?.kind !== 'logs.read') return waitOperationStatus(id, timeoutMs);
+    const reader: { received: boolean; result?: unknown } = { received: false };
+    const readers = logReaders.get(id) ?? new Set();
+    readers.add(reader); logReaders.set(id, readers);
+    try {
+        const operation = await waitOperationStatus(id, timeoutMs);
+        if (operation.state !== 'completed') return operation;
+        if (!reader.received) throw new Error('Log reply is no longer available. Request a new read.');
+        return { ...operation, result: reader.result };
+    } finally {
+        readers.delete(reader);
+        if (!readers.size) logReaders.delete(id);
+    }
+}
+async function waitOperationStatus(id: string, timeoutMs: number): Promise<HostOperationStatus> {
     const deadline = Date.now() + timeoutMs;
     while (true) {
         const operation = getHostOperationStatus(id);
@@ -668,8 +692,9 @@ export async function waitHostOperationStatus(id: string, timeoutMs = 30_000): P
     }
 }
 export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
-    await waitHostOperationStatus(id, timeoutMs);
-    return getHostOperation(id)!;
+    const status = await waitHostOperationStatus(id, timeoutMs);
+    const operation = getHostOperation(id)!;
+    return status.kind === 'logs.read' ? { ...operation, result: status.result } : operation;
 }
 
 export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean {