Read chat logs incrementally and drain retention (Refs #62)

src/lib/server/retention.ts

@@ -6,6 +6,7 @@ import { runtimeLaunchSql } from './fleet/runtime-config.js';
 import { ensureHostSchema } from './hosts/schema.js';
 import { ensureGameFileSchema } from './game-files/schema.js';
 import { ensureAuthSchema } from './auth/schema.js';
+import { logResultMetadata } from './hosts/log-results.js';
 import type { FleetSettings } from '../fleet-settings.js';
 
 interface Policy {
@@ -298,11 +299,45 @@ export function sampleRetentionPreview(settings = fleetSettings()) {
 }
 export function pruneSampleHistory() {
     initialize();
-    for (const p of samplePolicies().filter(p => p.enabled)) db().prepare(`DELETE FROM ${p.table} WHERE rowid IN (SELECT rowid FROM ${p.table} WHERE ${p.condition} LIMIT 5000)`).run(p.value);
+    let removed = 0;
+    for (const p of samplePolicies().filter(p => p.enabled)) removed += db().prepare(`DELETE FROM ${p.table} WHERE rowid IN (SELECT rowid FROM ${p.table} WHERE ${p.condition} LIMIT 5000)`).run(p.value).changes;
     const s = fleetSettings();
-    if (s.telemetryMaxPerServer) db().prepare(`DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM
-        (SELECT id,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS rank FROM telemetry_requests) WHERE rank>? LIMIT 5000)`).run(s.telemetryMaxPerServer);
-    if (s.telemetryMaxTotal) db().prepare('DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 5000 OFFSET ?)').run(s.telemetryMaxTotal);
+    if (s.telemetryMaxPerServer) removed += db().prepare(`DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM
+        (SELECT id,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS rank FROM telemetry_requests) WHERE rank>? LIMIT 5000)`).run(s.telemetryMaxPerServer).changes;
+    if (s.telemetryMaxTotal) removed += db().prepare('DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 5000 OFFSET ?)').run(s.telemetryMaxTotal).changes;
+    return removed;
+}
+
+/** Drain every eligible batch, yielding between transactions so reads and receipts can run. */
+export async function drainRetention(): Promise<number> {
+    let total = 0, logCursor = 0;
+    initialize();
+    while (true) {
+        const removed = pruneHistory() + pruneSampleHistory();
+        total += removed;
+        if (!removed) break;
+        await new Promise<void>(resolve => setImmediate(resolve));
+    }
+    // Remove expired rows first; only compact bodies in history that will survive.
+    if (db().prepare("SELECT 1 FROM settings WHERE key='log_history_metadata_v1'").get()) return total;
+    while (true) {
+        const rows = db().prepare(`SELECT rowid AS rid,id,result FROM ec_host_operations
+            WHERE rowid>? AND kind='logs.read' AND json_valid(result)
+                AND (json_type(result,'$.lines')='array' OR json_type(result,'$.contentBase64')='text'
+                    OR json_type(result,'$.contentSha256')='text')
+            ORDER BY rowid LIMIT 50`).all(logCursor) as { rid: number; id: string; result: string }[];
+        db().transaction(() => {
+            for (const row of rows) {
+                db().prepare('UPDATE ec_host_operations SET result=? WHERE id=?')
+                    .run(JSON.stringify(logResultMetadata(JSON.parse(row.result))), row.id);
+                db().prepare("DELETE FROM file_blob_references WHERE owner_type='operation-result' AND owner_id=?").run(row.id);
+            }
+            if (!rows.length) db().prepare("INSERT OR IGNORE INTO settings(key,value) VALUES('log_history_metadata_v1','1')").run();
+        })();
+        if (!rows.length) return total;
+        logCursor = rows.at(-1)!.rid;
+        await new Promise<void>(resolve => setImmediate(resolve));
+    }
 }
 
 function pruneRoutineObservations(): number {