Fix retention query performance and cleanup scheduling

src/lib/server/db/index.ts

@@ -353,6 +353,8 @@ export function db(): Database.Database {
         migrateTable(handle, 'beta_test_runs', [{ name: 'revision', ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1' }]);
         migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS);
         migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS);
+        // Older databases gain rolled_back_by above, after schema.sql runs.
+        handle.exec('CREATE INDEX IF NOT EXISTS idx_deployments_rolled_back_by ON deployments(rolled_back_by)');
         migrateTable(handle, 'server_launch_collections', [
             { name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
             { name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" }

src/lib/server/db/schema.sql

@@ -522,6 +522,7 @@ CREATE TABLE IF NOT EXISTS provisioning_sources (
     server TEXT NOT NULL,
     PRIMARY KEY (operation_id, kind, source_id, server)
 );
+CREATE INDEX IF NOT EXISTS idx_provisioning_sources_source ON provisioning_sources(kind, source_id);
 
 -- Blueprints retain map/count selections only. The legacy column name remains
 -- compatible with older snapshot-based definitions, read as selections at runtime.
@@ -732,6 +733,7 @@ CREATE TABLE IF NOT EXISTS watchdog_actions (
     cycle INTEGER NOT NULL DEFAULT 1,
     UNIQUE(incident_id,cycle,action)
 );
+CREATE INDEX IF NOT EXISTS idx_watchdog_actions_campaign ON watchdog_actions(campaign_id);
 
 -- Explicit history holds and failure resolution never bypass dependency checks.
 CREATE TABLE IF NOT EXISTS retention_controls (

src/lib/server/pipeline/sweep.ts

@@ -1,8 +1,6 @@
-import { pruneHistory } from '../retention.js';
 import { db } from '../db/index.js';
 import { emit } from './bus.js';
 import { requestRestart } from '../provisioning/index.js';
-import { pruneHostObservations } from '../hosts/index.js';
 
 interface PendingRow {
     deployment_id: string; server: string; restart_state: string;
@@ -20,7 +18,6 @@ export function sweep(): Promise<SweepResult> {
     return sweepInFlight;
 }
 async function runSweep(): Promise<SweepResult> {
-    pruneHostObservations();
     const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
         WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed')
         AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=ds.deployment_id AND c.resolved_at IS NOT NULL) ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
@@ -47,7 +44,6 @@ async function runSweep(): Promise<SweepResult> {
     }
     db().prepare(`UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS (
         SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`).run(nowIso());
-    pruneHistory('campaign');
     emit({ type: 'sweep-tick', frozen: false });
     return result;
 }

src/lib/server/pipeline/worker.ts

@@ -21,6 +21,7 @@ import { trackActivity } from './activity.js';
 import { withOperationSource } from '../hosts/operation-context.js';
 
 const TICK_MS = 15_000;
+const RETENTION_INTERVAL_MS = 5 * 60_000;
 // Match each agent's two health lanes without making unrelated hosts wait.
 const STATUS_CONCURRENCY_PER_HOST = 2;
 
@@ -31,6 +32,7 @@ let statusTimer: NodeJS.Timeout | null = null;
 let tickInFlight: Promise<void> | null = null;
 let tickRequested = false;
 const requesterFlights = new Map<string, Promise<void>>();
+let nextRetentionAt = 0;
 
 function tick(requestAnother = false): Promise<void> {
     if (requestAnother) tickRequested = true;
@@ -57,6 +59,10 @@ function reconcileRequesters(): void {
     ];
     for (const [key, label, work] of requesters) {
         if (requesterFlights.has(key)) continue;
+        if (key === 'retention') {
+            if (Date.now() < nextRetentionAt) continue;
+            nextRetentionAt = Date.now() + RETENTION_INTERVAL_MS;
+        }
         const flight = trackActivity(key, label, work)
             .catch(err => { console.error(`[worker] ${key} error:`, err); })
             .then(async () => { if (key !== 'game-files' && key !== 'retention') await tick(true); })

src/lib/server/retention.ts

@@ -108,16 +108,24 @@ function selection(policy: Policy, settings = fleetSettings()) {
     const days = settings[policy.days];
     const failureDays = settings.failureKeepDays;
     const cut = (n: number) => new Date(Date.now() - n * 86400_000).toISOString();
-    const age = ['observation','operation'].includes(policy.category) ? `o.${policy.clock} < @cutoff` : policy.category === 'audit' ? "o.created_at < @numericCutoff" : `julianday(o.${policy.clock}) < julianday(@cutoff)`;
+    const age = ['campaign','observation','operation'].includes(policy.category) ? `o.${policy.clock} < @cutoff` : policy.category === 'audit' ? "o.created_at < @numericCutoff" : `julianday(o.${policy.clock}) < julianday(@cutoff)`;
     const failed = policy.failed ?? '0';
     const enabled = days > 0 || policy.category === 'version' && settings.versionKeepCount > 0;
-    const normal = `(${enabled ? 1 : 0} AND NOT (${failed}) AND ${policy.category === 'version' && !days ? '1' : age})`;
-    const failure = `(${failureDays > 0 && !!policy.failed ? 1 : 0} AND (${failed}) AND EXISTS (SELECT 1 FROM retention_controls c WHERE c.category=@category AND c.record_id=CAST(o.${policy.id} AS TEXT) AND julianday(c.resolved_at)>=julianday(o.${policy.clock}) AND julianday(c.resolved_at)<julianday(@failureCutoff)))`;
+    // Keep age outside the correlated failure checks so SQLite can seek the
+    // clock index before examining dependencies. Resolved failures start from
+    // their controls, not another scan of the entire history table.
+    const candidates: string[] = [];
+    if (enabled) candidates.push(`SELECT o.rowid FROM ${policy.table} o
+        WHERE ${policy.category === 'version' && !days ? '1' : age} AND NOT (${failed})`);
+    if (failureDays > 0 && policy.failed) candidates.push(`SELECT o.rowid FROM retention_controls c
+        JOIN ${policy.table} o ON o.${policy.id}=c.record_id
+        WHERE c.category=@category AND julianday(c.resolved_at)<julianday(@failureCutoff)
+        AND julianday(c.resolved_at)>=julianday(o.${policy.clock}) AND (${failed})`);
     const minimum = policy.minimum === false ? 0 : settings.historyKeepCount;
     const minimumScope = policy.category === 'observation' ? "WHERE observation_success=1 AND state='completed'"
         : policy.category === 'operation' ? "WHERE history_expired_at IS NULL AND (routine=0 OR observation_success=0)" : '';
     const versionCount = policy.category === 'version' ? `AND o.id NOT IN (SELECT id FROM script_versions v WHERE v.script_id=o.script_id ORDER BY version DESC LIMIT ${settings.versionKeepCount})` : '';
-    return { sql: `FROM ${policy.table} o WHERE (${policy.eligible}) AND (${normal} OR ${failure})
+    return { sql: `FROM ${policy.table} o WHERE ${candidates.length ? `o.rowid IN (${candidates.join(' UNION ALL ')})` : '0'} AND (${policy.eligible})
         AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category=@category AND c.record_id=CAST(o.${policy.id} AS TEXT) AND c.protected=1)
         AND o.rowid NOT IN (SELECT rowid FROM ${policy.table} ${minimumScope} ORDER BY ${policy.clock} DESC,rowid DESC LIMIT ${minimum}) ${versionCount}`,
         params: { category: policy.category, cutoff: cut(days), numericCutoff: Date.now() - days * 86400_000, failureCutoff: cut(failureDays) } };