josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix retention query performance and cleanup scheduling

Browse Source
This commit is contained in:
josh
2026-10-06 10:39:32 -04:00
parent 9fd448339d
commit 88a950692a
8 changed files with 104 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/retention.ts
+12 -4
View File
@@ -108,16 +108,24 @@ function selection(policy: Policy, settings = fleetSettings()) {
const days = settings[policy.days];
const failureDays = settings.failureKeepDays;
const cut = (n: number) => new Date(Date.now() - n * 86400_000).toISOString();
const age = ['observation','operation'].includes(policy.category) ? `o.${policy.clock} < @cutoff` : policy.category === 'audit' ? "o.created_at < @numericCutoff" : `julianday(o.${policy.clock}) < julianday(@cutoff)`;
const age = ['campaign','observation','operation'].includes(policy.category) ? `o.${policy.clock} < @cutoff` : policy.category === 'audit' ? "o.created_at < @numericCutoff" : `julianday(o.${policy.clock}) < julianday(@cutoff)`;
const failed = policy.failed ?? '0';
const enabled = days > 0 || policy.category === 'version' && settings.versionKeepCount > 0;
const normal = `(${enabled ? 1 : 0} AND NOT (${failed}) AND ${policy.category === 'version' && !days ? '1' : age})`;
const failure = `(${failureDays > 0 && !!policy.failed ? 1 : 0} AND (${failed}) AND EXISTS (SELECT 1 FROM retention_controls c WHERE c.category=@category AND c.record_id=CAST(o.${policy.id} AS TEXT) AND julianday(c.resolved_at)>=julianday(o.${policy.clock}) AND julianday(c.resolved_at)<julianday(@failureCutoff)))`;
// Keep age outside the correlated failure checks so SQLite can seek the
// clock index before examining dependencies. Resolved failures start from
// their controls, not another scan of the entire history table.
const candidates: string[] = [];
if (enabled) candidates.push(`SELECT o.rowid FROM ${policy.table} o
WHERE ${policy.category === 'version' && !days ? '1' : age} AND NOT (${failed})`);
if (failureDays > 0 && policy.failed) candidates.push(`SELECT o.rowid FROM retention_controls c
JOIN ${policy.table} o ON o.${policy.id}=c.record_id
WHERE c.category=@category AND julianday(c.resolved_at)<julianday(@failureCutoff)
AND julianday(c.resolved_at)>=julianday(o.${policy.clock}) AND (${failed})`);
const minimum = policy.minimum === false ? 0 : settings.historyKeepCount;
const minimumScope = policy.category === 'observation' ? "WHERE observation_success=1 AND state='completed'"
: policy.category === 'operation' ? "WHERE history_expired_at IS NULL AND (routine=0 OR observation_success=0)" : '';
const versionCount = policy.category === 'version' ? `AND o.id NOT IN (SELECT id FROM script_versions v WHERE v.script_id=o.script_id ORDER BY version DESC LIMIT ${settings.versionKeepCount})` : '';
return { sql: `FROM ${policy.table} o WHERE (${policy.eligible}) AND (${normal} OR ${failure})
return { sql: `FROM ${policy.table} o WHERE ${candidates.length ? `o.rowid IN (${candidates.join(' UNION ALL ')})` : '0'} AND (${policy.eligible})
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category=@category AND c.record_id=CAST(o.${policy.id} AS TEXT) AND c.protected=1)
AND o.rowid NOT IN (SELECT rowid FROM ${policy.table} ${minimumScope} ORDER BY ${policy.clock} DESC,rowid DESC LIMIT ${minimum}) ${versionCount}`,
params: { category: policy.category, cutoff: cut(days), numericCutoff: Date.now() - days * 86400_000, failureCutoff: cut(failureDays) } };