josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Expire retained file bodies and reclaim SQLite storage

Browse Source
This commit is contained in:
josh
2026-10-06 14:43:37 -04:00
parent 62df14b180
commit 562b770725
9 changed files with 214 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/retention.ts
+53 -8
View File
@@ -1,4 +1,4 @@
import { blobSize, pruneFileBlobs } from './db/file-blobs.js';
import { blobSize, pruneFileBlobs, referenceBlobs } from './db/file-blobs.js';
import { createHash } from 'node:crypto';
import { db } from './db/index.js';
import { fleetSettings } from './settings.js';
@@ -78,11 +78,26 @@ function policies(): Policy[] {
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`,
bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
{ category: 'beta-file-content', label: 'Closed beta file snapshots', table: 'betas', id: 'id', clock: 'closed_at', days: 'fileContentKeepDays',
eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id)
eligible: `o.state IN ('promoted','rolled-back')
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1)
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT))
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` },
{ category: 'workflow-payload', label: 'Completed launch file bodies', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'payloadKeepDays',
failed: "o.state='failed'",
eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=o.id AND c.protected=1)
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='workflow-payload' AND e.record_id=o.id)`,
bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type='recipe' AND r.owner_id=o.id)" },
{ category: 'operation-payload', label: 'Completed command file bodies', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'payloadKeepDays',
failed: "o.state='failed'",
eligible: `o.state IN ('completed','failed') AND o.kind IN ('instance.create','instance.rebuild','instance.restart','instance.start','files.stage','files.read')
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='operation' AND c.record_id=o.id AND c.protected=1)
AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE o.id IN (n.operation_id,n.stage_operation_id) AND ((${unsettledDeployment}) OR EXISTS (
SELECT 1 FROM retention_controls c WHERE c.category IN ('provisioning','workflow-payload') AND c.record_id=d.id AND c.protected=1)))
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='operation-payload' AND e.record_id=o.id)`,
bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type IN ('operation','operation-result') AND r.owner_id=o.id)" },
{ category: 'version', label: 'Unused artifact versions', table: 'script_versions', id: 'id', clock: 'uploaded_at', days: 'versionKeepDays', minimum: false,
eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id)
AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id)
@@ -105,11 +120,37 @@ function versionReferences(): string {
return clauses.join(' AND ') || '1';
}
function initialize() { ensureHostSchema(db()); ensureAuthSchema(db()); }
function withoutFileBodies(value: unknown): unknown {
if (Array.isArray(value)) return value.map(withoutFileBodies);
if (!value || typeof value !== 'object') return value;
return Object.fromEntries(Object.entries(value).filter(([key]) => !['contentHash','contentSha256','contentBase64'].includes(key))
.map(([key, item]) => [key, withoutFileBodies(item)]));
}
function expirePayload(category: string, id: string): void {
if (category === 'workflow-payload') {
const row = db().prepare('SELECT recipe FROM server_launch_recipes WHERE deployment_id=?').get(id) as { recipe: string } | undefined;
if (row) db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(withoutFileBodies(JSON.parse(row.recipe))), id);
db().prepare('UPDATE native_provisioning SET files_json=NULL WHERE deployment_id=?').run(id);
referenceBlobs('recipe', id, []);
return;
}
const row = db().prepare(`SELECT COALESCE(p.payload,o.payload) payload,o.result FROM ec_host_operations o
LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(id) as { payload: string; result: string | null };
const payload = JSON.stringify(withoutFileBodies(JSON.parse(row.payload)));
db().prepare(`UPDATE ec_host_operations SET payload=?,result=?,request_hash=COALESCE(request_hash,?) WHERE id=?`)
.run(payload, row.result === null ? null : JSON.stringify(withoutFileBodies(JSON.parse(row.result))),
createHash('sha256').update(row.payload).digest('hex'), id);
db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(id);
referenceBlobs('operation', id, []);
referenceBlobs('operation-result', id, []);
}
function selection(policy: Policy, settings = fleetSettings()) {
const days = settings[policy.days];
const failureDays = settings.failureKeepDays;
const cut = (n: number) => new Date(Date.now() - n * 86400_000).toISOString();
const age = ['campaign','observation','operation'].includes(policy.category) ? `o.${policy.clock} < @cutoff` : policy.category === 'audit' ? "o.created_at < @numericCutoff" : `julianday(o.${policy.clock}) < julianday(@cutoff)`;
const age = policy.category === 'audit' ? 'o.created_at < @numericCutoff' : `o.${policy.clock} < @cutoff`;
const failed = policy.failed ?? '0';
const enabled = days > 0 || policy.category === 'version' && settings.versionKeepCount > 0;
// Keep age outside the correlated failure checks so SQLite can seek the
@@ -120,8 +161,8 @@ function selection(policy: Policy, settings = fleetSettings()) {
WHERE ${policy.category === 'version' && !days ? '1' : age} AND NOT (${failed})`);
if (failureDays > 0 && policy.failed) candidates.push(`SELECT o.rowid FROM retention_controls c
JOIN ${policy.table} o ON o.${policy.id}=c.record_id
WHERE c.category=@category AND julianday(c.resolved_at)<julianday(@failureCutoff)
AND julianday(c.resolved_at)>=julianday(o.${policy.clock}) AND (${failed})`);
WHERE c.category=@category AND c.resolved_at<@failureCutoff
AND c.resolved_at>=o.${policy.clock} AND (${failed})`);
const minimum = policy.minimum === false ? 0 : settings.historyKeepCount;
const minimumScope = policy.category === 'observation' ? "WHERE observation_success=1 AND state='completed'"
: policy.category === 'operation' ? "WHERE history_expired_at IS NULL AND (routine=0 OR observation_success=0)" : '';
@@ -187,9 +228,12 @@ export function pruneHistory(only?: string): number {
}
db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(row.id);
}
if (policy.category === 'beta-file-content') {
if (policy.category === 'workflow-payload' || policy.category === 'operation-payload') {
expirePayload(policy.category, row.id);
db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
} else if (policy.category === 'beta-file-content') {
db().prepare("UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?").run(row.id);
db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id=?').run(row.id);
db().prepare('UPDATE beta_file_snapshots SET prior_content_hash=NULL WHERE beta_id=?').run(row.id);
db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
} else if (policy.category === 'file-content') {
db().prepare('UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?').run(row.id);
@@ -203,7 +247,8 @@ export function pruneHistory(only?: string): number {
return rows.length;
})();
}
pruneFileBlobs();
const blobs = pruneFileBlobs();
if ((removed || blobs) && Number(db().pragma('freelist_count', { simple: true })) >= 256) db().pragma('incremental_vacuum(256)');
return removed;
}