josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Deduplicate artifact bytes and deliver files by authenticated hash

Browse Source
This commit is contained in:
josh
2026-10-06 14:35:13 -04:00
parent 18283c63a7
commit 62df14b180
45 changed files with 510 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/retention.ts
+11 -9
View File
@@ -1,3 +1,4 @@
import { blobSize, pruneFileBlobs } from './db/file-blobs.js';
import { createHash } from 'node:crypto';
import { db } from './db/index.js';
import { fleetSettings } from './settings.js';
@@ -42,7 +43,7 @@ function policies(): Policy[] {
return [
{ category: 'campaign', label: 'Campaigns', table: 'deployments', id: 'id', clock: 'started_at', days: 'deployKeepDays',
eligible: `${campaignSafe} AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='file-content' AND c.record_id=o.id AND c.protected=1)`, failed: failedCampaign,
bytes: `(SELECT COALESCE(SUM(COALESCE(length(content_before),0)+COALESCE(length(content_after),0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
{ category: 'provisioning', label: 'Provisioning workflows', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'provisioningKeepDays', failed: "o.state='failed'", eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
AND NOT EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=o.id AND r.healthy_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id) WHERE n.deployment_id=o.id AND a.state NOT IN ('completed','failed'))
@@ -70,22 +71,22 @@ function policies(): Policy[] {
{ category: 'beta', label: 'Completed betas', table: 'betas', id: 'id', clock: 'closed_at', days: 'betaKeepDays',
eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta-file-content' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id)
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
bytes: '(SELECT COALESCE(SUM(length(content)),0) FROM beta_revision_files WHERE beta_id=o.id)' },
bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` },
{ category: 'file-content', label: 'Campaign file bodies', table: 'deployments', id: 'id', clock: 'finished_at', days: 'fileContentKeepDays',
eligible: `${campaignSafe} AND NOT (${failedCampaign})
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=o.id AND c.protected=1)
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`,
bytes: '(SELECT COALESCE(SUM(COALESCE(length(content_before),0)+COALESCE(length(content_after),0)),0) FROM deployment_files WHERE deployment_id=o.id)' },
bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
{ category: 'beta-file-content', label: 'Closed beta file snapshots', table: 'betas', id: 'id', clock: 'closed_at', days: 'fileContentKeepDays',
eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id)
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1)
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT))
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
bytes: '(SELECT COALESCE(SUM(length(content)),0) FROM beta_revision_files WHERE beta_id=o.id)' },
bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` },
{ category: 'version', label: 'Unused artifact versions', table: 'script_versions', id: 'id', clock: 'uploaded_at', days: 'versionKeepDays', minimum: false,
eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id)
AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id)
AND ${versionReferences()}`, bytes: 'length(o.content)' },
AND ${versionReferences()}`, bytes: `${blobSize('o.content_hash')}` },
{ category: 'audit', label: 'Authentication audit events', table: 'auth_audit', id: 'id', clock: 'created_at', days: 'auditKeepDays', eligible: '1', bytes: 'length(o.details)' }
];
}
@@ -181,18 +182,18 @@ export function pruneHistory(only?: string): number {
db().prepare('DELETE FROM ec_host_operation_events WHERE operation_id=?').run(row.id);
if (policy.category === 'operation') {
const payload = db().prepare(`SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(row.id) as { payload: string };
db().prepare(`UPDATE ec_host_operations SET request_hash=?,payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`)
db().prepare(`UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?),payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`)
.run(createHash('sha256').update(payload.payload).digest('hex'),new Date().toISOString(),row.id);
}
db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(row.id);
}
if (policy.category === 'beta-file-content') {
db().prepare("UPDATE beta_revision_files SET content=X'' WHERE beta_id=?").run(row.id);
db().prepare("UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?").run(row.id);
db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id=?').run(row.id);
db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
} else if (policy.category === 'file-content') {
db().prepare('UPDATE deployment_files SET content_before=NULL,content_after=NULL WHERE deployment_id=?').run(row.id);
db().prepare("UPDATE staged_file_changes SET content=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')").run(row.id);
db().prepare('UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?').run(row.id);
db().prepare("UPDATE staged_file_changes SET content_hash=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')").run(row.id);
db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
} else if (policy.category !== 'operation') {
db().prepare(`DELETE FROM ${policy.table} WHERE rowid=?`).run(row.rid);
@@ -202,6 +203,7 @@ export function pruneHistory(only?: string): number {
return rows.length;
})();
}
pruneFileBlobs();
return removed;
}