josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Expire retained file bodies and reclaim SQLite storage

Browse Source
This commit is contained in:
josh
2026-10-06 14:43:37 -04:00
parent 62df14b180
commit 562b770725
9 changed files with 214 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/auth/schema.ts
+1
View File
@@ -22,6 +22,7 @@ export function ensureAuthSchema(handle: Database.Database): void {
id INTEGER PRIMARY KEY AUTOINCREMENT, actor_id TEXT, action TEXT NOT NULL,
target_id TEXT, details TEXT NOT NULL, created_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS auth_audit_created_at ON auth_audit(created_at);
`);
initialized.add(handle);
}
src/lib/server/db/index.ts
+10
View File
@@ -13,6 +13,7 @@ import { migrateNativeInstances } from '../backends/schema.js';
import { installPerformanceIndexes } from './performance-indexes.js';
import { migrateVersionLayout } from './version-layout.js';
import { migrateFileBlobs } from './file-blobs.js';
import { enableIncrementalVacuum, retentionClocks } from './retention-clocks.js';
import { tuneConnection } from './connection.js';
import { timeStatement } from './slow-statements.js';
@@ -318,6 +319,7 @@ export function db(): Database.Database {
// silently breaks every subsequent query without a chance to retry.
try {
handle.pragma('journal_mode = WAL');
handle.pragma('auto_vacuum = INCREMENTAL');
handle.pragma('foreign_keys = ON');
tuneConnection(handle);
dropLegacyBetaTables(handle);
@@ -414,6 +416,14 @@ export function db(): Database.Database {
ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`);
installPerformanceIndexes(handle);
installReportRevisions(handle);
retentionClocks(handle, {
deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'],
maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'],
leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at']
});
// Existing databases require one rebuild to gain the incremental-vacuum page map.
enableIncrementalVacuum(handle);
handle.pragma('optimize = 0x10002');
} catch (err) {
handle.close();
src/lib/server/db/retention-clocks.ts
+45
View File
@@ -0,0 +1,45 @@
import type Database from 'better-sqlite3';
/** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */
export function retentionClocks(handle: Database.Database, clocks: Record<string, string[]>): void {
handle.exec('CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)');
for (const [table, columns] of Object.entries(clocks)) {
const marker = `iso-clocks:${table}:v1`;
if (!handle.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?').get(marker)) {
handle.transaction(() => {
for (const column of columns) handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column})
WHERE julianday(${column}) IS NOT NULL AND ${column}!=strftime('%Y-%m-%dT%H:%M:%fZ',${column});`);
handle.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)').run(marker);
})();
}
const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as { name: string; partial: number }[];
const indexed = new Set(indexes.filter(index => !index.partial).map(index =>
(handle.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`).get() as { name: string }).name));
for (const column of columns) if (!indexed.has(column)) {
handle.exec(`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`);
}
}
}
/** VACUUM can renumber implicit rowids. Keep host enable cutoffs tied to command IDs across crashes. */
export function enableIncrementalVacuum(handle: Database.Database): void {
if (handle.pragma('auto_vacuum', { simple: true }) !== 2) {
handle.exec(`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`);
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) {
handle.exec(`INSERT OR IGNORE INTO database_vacuum_boundaries
SELECT e.host_id,(SELECT o.id FROM ec_host_operations o WHERE o.host_id=e.host_id
AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`);
}
handle.pragma('auto_vacuum = INCREMENTAL');
handle.exec('VACUUM');
}
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) {
handle.transaction(() => {
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) handle.exec(`
UPDATE ec_host_enable_requests AS e SET operation_floor=COALESCE((
SELECT o.rowid FROM database_vacuum_boundaries b JOIN ec_host_operations o ON o.id=b.operation_id
WHERE b.host_id=e.host_id),0) WHERE e.host_id IN (SELECT host_id FROM database_vacuum_boundaries)`);
handle.exec('DROP TABLE database_vacuum_boundaries');
})();
}
}
src/lib/server/hosts/schema.ts
+2
View File
@@ -1,5 +1,6 @@
import { createHash } from 'node:crypto';
import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js';
import { retentionClocks } from '../db/retention-clocks.js';
import type Database from 'better-sqlite3';
import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
@@ -185,5 +186,6 @@ export function ensureHostSchema(handle: Database.Database): void {
CREATE TRIGGER IF NOT EXISTS expire_operation_blobs AFTER UPDATE OF history_expired_at ON ec_host_operations
WHEN NEW.history_expired_at IS NOT NULL BEGIN
DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=NEW.id; END;`);
retentionClocks(handle, { ec_host_operations: ['created_at', 'updated_at'] });
initialized.add(handle);
}
src/lib/server/retention.ts
+53 -8
View File
@@ -1,4 +1,4 @@
import { blobSize, pruneFileBlobs } from './db/file-blobs.js';
import { blobSize, pruneFileBlobs, referenceBlobs } from './db/file-blobs.js';
import { createHash } from 'node:crypto';
import { db } from './db/index.js';
import { fleetSettings } from './settings.js';
@@ -78,11 +78,26 @@ function policies(): Policy[] {
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`,
bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
{ category: 'beta-file-content', label: 'Closed beta file snapshots', table: 'betas', id: 'id', clock: 'closed_at', days: 'fileContentKeepDays',
eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id)
eligible: `o.state IN ('promoted','rolled-back')
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1)
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT))
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` },
{ category: 'workflow-payload', label: 'Completed launch file bodies', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'payloadKeepDays',
failed: "o.state='failed'",
eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=o.id AND c.protected=1)
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='workflow-payload' AND e.record_id=o.id)`,
bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type='recipe' AND r.owner_id=o.id)" },
{ category: 'operation-payload', label: 'Completed command file bodies', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'payloadKeepDays',
failed: "o.state='failed'",
eligible: `o.state IN ('completed','failed') AND o.kind IN ('instance.create','instance.rebuild','instance.restart','instance.start','files.stage','files.read')
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='operation' AND c.record_id=o.id AND c.protected=1)
AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE o.id IN (n.operation_id,n.stage_operation_id) AND ((${unsettledDeployment}) OR EXISTS (
SELECT 1 FROM retention_controls c WHERE c.category IN ('provisioning','workflow-payload') AND c.record_id=d.id AND c.protected=1)))
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='operation-payload' AND e.record_id=o.id)`,
bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type IN ('operation','operation-result') AND r.owner_id=o.id)" },
{ category: 'version', label: 'Unused artifact versions', table: 'script_versions', id: 'id', clock: 'uploaded_at', days: 'versionKeepDays', minimum: false,
eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id)
AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id)
@@ -105,11 +120,37 @@ function versionReferences(): string {
return clauses.join(' AND ') || '1';
}
function initialize() { ensureHostSchema(db()); ensureAuthSchema(db()); }
function withoutFileBodies(value: unknown): unknown {
if (Array.isArray(value)) return value.map(withoutFileBodies);
if (!value || typeof value !== 'object') return value;
return Object.fromEntries(Object.entries(value).filter(([key]) => !['contentHash','contentSha256','contentBase64'].includes(key))
.map(([key, item]) => [key, withoutFileBodies(item)]));
}
function expirePayload(category: string, id: string): void {
if (category === 'workflow-payload') {
const row = db().prepare('SELECT recipe FROM server_launch_recipes WHERE deployment_id=?').get(id) as { recipe: string } | undefined;
if (row) db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(withoutFileBodies(JSON.parse(row.recipe))), id);
db().prepare('UPDATE native_provisioning SET files_json=NULL WHERE deployment_id=?').run(id);
referenceBlobs('recipe', id, []);
return;
}
const row = db().prepare(`SELECT COALESCE(p.payload,o.payload) payload,o.result FROM ec_host_operations o
LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(id) as { payload: string; result: string | null };
const payload = JSON.stringify(withoutFileBodies(JSON.parse(row.payload)));
db().prepare(`UPDATE ec_host_operations SET payload=?,result=?,request_hash=COALESCE(request_hash,?) WHERE id=?`)
.run(payload, row.result === null ? null : JSON.stringify(withoutFileBodies(JSON.parse(row.result))),
createHash('sha256').update(row.payload).digest('hex'), id);
db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(id);
referenceBlobs('operation', id, []);
referenceBlobs('operation-result', id, []);
}
function selection(policy: Policy, settings = fleetSettings()) {
const days = settings[policy.days];
const failureDays = settings.failureKeepDays;
const cut = (n: number) => new Date(Date.now() - n * 86400_000).toISOString();
const age = ['campaign','observation','operation'].includes(policy.category) ? `o.${policy.clock} < @cutoff` : policy.category === 'audit' ? "o.created_at < @numericCutoff" : `julianday(o.${policy.clock}) < julianday(@cutoff)`;
const age = policy.category === 'audit' ? 'o.created_at < @numericCutoff' : `o.${policy.clock} < @cutoff`;
const failed = policy.failed ?? '0';
const enabled = days > 0 || policy.category === 'version' && settings.versionKeepCount > 0;
// Keep age outside the correlated failure checks so SQLite can seek the
@@ -120,8 +161,8 @@ function selection(policy: Policy, settings = fleetSettings()) {
WHERE ${policy.category === 'version' && !days ? '1' : age} AND NOT (${failed})`);
if (failureDays > 0 && policy.failed) candidates.push(`SELECT o.rowid FROM retention_controls c
JOIN ${policy.table} o ON o.${policy.id}=c.record_id
WHERE c.category=@category AND julianday(c.resolved_at)<julianday(@failureCutoff)
AND julianday(c.resolved_at)>=julianday(o.${policy.clock}) AND (${failed})`);
WHERE c.category=@category AND c.resolved_at<@failureCutoff
AND c.resolved_at>=o.${policy.clock} AND (${failed})`);
const minimum = policy.minimum === false ? 0 : settings.historyKeepCount;
const minimumScope = policy.category === 'observation' ? "WHERE observation_success=1 AND state='completed'"
: policy.category === 'operation' ? "WHERE history_expired_at IS NULL AND (routine=0 OR observation_success=0)" : '';
@@ -187,9 +228,12 @@ export function pruneHistory(only?: string): number {
}
db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(row.id);
}
if (policy.category === 'beta-file-content') {
if (policy.category === 'workflow-payload' || policy.category === 'operation-payload') {
expirePayload(policy.category, row.id);
db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
} else if (policy.category === 'beta-file-content') {
db().prepare("UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?").run(row.id);
db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id=?').run(row.id);
db().prepare('UPDATE beta_file_snapshots SET prior_content_hash=NULL WHERE beta_id=?').run(row.id);
db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
} else if (policy.category === 'file-content') {
db().prepare('UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?').run(row.id);
@@ -203,7 +247,8 @@ export function pruneHistory(only?: string): number {
return rows.length;
})();
}
pruneFileBlobs();
const blobs = pruneFileBlobs();
if ((removed || blobs) && Number(db().pragma('freelist_count', { simple: true })) >= 256) db().pragma('incremental_vacuum(256)');
return removed;
}