Expire retained file bodies and reclaim SQLite storage

src/lib/server/db/index.ts

@@ -13,6 +13,7 @@ import { migrateNativeInstances } from '../backends/schema.js';
 import { installPerformanceIndexes } from './performance-indexes.js';
 import { migrateVersionLayout } from './version-layout.js';
 import { migrateFileBlobs } from './file-blobs.js';
+import { enableIncrementalVacuum, retentionClocks } from './retention-clocks.js';
 import { tuneConnection } from './connection.js';
 import { timeStatement } from './slow-statements.js';
 
@@ -318,6 +319,7 @@ export function db(): Database.Database {
     // silently breaks every subsequent query without a chance to retry.
     try {
         handle.pragma('journal_mode = WAL');
+        handle.pragma('auto_vacuum = INCREMENTAL');
         handle.pragma('foreign_keys = ON');
         tuneConnection(handle);
         dropLegacyBetaTables(handle);
@@ -414,6 +416,14 @@ export function db(): Database.Database {
             ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`);
         installPerformanceIndexes(handle);
         installReportRevisions(handle);
+        retentionClocks(handle, {
+            deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'],
+            maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'],
+            leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
+            betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at']
+        });
+        // Existing databases require one rebuild to gain the incremental-vacuum page map.
+        enableIncrementalVacuum(handle);
         handle.pragma('optimize = 0x10002');
     } catch (err) {
         handle.close();

src/lib/server/db/retention-clocks.ts

@@ -0,0 +1,45 @@
+import type Database from 'better-sqlite3';
+
+/** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */
+export function retentionClocks(handle: Database.Database, clocks: Record<string, string[]>): void {
+    handle.exec('CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)');
+    for (const [table, columns] of Object.entries(clocks)) {
+        const marker = `iso-clocks:${table}:v1`;
+        if (!handle.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?').get(marker)) {
+            handle.transaction(() => {
+                for (const column of columns) handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column})
+                    WHERE julianday(${column}) IS NOT NULL AND ${column}!=strftime('%Y-%m-%dT%H:%M:%fZ',${column});`);
+                handle.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)').run(marker);
+            })();
+        }
+        const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as { name: string; partial: number }[];
+        const indexed = new Set(indexes.filter(index => !index.partial).map(index =>
+            (handle.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`).get() as { name: string }).name));
+        for (const column of columns) if (!indexed.has(column)) {
+            handle.exec(`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`);
+        }
+    }
+}
+
+/** VACUUM can renumber implicit rowids. Keep host enable cutoffs tied to command IDs across crashes. */
+export function enableIncrementalVacuum(handle: Database.Database): void {
+    if (handle.pragma('auto_vacuum', { simple: true }) !== 2) {
+        handle.exec(`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`);
+        if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) {
+            handle.exec(`INSERT OR IGNORE INTO database_vacuum_boundaries
+                SELECT e.host_id,(SELECT o.id FROM ec_host_operations o WHERE o.host_id=e.host_id
+                    AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`);
+        }
+        handle.pragma('auto_vacuum = INCREMENTAL');
+        handle.exec('VACUUM');
+    }
+    if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) {
+        handle.transaction(() => {
+            if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) handle.exec(`
+                UPDATE ec_host_enable_requests AS e SET operation_floor=COALESCE((
+                    SELECT o.rowid FROM database_vacuum_boundaries b JOIN ec_host_operations o ON o.id=b.operation_id
+                    WHERE b.host_id=e.host_id),0) WHERE e.host_id IN (SELECT host_id FROM database_vacuum_boundaries)`);
+            handle.exec('DROP TABLE database_vacuum_boundaries');
+        })();
+    }
+}