Expire retained file bodies and reclaim SQLite storage

src/lib/fleet-settings.ts

@@ -13,6 +13,7 @@ export interface FleetSettings {
     matchKeepDays: number;
     betaKeepDays: number;
     fileContentKeepDays: number;
+    payloadKeepDays: number;
     versionKeepDays: number;
     versionKeepCount: number;
     gameFileGraceDays: number;
@@ -49,7 +50,8 @@ export const defaultFleetSettings: FleetSettings = {
     telemetryMaxTotal: 10000,
     matchKeepDays: 0,
     betaKeepDays: 0,
-    fileContentKeepDays: 0,
+    fileContentKeepDays: 30,
+    payloadKeepDays: 30,
     versionKeepDays: 0,
     versionKeepCount: 0,
     gameFileGraceDays: 0,
@@ -86,6 +88,7 @@ export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
     matchKeepDays: 'Match history retention',
     betaKeepDays: 'Completed beta history retention',
     fileContentKeepDays: 'File diffs and rollback snapshot retention',
+    payloadKeepDays: 'Launch and command file retention',
     versionKeepDays: 'Unused artifact version retention',
     versionKeepCount: 'Minimum artifact versions to keep',
     gameFileGraceDays: 'Unused game-file grace period',
@@ -129,7 +132,7 @@ export function validateFleetSettings(input: unknown): FleetSettings {
 
 export const additionalConcurrencyKeys: (keyof FleetSettings)[] = ["restartConcurrencyPerHost", "createConcurrencyPerHost", "destroyConcurrencyPerHost", "lifecycleConcurrency", "rebuildConcurrency", "preparationConcurrency"];
 export const concurrencyKeys: (keyof FleetSettings)[] = ['restartConcurrency', 'createConcurrency', 'destroyConcurrency', ...additionalConcurrencyKeys];
-export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays', 'provisioningKeepDays', 'actionKeepDays', 'failureKeepDays', 'maintenanceKeepDays', 'recoveryKeepDays', 'hostResourceKeepDays', 'instanceResourceKeepDays', 'metricsKeepDays', 'telemetryKeepDays', 'telemetryMaxPerServer', 'telemetryMaxTotal', 'matchKeepDays', 'betaKeepDays', 'fileContentKeepDays', 'versionKeepDays', 'versionKeepCount', 'gameFileGraceDays', 'gameFileKeepCount', 'uploadKeepDays', 'auditKeepDays', 'historyKeepCount'];
+export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays', 'provisioningKeepDays', 'actionKeepDays', 'failureKeepDays', 'maintenanceKeepDays', 'recoveryKeepDays', 'hostResourceKeepDays', 'instanceResourceKeepDays', 'metricsKeepDays', 'telemetryKeepDays', 'telemetryMaxPerServer', 'telemetryMaxTotal', 'matchKeepDays', 'betaKeepDays', 'fileContentKeepDays', 'payloadKeepDays', 'versionKeepDays', 'versionKeepCount', 'gameFileGraceDays', 'gameFileKeepCount', 'uploadKeepDays', 'auditKeepDays', 'historyKeepCount'];
 export function settingMinimum(key: keyof FleetSettings): number {
     return ['restartConcurrency','createConcurrency','destroyConcurrency','gameFileKeepCount'].includes(key) ? 1 : 0;
 }
@@ -149,6 +152,7 @@ export const settingHelp: Partial<Record<keyof FleetSettings, string>> = {
     matchKeepDays: 'Completed and interrupted matches and player details. Expiry changes leaderboards and player totals; no lifetime aggregates are preserved.',
     betaKeepDays: 'Closed betas, revisions and test evidence, after linked campaigns and recovery work no longer need them.',
     fileContentKeepDays: 'Expired campaign file bodies are removed while summaries remain. Rollback becomes unavailable for those campaigns. Active beta snapshots remain protected.',
+    payloadKeepDays: 'Expire file bodies in completed launch recipes, commands and receipts while keeping their history. Unfinished work and protected records keep their files. 0 keeps files indefinitely.',
     versionKeepDays: 'Unused script, plugin, configuration and baseline versions. Current and referenced versions are always protected.',
     versionKeepCount: 'Keep at least this many recent versions per artifact. With no age expiry, a positive count removes older unused versions.',
     gameFileGraceDays: 'Delay removal after a version becomes superseded. Latest files, referenced versions and offline host copies remain protected. 0 adds no delay.',

src/lib/server/auth/schema.ts

@@ -22,6 +22,7 @@ export function ensureAuthSchema(handle: Database.Database): void {
             id INTEGER PRIMARY KEY AUTOINCREMENT, actor_id TEXT, action TEXT NOT NULL,
             target_id TEXT, details TEXT NOT NULL, created_at INTEGER NOT NULL
         );
+        CREATE INDEX IF NOT EXISTS auth_audit_created_at ON auth_audit(created_at);
     `);
     initialized.add(handle);
 }

src/lib/server/db/index.ts

@@ -13,6 +13,7 @@ import { migrateNativeInstances } from '../backends/schema.js';
 import { installPerformanceIndexes } from './performance-indexes.js';
 import { migrateVersionLayout } from './version-layout.js';
 import { migrateFileBlobs } from './file-blobs.js';
+import { enableIncrementalVacuum, retentionClocks } from './retention-clocks.js';
 import { tuneConnection } from './connection.js';
 import { timeStatement } from './slow-statements.js';
 
@@ -318,6 +319,7 @@ export function db(): Database.Database {
     // silently breaks every subsequent query without a chance to retry.
     try {
         handle.pragma('journal_mode = WAL');
+        handle.pragma('auto_vacuum = INCREMENTAL');
         handle.pragma('foreign_keys = ON');
         tuneConnection(handle);
         dropLegacyBetaTables(handle);
@@ -414,6 +416,14 @@ export function db(): Database.Database {
             ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`);
         installPerformanceIndexes(handle);
         installReportRevisions(handle);
+        retentionClocks(handle, {
+            deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'],
+            maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'],
+            leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
+            betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at']
+        });
+        // Existing databases require one rebuild to gain the incremental-vacuum page map.
+        enableIncrementalVacuum(handle);
         handle.pragma('optimize = 0x10002');
     } catch (err) {
         handle.close();

src/lib/server/db/retention-clocks.ts

@@ -0,0 +1,45 @@
+import type Database from 'better-sqlite3';
+
+/** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */
+export function retentionClocks(handle: Database.Database, clocks: Record<string, string[]>): void {
+    handle.exec('CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)');
+    for (const [table, columns] of Object.entries(clocks)) {
+        const marker = `iso-clocks:${table}:v1`;
+        if (!handle.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?').get(marker)) {
+            handle.transaction(() => {
+                for (const column of columns) handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column})
+                    WHERE julianday(${column}) IS NOT NULL AND ${column}!=strftime('%Y-%m-%dT%H:%M:%fZ',${column});`);
+                handle.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)').run(marker);
+            })();
+        }
+        const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as { name: string; partial: number }[];
+        const indexed = new Set(indexes.filter(index => !index.partial).map(index =>
+            (handle.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`).get() as { name: string }).name));
+        for (const column of columns) if (!indexed.has(column)) {
+            handle.exec(`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`);
+        }
+    }
+}
+
+/** VACUUM can renumber implicit rowids. Keep host enable cutoffs tied to command IDs across crashes. */
+export function enableIncrementalVacuum(handle: Database.Database): void {
+    if (handle.pragma('auto_vacuum', { simple: true }) !== 2) {
+        handle.exec(`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`);
+        if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) {
+            handle.exec(`INSERT OR IGNORE INTO database_vacuum_boundaries
+                SELECT e.host_id,(SELECT o.id FROM ec_host_operations o WHERE o.host_id=e.host_id
+                    AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`);
+        }
+        handle.pragma('auto_vacuum = INCREMENTAL');
+        handle.exec('VACUUM');
+    }
+    if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) {
+        handle.transaction(() => {
+            if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) handle.exec(`
+                UPDATE ec_host_enable_requests AS e SET operation_floor=COALESCE((
+                    SELECT o.rowid FROM database_vacuum_boundaries b JOIN ec_host_operations o ON o.id=b.operation_id
+                    WHERE b.host_id=e.host_id),0) WHERE e.host_id IN (SELECT host_id FROM database_vacuum_boundaries)`);
+            handle.exec('DROP TABLE database_vacuum_boundaries');
+        })();
+    }
+}

src/lib/server/hosts/schema.ts

@@ -1,5 +1,6 @@
 import { createHash } from 'node:crypto';
 import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js';
+import { retentionClocks } from '../db/retention-clocks.js';
 import type Database from 'better-sqlite3';
 import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
 
@@ -185,5 +186,6 @@ export function ensureHostSchema(handle: Database.Database): void {
         CREATE TRIGGER IF NOT EXISTS expire_operation_blobs AFTER UPDATE OF history_expired_at ON ec_host_operations
         WHEN NEW.history_expired_at IS NOT NULL BEGIN
         DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=NEW.id; END;`);
+    retentionClocks(handle, { ec_host_operations: ['created_at', 'updated_at'] });
     initialized.add(handle);
 }

src/lib/server/retention.ts

@@ -1,4 +1,4 @@
-import { blobSize, pruneFileBlobs } from './db/file-blobs.js';
+import { blobSize, pruneFileBlobs, referenceBlobs } from './db/file-blobs.js';
 import { createHash } from 'node:crypto';
 import { db } from './db/index.js';
 import { fleetSettings } from './settings.js';
@@ -78,11 +78,26 @@ function policies(): Policy[] {
                 AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`,
             bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
         { category: 'beta-file-content', label: 'Closed beta file snapshots', table: 'betas', id: 'id', clock: 'closed_at', days: 'fileContentKeepDays',
-            eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id)
+            eligible: `o.state IN ('promoted','rolled-back')
                 AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1)
                 AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT))
                 AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
             bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` },
+        { category: 'workflow-payload', label: 'Completed launch file bodies', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'payloadKeepDays',
+            failed: "o.state='failed'",
+            eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
+                AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=o.id AND c.protected=1)
+                AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='workflow-payload' AND e.record_id=o.id)`,
+            bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type='recipe' AND r.owner_id=o.id)" },
+        { category: 'operation-payload', label: 'Completed command file bodies', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'payloadKeepDays',
+            failed: "o.state='failed'",
+            eligible: `o.state IN ('completed','failed') AND o.kind IN ('instance.create','instance.rebuild','instance.restart','instance.start','files.stage','files.read')
+                AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='operation' AND c.record_id=o.id AND c.protected=1)
+                AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
+                    WHERE o.id IN (n.operation_id,n.stage_operation_id) AND ((${unsettledDeployment}) OR EXISTS (
+                        SELECT 1 FROM retention_controls c WHERE c.category IN ('provisioning','workflow-payload') AND c.record_id=d.id AND c.protected=1)))
+                AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='operation-payload' AND e.record_id=o.id)`,
+            bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type IN ('operation','operation-result') AND r.owner_id=o.id)" },
         { category: 'version', label: 'Unused artifact versions', table: 'script_versions', id: 'id', clock: 'uploaded_at', days: 'versionKeepDays', minimum: false,
             eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id)
                 AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id)
@@ -105,11 +120,37 @@ function versionReferences(): string {
     return clauses.join(' AND ') || '1';
 }
 function initialize() { ensureHostSchema(db()); ensureAuthSchema(db()); }
+
+function withoutFileBodies(value: unknown): unknown {
+    if (Array.isArray(value)) return value.map(withoutFileBodies);
+    if (!value || typeof value !== 'object') return value;
+    return Object.fromEntries(Object.entries(value).filter(([key]) => !['contentHash','contentSha256','contentBase64'].includes(key))
+        .map(([key, item]) => [key, withoutFileBodies(item)]));
+}
+
+function expirePayload(category: string, id: string): void {
+    if (category === 'workflow-payload') {
+        const row = db().prepare('SELECT recipe FROM server_launch_recipes WHERE deployment_id=?').get(id) as { recipe: string } | undefined;
+        if (row) db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(withoutFileBodies(JSON.parse(row.recipe))), id);
+        db().prepare('UPDATE native_provisioning SET files_json=NULL WHERE deployment_id=?').run(id);
+        referenceBlobs('recipe', id, []);
+        return;
+    }
+    const row = db().prepare(`SELECT COALESCE(p.payload,o.payload) payload,o.result FROM ec_host_operations o
+        LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(id) as { payload: string; result: string | null };
+    const payload = JSON.stringify(withoutFileBodies(JSON.parse(row.payload)));
+    db().prepare(`UPDATE ec_host_operations SET payload=?,result=?,request_hash=COALESCE(request_hash,?) WHERE id=?`)
+        .run(payload, row.result === null ? null : JSON.stringify(withoutFileBodies(JSON.parse(row.result))),
+            createHash('sha256').update(row.payload).digest('hex'), id);
+    db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(id);
+    referenceBlobs('operation', id, []);
+    referenceBlobs('operation-result', id, []);
+}
 function selection(policy: Policy, settings = fleetSettings()) {
     const days = settings[policy.days];
     const failureDays = settings.failureKeepDays;
     const cut = (n: number) => new Date(Date.now() - n * 86400_000).toISOString();
-    const age = ['campaign','observation','operation'].includes(policy.category) ? `o.${policy.clock} < @cutoff` : policy.category === 'audit' ? "o.created_at < @numericCutoff" : `julianday(o.${policy.clock}) < julianday(@cutoff)`;
+    const age = policy.category === 'audit' ? 'o.created_at < @numericCutoff' : `o.${policy.clock} < @cutoff`;
     const failed = policy.failed ?? '0';
     const enabled = days > 0 || policy.category === 'version' && settings.versionKeepCount > 0;
     // Keep age outside the correlated failure checks so SQLite can seek the
@@ -120,8 +161,8 @@ function selection(policy: Policy, settings = fleetSettings()) {
         WHERE ${policy.category === 'version' && !days ? '1' : age} AND NOT (${failed})`);
     if (failureDays > 0 && policy.failed) candidates.push(`SELECT o.rowid FROM retention_controls c
         JOIN ${policy.table} o ON o.${policy.id}=c.record_id
-        WHERE c.category=@category AND julianday(c.resolved_at)<julianday(@failureCutoff)
-        AND julianday(c.resolved_at)>=julianday(o.${policy.clock}) AND (${failed})`);
+        WHERE c.category=@category AND c.resolved_at<@failureCutoff
+        AND c.resolved_at>=o.${policy.clock} AND (${failed})`);
     const minimum = policy.minimum === false ? 0 : settings.historyKeepCount;
     const minimumScope = policy.category === 'observation' ? "WHERE observation_success=1 AND state='completed'"
         : policy.category === 'operation' ? "WHERE history_expired_at IS NULL AND (routine=0 OR observation_success=0)" : '';
@@ -187,9 +228,12 @@ export function pruneHistory(only?: string): number {
                     }
                     db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(row.id);
                 }
-                if (policy.category === 'beta-file-content') {
+                if (policy.category === 'workflow-payload' || policy.category === 'operation-payload') {
+                    expirePayload(policy.category, row.id);
+                    db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
+                } else if (policy.category === 'beta-file-content') {
                     db().prepare("UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?").run(row.id);
-                    db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id=?').run(row.id);
+                    db().prepare('UPDATE beta_file_snapshots SET prior_content_hash=NULL WHERE beta_id=?').run(row.id);
                     db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
                 } else if (policy.category === 'file-content') {
                     db().prepare('UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?').run(row.id);
@@ -203,7 +247,8 @@ export function pruneHistory(only?: string): number {
             return rows.length;
         })();
     }
-    pruneFileBlobs();
+    const blobs = pruneFileBlobs();
+    if ((removed || blobs) && Number(db().pragma('freelist_count', { simple: true })) >= 256) db().pragma('incremental_vacuum(256)');
     return removed;
 }