Keep startup database rebuilds out of memory

src/lib/server/db/connection.ts

@@ -2,7 +2,9 @@ import type Database from 'better-sqlite3';
 
 export function tuneConnection(handle: Database.Database, reader = false): void {
     handle.pragma(`cache_size = -${reader ? 32768 : 131072}`);
-    handle.pragma('temp_store = MEMORY');
+    // MEMORY also puts VACUUM's transient database in RAM. Disk-backed temps
+    // keep migrations and large sorts from growing with the entire database.
+    handle.pragma('temp_store = FILE');
     // mmap is opt-in because the controller data directory can be a network mount.
     const mmapMiB = Number(process.env.EC_SQLITE_MMAP_MB ?? 0);
     if (Number.isSafeInteger(mmapMiB) && mmapMiB >= 0 && mmapMiB <= 32768) {
@@ -10,3 +12,25 @@ export function tuneConnection(handle: Database.Database, reader = false): void
     }
     if (!reader) handle.pragma('journal_size_limit = 67108864');
 }
+
+/** Startup needs two page caches during VACUUM, plus buffers for legacy files. */
+export function tuneMigrationConnection(handle: Database.Database): void {
+    handle.pragma('cache_size = -16384');
+    handle.pragma('temp_store = FILE');
+    handle.pragma('mmap_size = 0');
+    handle.pragma('journal_size_limit = 67108864');
+}
+
+export function migrationStep<T>(name: string, work: () => T): T {
+    const started = Date.now();
+    console.info(`[database] migration starting: ${name}`);
+    try {
+        const result = work();
+        console.info(`[database] migration finished: ${name}`, { elapsedMs: Date.now() - started,
+            rssMiB: Math.round(process.memoryUsage().rss / 1048576) });
+        return result;
+    } catch (error) {
+        console.error(`[database] migration failed: ${name}`, error);
+        throw error;
+    }
+}

src/lib/server/db/file-blobs.ts

@@ -1,6 +1,7 @@
 import { createHash } from 'node:crypto';
 import type Database from 'better-sqlite3';
 import { db } from './index.js';
+import { migrationStep } from './connection.js';
 
 export function ensureFileBlobSchema(handle: Database.Database): void {
     handle.exec(`CREATE TABLE IF NOT EXISTS file_blobs(sha256 TEXT PRIMARY KEY,size INTEGER NOT NULL,content BLOB NOT NULL);
@@ -84,18 +85,18 @@ export function* migrationRows<T>(handle: Database.Database, sql: string): Gener
 export function migrateFileBlobs(handle: Database.Database): void {
     for (const [table, column] of blobColumns) {
         if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue;
-        handle.transaction(() => {
+        migrationStep(`shared files: ${table}.${column}`, () => handle.transaction(() => {
             handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`);
             const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
             for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) {
                 update.run(putFileBlob(row.content, handle), row.rid);
             }
             handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
-        })();
+        })());
     }
     for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`);
     if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) {
-        handle.transaction(() => {
+        migrationStep('shared files: launch recipes', () => handle.transaction(() => {
             for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle,
                 'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) {
                 const packed = packFileJson(JSON.parse(row.recipe), handle);
@@ -103,7 +104,7 @@ export function migrateFileBlobs(handle: Database.Database): void {
                 referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
             }
             handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
-        })();
+        })());
     }
     handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN
         DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`);

src/lib/server/db/index.ts

@@ -11,10 +11,9 @@ import { initializeFleetSettings } from './fleet-settings.js';
 import { migrateServerHistory } from './server-history.js';
 import { migrateNativeInstances } from '../backends/schema.js';
 import { installPerformanceIndexes } from './performance-indexes.js';
-import { migrateVersionLayout } from './version-layout.js';
 import { migrateFileBlobs } from './file-blobs.js';
 import { enableIncrementalVacuum, retentionClocks } from './retention-clocks.js';
-import { tuneConnection } from './connection.js';
+import { migrationStep, tuneConnection, tuneMigrationConnection } from './connection.js';
 import { timeStatement } from './slow-statements.js';
 
 let _db: Database.Database | null = null;
@@ -318,10 +317,11 @@ export function db(): Database.Database {
     // singleton once migrations succeed. A half-migrated handle that's cached
     // silently breaks every subsequent query without a chance to retry.
     try {
+        console.info('[database] startup migrations starting');
         handle.pragma('journal_mode = WAL');
         handle.pragma('auto_vacuum = INCREMENTAL');
         handle.pragma('foreign_keys = ON');
-        tuneConnection(handle);
+        tuneMigrationConnection(handle);
         dropLegacyBetaTables(handle);
         handle.exec(schemaSql);
         migrateProvisioningActions(handle);
@@ -394,8 +394,9 @@ export function db(): Database.Database {
         ]);
         backfillLeaderboard(handle);
         initializeFleetSettings(handle);
-        migrateNativeInstances(handle);
-        migrateVersionLayout(handle);
+        migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
+        // File-blob migration removes the legacy content column altogether.
+        // Reordering it first would unnecessarily copy every artifact twice.
         migrateFileBlobs(handle);
         // Lifecycle history is retained indefinitely. Admission and status reads
         // must narrow by server or active state instead of scanning that history.
@@ -416,16 +417,20 @@ export function db(): Database.Database {
             ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`);
         installPerformanceIndexes(handle);
         installReportRevisions(handle);
-        retentionClocks(handle, {
+        migrationStep('retention clocks and indexes', () => retentionClocks(handle, {
             deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'],
             maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'],
             leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
             betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at']
-        });
+        }));
         // Existing databases require one rebuild to gain the incremental-vacuum page map.
         enableIncrementalVacuum(handle);
         handle.pragma('optimize = 0x10002');
+        handle.pragma('shrink_memory');
+        tuneConnection(handle);
+        console.info('[database] startup migrations complete');
     } catch (err) {
+        console.error('[database] startup migrations failed', err);
         handle.close();
         throw err;
     }

src/lib/server/db/retention-clocks.ts

@@ -1,4 +1,5 @@
 import type Database from 'better-sqlite3';
+import { migrationStep, tuneMigrationConnection } from './connection.js';
 
 /** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */
 export function retentionClocks(handle: Database.Database, clocks: Record<string, string[]>): void {
@@ -31,7 +32,24 @@ export function enableIncrementalVacuum(handle: Database.Database): void {
                     AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`);
         }
         handle.pragma('auto_vacuum = INCREMENTAL');
-        handle.exec('VACUUM');
+        // Guard this operation itself as well as startup: future callers must
+        // never inherit MEMORY temp storage or an oversized rebuild cache.
+        const previous = {
+            temp: handle.pragma('temp_store', { simple: true }),
+            cache: handle.pragma('cache_size', { simple: true }),
+            mmap: handle.pragma('mmap_size', { simple: true }),
+            journal: handle.pragma('journal_size_limit', { simple: true })
+        };
+        try {
+            tuneMigrationConnection(handle);
+            migrationStep('incremental vacuum page map (disk-backed rebuild)', () => handle.exec('VACUUM'));
+        } finally {
+            handle.pragma('shrink_memory');
+            handle.pragma(`temp_store = ${previous.temp}`);
+            handle.pragma(`cache_size = ${previous.cache}`);
+            if (previous.mmap !== undefined) handle.pragma(`mmap_size = ${previous.mmap}`);
+            handle.pragma(`journal_size_limit = ${previous.journal}`);
+        }
     }
     if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) {
         handle.transaction(() => {