Keep startup database rebuilds out of memory

Dockerfile

@@ -24,6 +24,7 @@ COPY --from=build /app/node_modules ./node_modules
 COPY --from=build /app/package.json ./package.json
 COPY --from=build /app/src/lib/server/db/schema.sql ./build/schema.sql
 COPY scripts/reset-admin.mjs ./scripts/reset-admin.mjs
+COPY scripts/database-diagnostics.mjs ./scripts/database-diagnostics.mjs
 COPY src/lib/server/auth/passwords.js ./src/lib/server/auth/passwords.js
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
@@ -32,6 +33,7 @@ ENV NODE_ENV=production
 ENV PORT=3000
 ENV HOST=0.0.0.0
 ENV BODY_SIZE_LIMIT=Infinity
+ENV SQLITE_TMPDIR=/data/.sqlite-tmp
 EXPOSE 3000
 
 ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"]

README.md

@@ -198,6 +198,15 @@ after database changes, capture these timings under the same fleet load and run
 the largest tables/indexes, database and WAL sizes, and free pages. This command
 opens an existing database read-only and does not run app migrations.
 The controller uses a 128 MiB page cache and each reader worker uses 32 MiB.
+Main-database migrations use a 16 MiB cache with memory mapping disabled; SQLite
+temporary storage stays disk-backed during migrations and normal operation.
+The container uses `/data/.sqlite-tmp` for SQLite scratch files so a memory-backed
+`/tmp` cannot turn a database rebuild into an allocation of the whole database.
+Override `SQLITE_TMPDIR` only with another writable, disk-backed directory.
+Startup logs identify migration phases, elapsed time and resident memory. A
+completed migration is retained across restarts; SQLite rolls back interrupted
+transactions and startup retries those migrations. The one-time vacuum still needs disk space for its temporary
+copy. Do not remove `cherry.db-wal` to recover from an interrupted upgrade.
 For a local data disk, set `EC_SQLITE_MMAP_MB=256` to enable memory-mapped reads.
 Query statistics are optimized at startup and hourly; WAL reuse is capped at
 64 MiB after checkpoints (active transactions can temporarily grow it larger).

docs/settings.md

@@ -93,6 +93,9 @@ available longer. Previously saved retention choices are preserved on upgrade.
 The database upgrade normalizes retention clocks to UTC and enables incremental
 vacuum. An existing database without a vacuum page map is rebuilt once at startup;
 allow time for this rebuild and free disk space up to twice the database size.
+The rebuild uses disk-backed temporary storage and a 16 MiB page cache instead
+of holding a temporary copy of the database in RAM. Container scratch files use
+`/data/.sqlite-tmp`; `[database]` startup logs identify the current migration.
 Normal cleanup releases unused file blobs in bounded batches and incrementally
 returns free pages to disk after larger prunes.
 

entrypoint.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 set -eu
 
+if [ -n "${SQLITE_TMPDIR:-}" ]; then
+    mkdir -p "$SQLITE_TMPDIR"
+fi
+
 echo "[entrypoint] starting node server on ${HOST:-0.0.0.0}:${PORT:-3000}"
 exec node build/index.js

src/lib/server/db/connection.ts

@@ -2,7 +2,9 @@ import type Database from 'better-sqlite3';
 
 export function tuneConnection(handle: Database.Database, reader = false): void {
     handle.pragma(`cache_size = -${reader ? 32768 : 131072}`);
-    handle.pragma('temp_store = MEMORY');
+    // MEMORY also puts VACUUM's transient database in RAM. Disk-backed temps
+    // keep migrations and large sorts from growing with the entire database.
+    handle.pragma('temp_store = FILE');
     // mmap is opt-in because the controller data directory can be a network mount.
     const mmapMiB = Number(process.env.EC_SQLITE_MMAP_MB ?? 0);
     if (Number.isSafeInteger(mmapMiB) && mmapMiB >= 0 && mmapMiB <= 32768) {
@@ -10,3 +12,25 @@ export function tuneConnection(handle: Database.Database, reader = false): void
     }
     if (!reader) handle.pragma('journal_size_limit = 67108864');
 }
+
+/** Startup needs two page caches during VACUUM, plus buffers for legacy files. */
+export function tuneMigrationConnection(handle: Database.Database): void {
+    handle.pragma('cache_size = -16384');
+    handle.pragma('temp_store = FILE');
+    handle.pragma('mmap_size = 0');
+    handle.pragma('journal_size_limit = 67108864');
+}
+
+export function migrationStep<T>(name: string, work: () => T): T {
+    const started = Date.now();
+    console.info(`[database] migration starting: ${name}`);
+    try {
+        const result = work();
+        console.info(`[database] migration finished: ${name}`, { elapsedMs: Date.now() - started,
+            rssMiB: Math.round(process.memoryUsage().rss / 1048576) });
+        return result;
+    } catch (error) {
+        console.error(`[database] migration failed: ${name}`, error);
+        throw error;
+    }
+}

src/lib/server/db/file-blobs.ts

@@ -1,6 +1,7 @@
 import { createHash } from 'node:crypto';
 import type Database from 'better-sqlite3';
 import { db } from './index.js';
+import { migrationStep } from './connection.js';
 
 export function ensureFileBlobSchema(handle: Database.Database): void {
     handle.exec(`CREATE TABLE IF NOT EXISTS file_blobs(sha256 TEXT PRIMARY KEY,size INTEGER NOT NULL,content BLOB NOT NULL);
@@ -84,18 +85,18 @@ export function* migrationRows<T>(handle: Database.Database, sql: string): Gener
 export function migrateFileBlobs(handle: Database.Database): void {
     for (const [table, column] of blobColumns) {
         if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue;
-        handle.transaction(() => {
+        migrationStep(`shared files: ${table}.${column}`, () => handle.transaction(() => {
             handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`);
             const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
             for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) {
                 update.run(putFileBlob(row.content, handle), row.rid);
             }
             handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
-        })();
+        })());
     }
     for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`);
     if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) {
-        handle.transaction(() => {
+        migrationStep('shared files: launch recipes', () => handle.transaction(() => {
             for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle,
                 'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) {
                 const packed = packFileJson(JSON.parse(row.recipe), handle);
@@ -103,7 +104,7 @@ export function migrateFileBlobs(handle: Database.Database): void {
                 referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
             }
             handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
-        })();
+        })());
     }
     handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN
         DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`);

src/lib/server/db/index.ts

@@ -11,10 +11,9 @@ import { initializeFleetSettings } from './fleet-settings.js';
 import { migrateServerHistory } from './server-history.js';
 import { migrateNativeInstances } from '../backends/schema.js';
 import { installPerformanceIndexes } from './performance-indexes.js';
-import { migrateVersionLayout } from './version-layout.js';
 import { migrateFileBlobs } from './file-blobs.js';
 import { enableIncrementalVacuum, retentionClocks } from './retention-clocks.js';
-import { tuneConnection } from './connection.js';
+import { migrationStep, tuneConnection, tuneMigrationConnection } from './connection.js';
 import { timeStatement } from './slow-statements.js';
 
 let _db: Database.Database | null = null;
@@ -318,10 +317,11 @@ export function db(): Database.Database {
     // singleton once migrations succeed. A half-migrated handle that's cached
     // silently breaks every subsequent query without a chance to retry.
     try {
+        console.info('[database] startup migrations starting');
         handle.pragma('journal_mode = WAL');
         handle.pragma('auto_vacuum = INCREMENTAL');
         handle.pragma('foreign_keys = ON');
-        tuneConnection(handle);
+        tuneMigrationConnection(handle);
         dropLegacyBetaTables(handle);
         handle.exec(schemaSql);
         migrateProvisioningActions(handle);
@@ -394,8 +394,9 @@ export function db(): Database.Database {
         ]);
         backfillLeaderboard(handle);
         initializeFleetSettings(handle);
-        migrateNativeInstances(handle);
-        migrateVersionLayout(handle);
+        migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
+        // File-blob migration removes the legacy content column altogether.
+        // Reordering it first would unnecessarily copy every artifact twice.
         migrateFileBlobs(handle);
         // Lifecycle history is retained indefinitely. Admission and status reads
         // must narrow by server or active state instead of scanning that history.
@@ -416,16 +417,20 @@ export function db(): Database.Database {
             ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`);
         installPerformanceIndexes(handle);
         installReportRevisions(handle);
-        retentionClocks(handle, {
+        migrationStep('retention clocks and indexes', () => retentionClocks(handle, {
             deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'],
             maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'],
             leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
             betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at']
-        });
+        }));
         // Existing databases require one rebuild to gain the incremental-vacuum page map.
         enableIncrementalVacuum(handle);
         handle.pragma('optimize = 0x10002');
+        handle.pragma('shrink_memory');
+        tuneConnection(handle);
+        console.info('[database] startup migrations complete');
     } catch (err) {
+        console.error('[database] startup migrations failed', err);
         handle.close();
         throw err;
     }

src/lib/server/db/retention-clocks.ts

@@ -1,4 +1,5 @@
 import type Database from 'better-sqlite3';
+import { migrationStep, tuneMigrationConnection } from './connection.js';
 
 /** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */
 export function retentionClocks(handle: Database.Database, clocks: Record<string, string[]>): void {
@@ -31,7 +32,24 @@ export function enableIncrementalVacuum(handle: Database.Database): void {
                     AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`);
         }
         handle.pragma('auto_vacuum = INCREMENTAL');
-        handle.exec('VACUUM');
+        // Guard this operation itself as well as startup: future callers must
+        // never inherit MEMORY temp storage or an oversized rebuild cache.
+        const previous = {
+            temp: handle.pragma('temp_store', { simple: true }),
+            cache: handle.pragma('cache_size', { simple: true }),
+            mmap: handle.pragma('mmap_size', { simple: true }),
+            journal: handle.pragma('journal_size_limit', { simple: true })
+        };
+        try {
+            tuneMigrationConnection(handle);
+            migrationStep('incremental vacuum page map (disk-backed rebuild)', () => handle.exec('VACUUM'));
+        } finally {
+            handle.pragma('shrink_memory');
+            handle.pragma(`temp_store = ${previous.temp}`);
+            handle.pragma(`cache_size = ${previous.cache}`);
+            if (previous.mmap !== undefined) handle.pragma(`mmap_size = ${previous.mmap}`);
+            handle.pragma(`journal_size_limit = ${previous.journal}`);
+        }
     }
     if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) {
         handle.transaction(() => {

src/lib/server/hosts/schema.ts

@@ -2,6 +2,7 @@ import { createHash } from 'node:crypto';
 import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js';
 import { retentionClocks } from '../db/retention-clocks.js';
 import { installHistoryCounts } from './history-counts.js';
+import { migrationStep } from '../db/connection.js';
 import type Database from 'better-sqlite3';
 import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
 
@@ -10,6 +11,11 @@ export function ensureHostSchema(handle: Database.Database): void {
     if (initialized.has(handle)) return;
     // Background readers open only after controller migrations have completed.
     if (handle.readonly) { initialized.add(handle); return; }
+    migrationStep('host schema and saved command files', () => migrateHostSchema(handle));
+    initialized.add(handle);
+}
+
+function migrateHostSchema(handle: Database.Database): void {
     ensureFileBlobSchema(handle);
     handle.exec(`
         CREATE TABLE IF NOT EXISTS ec_hosts (
@@ -189,5 +195,4 @@ export function ensureHostSchema(handle: Database.Database): void {
         DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=NEW.id; END;`);
     retentionClocks(handle, { ec_host_operations: ['created_at', 'updated_at'] });
     installHistoryCounts(handle);
-    initialized.add(handle);
 }

tests/helpers/vacuum-memory.ts

@@ -0,0 +1,23 @@
+import Database from 'better-sqlite3';
+import { enableIncrementalVacuum } from '../../src/lib/server/db/retention-clocks.js';
+
+// Run in a separate process so native SQLite memory, not just V8's heap, is measured.
+const handle = new Database(process.argv[2]);
+try {
+    handle.pragma('journal_mode = WAL');
+    handle.pragma('cache_size = -16384');
+    handle.exec('CREATE TABLE records(id INTEGER PRIMARY KEY, content BLOB)');
+    const insert = handle.prepare('INSERT INTO records(content) VALUES(zeroblob(1048576))');
+    handle.transaction(() => { for (let i = 0; i < 256; i++) insert.run(); })();
+    handle.pragma('wal_checkpoint(TRUNCATE)');
+    handle.pragma('shrink_memory');
+    // Reproduce the dangerous caller settings. The rebuild must override these.
+    handle.pragma('temp_store = MEMORY');
+    handle.pragma('cache_size = -131072');
+    const beforeKiB = process.resourceUsage().maxRSS;
+    enableIncrementalVacuum(handle);
+    console.log(JSON.stringify({ beforeKiB, peakKiB: process.resourceUsage().maxRSS,
+        vacuum: handle.pragma('auto_vacuum', { simple: true }),
+        integrity: handle.pragma('quick_check', { simple: true }),
+        rows: handle.prepare('SELECT COUNT(*) n FROM records').get() }));
+} finally { handle.close(); }

tests/unit/db-statements.test.ts

@@ -15,7 +15,7 @@ describe('prepared statement caching', () => {
         try {
             const handle = db();
             expect(handle.pragma('cache_size', { simple: true })).toBe(-131072);
-            expect(handle.pragma('temp_store', { simple: true })).toBe(2);
+            expect(handle.pragma('temp_store', { simple: true })).toBe(1);
             expect(handle.pragma('journal_size_limit', { simple: true })).toBe(67108864);
             const pragma = vi.spyOn(handle, 'pragma');
             vi.advanceTimersByTime(60 * 60_000);

tests/unit/migration-memory.test.ts

@@ -0,0 +1,75 @@
+import Database from 'better-sqlite3';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { fileURLToPath } from 'node:url';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { enableIncrementalVacuum } from '../../src/lib/server/db/retention-clocks.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); await fixture.cleanup(); });
+
+it('keeps a 256 MiB database rebuild below a 160 MiB increase in resident memory', async () => {
+    const helper = fileURLToPath(new URL('../helpers/vacuum-memory.ts', import.meta.url));
+    const { stdout } = await promisify(execFile)(process.execPath, ['--import', 'tsx', helper, fixture.dbPath], { timeout: 60000 });
+    const result = JSON.parse(stdout.trim().split('\n').at(-1)!);
+    expect(result).toMatchObject({ vacuum: 2, integrity: 'ok', rows: { n: 256 } });
+    expect(result.peakKiB - result.beforeKiB).toBeLessThan(160 * 1024);
+}, 65000);
+
+it.each(['before', 'after'])('recovers a rebuild interrupted %s copying without losing host preparation boundaries', mode => {
+    let handle = new Database(fixture.dbPath);
+    try {
+        handle.exec(`CREATE TABLE ec_host_operations(id TEXT,host_id TEXT);
+            CREATE TABLE ec_host_enable_requests(host_id TEXT,operation_floor INTEGER);
+            INSERT INTO ec_host_operations(rowid,id,host_id) VALUES(10,'before','host'),(30,'after','host');
+            INSERT INTO ec_host_enable_requests VALUES('host',20)`);
+        handle.pragma('temp_store = MEMORY'); handle.pragma('cache_size = -131072');
+        const exec = handle.exec.bind(handle);
+        const mock = vi.spyOn(handle, 'exec').mockImplementation(sql => {
+            if (sql === 'VACUUM') {
+                expect(handle.pragma('temp_store', { simple: true })).toBe(1);
+                expect(handle.pragma('cache_size', { simple: true })).toBe(-16384);
+                if (mode === 'after') exec(sql);
+                throw new Error('simulated interruption');
+            }
+            return exec(sql);
+        });
+        expect(() => enableIncrementalVacuum(handle)).toThrow('simulated interruption');
+        mock.mockRestore();
+        expect(handle.pragma('temp_store', { simple: true })).toBe(2);
+        expect(handle.pragma('cache_size', { simple: true })).toBe(-131072);
+        handle.close(); handle = new Database(fixture.dbPath);
+        enableIncrementalVacuum(handle);
+        expect(handle.pragma('auto_vacuum', { simple: true })).toBe(2);
+        expect(handle.prepare(`SELECT o.id FROM ec_host_operations o JOIN ec_host_enable_requests e ON e.host_id=o.host_id
+            WHERE o.rowid>e.operation_floor`).all()).toEqual([{ id: 'after' }]);
+        expect(handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()).toBeUndefined();
+        expect(handle.pragma('quick_check', { simple: true })).toBe('ok');
+    } finally { handle.close(); }
+});
+
+it('uses migration settings before startup writes and restores runtime tuning only after the rebuild', async () => {
+    const legacy = new Database(fixture.dbPath);
+    legacy.exec('CREATE TABLE legacy(id INTEGER PRIMARY KEY, content BLOB); INSERT INTO legacy(content) VALUES(zeroblob(1024))');
+    legacy.close();
+    vi.stubEnv('EC_SQLITE_MMAP_MB', '256');
+    const exec = Database.prototype.exec;
+    let rebuilt = false, migrated = false;
+    const mock = vi.spyOn(Database.prototype, 'exec').mockImplementation(function(this: Database.Database, sql) {
+        if (sql === 'VACUUM' || sql.includes('CREATE TABLE IF NOT EXISTS file_blobs')) {
+            expect(this.pragma('temp_store', { simple: true })).toBe(1);
+            expect(this.pragma('cache_size', { simple: true })).toBe(-16384);
+            expect(this.pragma('mmap_size', { simple: true })).toBe(0);
+            if (sql === 'VACUUM') rebuilt = true; else migrated = true;
+        }
+        return exec.call(this, sql);
+    });
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const handle = db(); mock.mockRestore();
+    expect(rebuilt && migrated).toBe(true);
+    expect(handle.pragma('temp_store', { simple: true })).toBe(1);
+    expect(handle.pragma('cache_size', { simple: true })).toBe(-131072);
+    expect(handle.pragma('mmap_size', { simple: true })).toBe(256 * 1024 * 1024);
+});