Keep startup database rebuilds out of memory

tests/helpers/vacuum-memory.ts

@@ -0,0 +1,23 @@
+import Database from 'better-sqlite3';
+import { enableIncrementalVacuum } from '../../src/lib/server/db/retention-clocks.js';
+
+// Run in a separate process so native SQLite memory, not just V8's heap, is measured.
+const handle = new Database(process.argv[2]);
+try {
+    handle.pragma('journal_mode = WAL');
+    handle.pragma('cache_size = -16384');
+    handle.exec('CREATE TABLE records(id INTEGER PRIMARY KEY, content BLOB)');
+    const insert = handle.prepare('INSERT INTO records(content) VALUES(zeroblob(1048576))');
+    handle.transaction(() => { for (let i = 0; i < 256; i++) insert.run(); })();
+    handle.pragma('wal_checkpoint(TRUNCATE)');
+    handle.pragma('shrink_memory');
+    // Reproduce the dangerous caller settings. The rebuild must override these.
+    handle.pragma('temp_store = MEMORY');
+    handle.pragma('cache_size = -131072');
+    const beforeKiB = process.resourceUsage().maxRSS;
+    enableIncrementalVacuum(handle);
+    console.log(JSON.stringify({ beforeKiB, peakKiB: process.resourceUsage().maxRSS,
+        vacuum: handle.pragma('auto_vacuum', { simple: true }),
+        integrity: handle.pragma('quick_check', { simple: true }),
+        rows: handle.prepare('SELECT COUNT(*) n FROM records').get() }));
+} finally { handle.close(); }

tests/unit/db-statements.test.ts

@@ -15,7 +15,7 @@ describe('prepared statement caching', () => {
         try {
             const handle = db();
             expect(handle.pragma('cache_size', { simple: true })).toBe(-131072);
-            expect(handle.pragma('temp_store', { simple: true })).toBe(2);
+            expect(handle.pragma('temp_store', { simple: true })).toBe(1);
             expect(handle.pragma('journal_size_limit', { simple: true })).toBe(67108864);
             const pragma = vi.spyOn(handle, 'pragma');
             vi.advanceTimersByTime(60 * 60_000);

tests/unit/migration-memory.test.ts

@@ -0,0 +1,75 @@
+import Database from 'better-sqlite3';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { fileURLToPath } from 'node:url';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { enableIncrementalVacuum } from '../../src/lib/server/db/retention-clocks.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); await fixture.cleanup(); });
+
+it('keeps a 256 MiB database rebuild below a 160 MiB increase in resident memory', async () => {
+    const helper = fileURLToPath(new URL('../helpers/vacuum-memory.ts', import.meta.url));
+    const { stdout } = await promisify(execFile)(process.execPath, ['--import', 'tsx', helper, fixture.dbPath], { timeout: 60000 });
+    const result = JSON.parse(stdout.trim().split('\n').at(-1)!);
+    expect(result).toMatchObject({ vacuum: 2, integrity: 'ok', rows: { n: 256 } });
+    expect(result.peakKiB - result.beforeKiB).toBeLessThan(160 * 1024);
+}, 65000);
+
+it.each(['before', 'after'])('recovers a rebuild interrupted %s copying without losing host preparation boundaries', mode => {
+    let handle = new Database(fixture.dbPath);
+    try {
+        handle.exec(`CREATE TABLE ec_host_operations(id TEXT,host_id TEXT);
+            CREATE TABLE ec_host_enable_requests(host_id TEXT,operation_floor INTEGER);
+            INSERT INTO ec_host_operations(rowid,id,host_id) VALUES(10,'before','host'),(30,'after','host');
+            INSERT INTO ec_host_enable_requests VALUES('host',20)`);
+        handle.pragma('temp_store = MEMORY'); handle.pragma('cache_size = -131072');
+        const exec = handle.exec.bind(handle);
+        const mock = vi.spyOn(handle, 'exec').mockImplementation(sql => {
+            if (sql === 'VACUUM') {
+                expect(handle.pragma('temp_store', { simple: true })).toBe(1);
+                expect(handle.pragma('cache_size', { simple: true })).toBe(-16384);
+                if (mode === 'after') exec(sql);
+                throw new Error('simulated interruption');
+            }
+            return exec(sql);
+        });
+        expect(() => enableIncrementalVacuum(handle)).toThrow('simulated interruption');
+        mock.mockRestore();
+        expect(handle.pragma('temp_store', { simple: true })).toBe(2);
+        expect(handle.pragma('cache_size', { simple: true })).toBe(-131072);
+        handle.close(); handle = new Database(fixture.dbPath);
+        enableIncrementalVacuum(handle);
+        expect(handle.pragma('auto_vacuum', { simple: true })).toBe(2);
+        expect(handle.prepare(`SELECT o.id FROM ec_host_operations o JOIN ec_host_enable_requests e ON e.host_id=o.host_id
+            WHERE o.rowid>e.operation_floor`).all()).toEqual([{ id: 'after' }]);
+        expect(handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()).toBeUndefined();
+        expect(handle.pragma('quick_check', { simple: true })).toBe('ok');
+    } finally { handle.close(); }
+});
+
+it('uses migration settings before startup writes and restores runtime tuning only after the rebuild', async () => {
+    const legacy = new Database(fixture.dbPath);
+    legacy.exec('CREATE TABLE legacy(id INTEGER PRIMARY KEY, content BLOB); INSERT INTO legacy(content) VALUES(zeroblob(1024))');
+    legacy.close();
+    vi.stubEnv('EC_SQLITE_MMAP_MB', '256');
+    const exec = Database.prototype.exec;
+    let rebuilt = false, migrated = false;
+    const mock = vi.spyOn(Database.prototype, 'exec').mockImplementation(function(this: Database.Database, sql) {
+        if (sql === 'VACUUM' || sql.includes('CREATE TABLE IF NOT EXISTS file_blobs')) {
+            expect(this.pragma('temp_store', { simple: true })).toBe(1);
+            expect(this.pragma('cache_size', { simple: true })).toBe(-16384);
+            expect(this.pragma('mmap_size', { simple: true })).toBe(0);
+            if (sql === 'VACUUM') rebuilt = true; else migrated = true;
+        }
+        return exec.call(this, sql);
+    });
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const handle = db(); mock.mockRestore();
+    expect(rebuilt && migrated).toBe(true);
+    expect(handle.pragma('temp_store', { simple: true })).toBe(1);
+    expect(handle.pragma('cache_size', { simple: true })).toBe(-131072);
+    expect(handle.pragma('mmap_size', { simple: true })).toBe(256 * 1024 * 1024);
+});