From 6999a33aef5943e7c5ee0cf2f19e88cf38bb02c2 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Tue, 6 Oct 2026 15:18:13 -0400 Subject: [PATCH] Keep startup database rebuilds out of memory --- Dockerfile | 2 + README.md | 9 ++++ docs/settings.md | 3 ++ entrypoint.sh | 4 ++ src/lib/server/db/connection.ts | 26 +++++++++- src/lib/server/db/file-blobs.ts | 9 ++-- src/lib/server/db/index.ts | 19 ++++--- src/lib/server/db/retention-clocks.ts | 20 ++++++- src/lib/server/hosts/schema.ts | 7 ++- tests/helpers/vacuum-memory.ts | 23 ++++++++ tests/unit/db-statements.test.ts | 2 +- tests/unit/migration-memory.test.ts | 75 +++++++++++++++++++++++++++ 12 files changed, 184 insertions(+), 15 deletions(-) create mode 100644 tests/helpers/vacuum-memory.ts create mode 100644 tests/unit/migration-memory.test.ts diff --git a/Dockerfile b/Dockerfile index 769bbb2..de06592 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ COPY --from=build /app/node_modules ./node_modules COPY --from=build /app/package.json ./package.json COPY --from=build /app/src/lib/server/db/schema.sql ./build/schema.sql COPY scripts/reset-admin.mjs ./scripts/reset-admin.mjs +COPY scripts/database-diagnostics.mjs ./scripts/database-diagnostics.mjs COPY src/lib/server/auth/passwords.js ./src/lib/server/auth/passwords.js COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh @@ -32,6 +33,7 @@ ENV NODE_ENV=production ENV PORT=3000 ENV HOST=0.0.0.0 ENV BODY_SIZE_LIMIT=Infinity +ENV SQLITE_TMPDIR=/data/.sqlite-tmp EXPOSE 3000 ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"] diff --git a/README.md b/README.md index 7eabc58..b5961be 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,15 @@ after database changes, capture these timings under the same fleet load and run the largest tables/indexes, database and WAL sizes, and free pages. This command opens an existing database read-only and does not run app migrations. The controller uses a 128 MiB page cache and each reader worker uses 32 MiB. +Main-database migrations use a 16 MiB cache with memory mapping disabled; SQLite +temporary storage stays disk-backed during migrations and normal operation. +The container uses `/data/.sqlite-tmp` for SQLite scratch files so a memory-backed +`/tmp` cannot turn a database rebuild into an allocation of the whole database. +Override `SQLITE_TMPDIR` only with another writable, disk-backed directory. +Startup logs identify migration phases, elapsed time and resident memory. A +completed migration is retained across restarts; SQLite rolls back interrupted +transactions and startup retries those migrations. The one-time vacuum still needs disk space for its temporary +copy. Do not remove `cherry.db-wal` to recover from an interrupted upgrade. For a local data disk, set `EC_SQLITE_MMAP_MB=256` to enable memory-mapped reads. Query statistics are optimized at startup and hourly; WAL reuse is capped at 64 MiB after checkpoints (active transactions can temporarily grow it larger). diff --git a/docs/settings.md b/docs/settings.md index 924548b..64e22c5 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -93,6 +93,9 @@ available longer. Previously saved retention choices are preserved on upgrade. The database upgrade normalizes retention clocks to UTC and enables incremental vacuum. An existing database without a vacuum page map is rebuilt once at startup; allow time for this rebuild and free disk space up to twice the database size. +The rebuild uses disk-backed temporary storage and a 16 MiB page cache instead +of holding a temporary copy of the database in RAM. Container scratch files use +`/data/.sqlite-tmp`; `[database]` startup logs identify the current migration. Normal cleanup releases unused file blobs in bounded batches and incrementally returns free pages to disk after larger prunes. diff --git a/entrypoint.sh b/entrypoint.sh index 69eebc5..3463b88 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,5 +1,9 @@ #!/bin/sh set -eu +if [ -n "${SQLITE_TMPDIR:-}" ]; then + mkdir -p "$SQLITE_TMPDIR" +fi + echo "[entrypoint] starting node server on ${HOST:-0.0.0.0}:${PORT:-3000}" exec node build/index.js diff --git a/src/lib/server/db/connection.ts b/src/lib/server/db/connection.ts index 24d8c1b..f7da6fe 100644 --- a/src/lib/server/db/connection.ts +++ b/src/lib/server/db/connection.ts @@ -2,7 +2,9 @@ import type Database from 'better-sqlite3'; export function tuneConnection(handle: Database.Database, reader = false): void { handle.pragma(`cache_size = -${reader ? 32768 : 131072}`); - handle.pragma('temp_store = MEMORY'); + // MEMORY also puts VACUUM's transient database in RAM. Disk-backed temps + // keep migrations and large sorts from growing with the entire database. + handle.pragma('temp_store = FILE'); // mmap is opt-in because the controller data directory can be a network mount. const mmapMiB = Number(process.env.EC_SQLITE_MMAP_MB ?? 0); if (Number.isSafeInteger(mmapMiB) && mmapMiB >= 0 && mmapMiB <= 32768) { @@ -10,3 +12,25 @@ export function tuneConnection(handle: Database.Database, reader = false): void } if (!reader) handle.pragma('journal_size_limit = 67108864'); } + +/** Startup needs two page caches during VACUUM, plus buffers for legacy files. */ +export function tuneMigrationConnection(handle: Database.Database): void { + handle.pragma('cache_size = -16384'); + handle.pragma('temp_store = FILE'); + handle.pragma('mmap_size = 0'); + handle.pragma('journal_size_limit = 67108864'); +} + +export function migrationStep(name: string, work: () => T): T { + const started = Date.now(); + console.info(`[database] migration starting: ${name}`); + try { + const result = work(); + console.info(`[database] migration finished: ${name}`, { elapsedMs: Date.now() - started, + rssMiB: Math.round(process.memoryUsage().rss / 1048576) }); + return result; + } catch (error) { + console.error(`[database] migration failed: ${name}`, error); + throw error; + } +} diff --git a/src/lib/server/db/file-blobs.ts b/src/lib/server/db/file-blobs.ts index 2116474..afe1f8a 100644 --- a/src/lib/server/db/file-blobs.ts +++ b/src/lib/server/db/file-blobs.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import type Database from 'better-sqlite3'; import { db } from './index.js'; +import { migrationStep } from './connection.js'; export function ensureFileBlobSchema(handle: Database.Database): void { handle.exec(`CREATE TABLE IF NOT EXISTS file_blobs(sha256 TEXT PRIMARY KEY,size INTEGER NOT NULL,content BLOB NOT NULL); @@ -84,18 +85,18 @@ export function* migrationRows(handle: Database.Database, sql: string): Gener export function migrateFileBlobs(handle: Database.Database): void { for (const [table, column] of blobColumns) { if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue; - handle.transaction(() => { + migrationStep(`shared files: ${table}.${column}`, () => handle.transaction(() => { handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`); const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`); for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) { update.run(putFileBlob(row.content, handle), row.rid); } handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`); - })(); + })()); } for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`); if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) { - handle.transaction(() => { + migrationStep('shared files: launch recipes', () => handle.transaction(() => { for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle, 'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) { const packed = packFileJson(JSON.parse(row.recipe), handle); @@ -103,7 +104,7 @@ export function migrateFileBlobs(handle: Database.Database): void { referenceBlobs('recipe', row.deployment_id, packed.hashes, handle); } handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')"); - })(); + })()); } handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`); diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 05e489e..3a44a4d 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -11,10 +11,9 @@ import { initializeFleetSettings } from './fleet-settings.js'; import { migrateServerHistory } from './server-history.js'; import { migrateNativeInstances } from '../backends/schema.js'; import { installPerformanceIndexes } from './performance-indexes.js'; -import { migrateVersionLayout } from './version-layout.js'; import { migrateFileBlobs } from './file-blobs.js'; import { enableIncrementalVacuum, retentionClocks } from './retention-clocks.js'; -import { tuneConnection } from './connection.js'; +import { migrationStep, tuneConnection, tuneMigrationConnection } from './connection.js'; import { timeStatement } from './slow-statements.js'; let _db: Database.Database | null = null; @@ -318,10 +317,11 @@ export function db(): Database.Database { // singleton once migrations succeed. A half-migrated handle that's cached // silently breaks every subsequent query without a chance to retry. try { + console.info('[database] startup migrations starting'); handle.pragma('journal_mode = WAL'); handle.pragma('auto_vacuum = INCREMENTAL'); handle.pragma('foreign_keys = ON'); - tuneConnection(handle); + tuneMigrationConnection(handle); dropLegacyBetaTables(handle); handle.exec(schemaSql); migrateProvisioningActions(handle); @@ -394,8 +394,9 @@ export function db(): Database.Database { ]); backfillLeaderboard(handle); initializeFleetSettings(handle); - migrateNativeInstances(handle); - migrateVersionLayout(handle); + migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle)); + // File-blob migration removes the legacy content column altogether. + // Reordering it first would unnecessarily copy every artifact twice. migrateFileBlobs(handle); // Lifecycle history is retained indefinitely. Admission and status reads // must narrow by server or active state instead of scanning that history. @@ -416,16 +417,20 @@ export function db(): Database.Database { ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`); installPerformanceIndexes(handle); installReportRevisions(handle); - retentionClocks(handle, { + migrationStep('retention clocks and indexes', () => retentionClocks(handle, { deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'], maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'], leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'], betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at'] - }); + })); // Existing databases require one rebuild to gain the incremental-vacuum page map. enableIncrementalVacuum(handle); handle.pragma('optimize = 0x10002'); + handle.pragma('shrink_memory'); + tuneConnection(handle); + console.info('[database] startup migrations complete'); } catch (err) { + console.error('[database] startup migrations failed', err); handle.close(); throw err; } diff --git a/src/lib/server/db/retention-clocks.ts b/src/lib/server/db/retention-clocks.ts index 65e97cd..203c1d2 100644 --- a/src/lib/server/db/retention-clocks.ts +++ b/src/lib/server/db/retention-clocks.ts @@ -1,4 +1,5 @@ import type Database from 'better-sqlite3'; +import { migrationStep, tuneMigrationConnection } from './connection.js'; /** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */ export function retentionClocks(handle: Database.Database, clocks: Record): void { @@ -31,7 +32,24 @@ export function enableIncrementalVacuum(handle: Database.Database): void { AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`); } handle.pragma('auto_vacuum = INCREMENTAL'); - handle.exec('VACUUM'); + // Guard this operation itself as well as startup: future callers must + // never inherit MEMORY temp storage or an oversized rebuild cache. + const previous = { + temp: handle.pragma('temp_store', { simple: true }), + cache: handle.pragma('cache_size', { simple: true }), + mmap: handle.pragma('mmap_size', { simple: true }), + journal: handle.pragma('journal_size_limit', { simple: true }) + }; + try { + tuneMigrationConnection(handle); + migrationStep('incremental vacuum page map (disk-backed rebuild)', () => handle.exec('VACUUM')); + } finally { + handle.pragma('shrink_memory'); + handle.pragma(`temp_store = ${previous.temp}`); + handle.pragma(`cache_size = ${previous.cache}`); + if (previous.mmap !== undefined) handle.pragma(`mmap_size = ${previous.mmap}`); + handle.pragma(`journal_size_limit = ${previous.journal}`); + } } if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) { handle.transaction(() => { diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts index a00edbd..f359d0e 100644 --- a/src/lib/server/hosts/schema.ts +++ b/src/lib/server/hosts/schema.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js'; import { retentionClocks } from '../db/retention-clocks.js'; import { installHistoryCounts } from './history-counts.js'; +import { migrationStep } from '../db/connection.js'; import type Database from 'better-sqlite3'; import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js'; @@ -10,6 +11,11 @@ export function ensureHostSchema(handle: Database.Database): void { if (initialized.has(handle)) return; // Background readers open only after controller migrations have completed. if (handle.readonly) { initialized.add(handle); return; } + migrationStep('host schema and saved command files', () => migrateHostSchema(handle)); + initialized.add(handle); +} + +function migrateHostSchema(handle: Database.Database): void { ensureFileBlobSchema(handle); handle.exec(` CREATE TABLE IF NOT EXISTS ec_hosts ( @@ -189,5 +195,4 @@ export function ensureHostSchema(handle: Database.Database): void { DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=NEW.id; END;`); retentionClocks(handle, { ec_host_operations: ['created_at', 'updated_at'] }); installHistoryCounts(handle); - initialized.add(handle); } diff --git a/tests/helpers/vacuum-memory.ts b/tests/helpers/vacuum-memory.ts new file mode 100644 index 0000000..fdba825 --- /dev/null +++ b/tests/helpers/vacuum-memory.ts @@ -0,0 +1,23 @@ +import Database from 'better-sqlite3'; +import { enableIncrementalVacuum } from '../../src/lib/server/db/retention-clocks.js'; + +// Run in a separate process so native SQLite memory, not just V8's heap, is measured. +const handle = new Database(process.argv[2]); +try { + handle.pragma('journal_mode = WAL'); + handle.pragma('cache_size = -16384'); + handle.exec('CREATE TABLE records(id INTEGER PRIMARY KEY, content BLOB)'); + const insert = handle.prepare('INSERT INTO records(content) VALUES(zeroblob(1048576))'); + handle.transaction(() => { for (let i = 0; i < 256; i++) insert.run(); })(); + handle.pragma('wal_checkpoint(TRUNCATE)'); + handle.pragma('shrink_memory'); + // Reproduce the dangerous caller settings. The rebuild must override these. + handle.pragma('temp_store = MEMORY'); + handle.pragma('cache_size = -131072'); + const beforeKiB = process.resourceUsage().maxRSS; + enableIncrementalVacuum(handle); + console.log(JSON.stringify({ beforeKiB, peakKiB: process.resourceUsage().maxRSS, + vacuum: handle.pragma('auto_vacuum', { simple: true }), + integrity: handle.pragma('quick_check', { simple: true }), + rows: handle.prepare('SELECT COUNT(*) n FROM records').get() })); +} finally { handle.close(); } diff --git a/tests/unit/db-statements.test.ts b/tests/unit/db-statements.test.ts index e377f75..31382d8 100644 --- a/tests/unit/db-statements.test.ts +++ b/tests/unit/db-statements.test.ts @@ -15,7 +15,7 @@ describe('prepared statement caching', () => { try { const handle = db(); expect(handle.pragma('cache_size', { simple: true })).toBe(-131072); - expect(handle.pragma('temp_store', { simple: true })).toBe(2); + expect(handle.pragma('temp_store', { simple: true })).toBe(1); expect(handle.pragma('journal_size_limit', { simple: true })).toBe(67108864); const pragma = vi.spyOn(handle, 'pragma'); vi.advanceTimersByTime(60 * 60_000); diff --git a/tests/unit/migration-memory.test.ts b/tests/unit/migration-memory.test.ts new file mode 100644 index 0000000..6ad3921 --- /dev/null +++ b/tests/unit/migration-memory.test.ts @@ -0,0 +1,75 @@ +import Database from 'better-sqlite3'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { enableIncrementalVacuum } from '../../src/lib/server/db/retention-clocks.js'; + +let fixture: Fixture; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); await fixture.cleanup(); }); + +it('keeps a 256 MiB database rebuild below a 160 MiB increase in resident memory', async () => { + const helper = fileURLToPath(new URL('../helpers/vacuum-memory.ts', import.meta.url)); + const { stdout } = await promisify(execFile)(process.execPath, ['--import', 'tsx', helper, fixture.dbPath], { timeout: 60000 }); + const result = JSON.parse(stdout.trim().split('\n').at(-1)!); + expect(result).toMatchObject({ vacuum: 2, integrity: 'ok', rows: { n: 256 } }); + expect(result.peakKiB - result.beforeKiB).toBeLessThan(160 * 1024); +}, 65000); + +it.each(['before', 'after'])('recovers a rebuild interrupted %s copying without losing host preparation boundaries', mode => { + let handle = new Database(fixture.dbPath); + try { + handle.exec(`CREATE TABLE ec_host_operations(id TEXT,host_id TEXT); + CREATE TABLE ec_host_enable_requests(host_id TEXT,operation_floor INTEGER); + INSERT INTO ec_host_operations(rowid,id,host_id) VALUES(10,'before','host'),(30,'after','host'); + INSERT INTO ec_host_enable_requests VALUES('host',20)`); + handle.pragma('temp_store = MEMORY'); handle.pragma('cache_size = -131072'); + const exec = handle.exec.bind(handle); + const mock = vi.spyOn(handle, 'exec').mockImplementation(sql => { + if (sql === 'VACUUM') { + expect(handle.pragma('temp_store', { simple: true })).toBe(1); + expect(handle.pragma('cache_size', { simple: true })).toBe(-16384); + if (mode === 'after') exec(sql); + throw new Error('simulated interruption'); + } + return exec(sql); + }); + expect(() => enableIncrementalVacuum(handle)).toThrow('simulated interruption'); + mock.mockRestore(); + expect(handle.pragma('temp_store', { simple: true })).toBe(2); + expect(handle.pragma('cache_size', { simple: true })).toBe(-131072); + handle.close(); handle = new Database(fixture.dbPath); + enableIncrementalVacuum(handle); + expect(handle.pragma('auto_vacuum', { simple: true })).toBe(2); + expect(handle.prepare(`SELECT o.id FROM ec_host_operations o JOIN ec_host_enable_requests e ON e.host_id=o.host_id + WHERE o.rowid>e.operation_floor`).all()).toEqual([{ id: 'after' }]); + expect(handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()).toBeUndefined(); + expect(handle.pragma('quick_check', { simple: true })).toBe('ok'); + } finally { handle.close(); } +}); + +it('uses migration settings before startup writes and restores runtime tuning only after the rebuild', async () => { + const legacy = new Database(fixture.dbPath); + legacy.exec('CREATE TABLE legacy(id INTEGER PRIMARY KEY, content BLOB); INSERT INTO legacy(content) VALUES(zeroblob(1024))'); + legacy.close(); + vi.stubEnv('EC_SQLITE_MMAP_MB', '256'); + const exec = Database.prototype.exec; + let rebuilt = false, migrated = false; + const mock = vi.spyOn(Database.prototype, 'exec').mockImplementation(function(this: Database.Database, sql) { + if (sql === 'VACUUM' || sql.includes('CREATE TABLE IF NOT EXISTS file_blobs')) { + expect(this.pragma('temp_store', { simple: true })).toBe(1); + expect(this.pragma('cache_size', { simple: true })).toBe(-16384); + expect(this.pragma('mmap_size', { simple: true })).toBe(0); + if (sql === 'VACUUM') rebuilt = true; else migrated = true; + } + return exec.call(this, sql); + }); + const { db } = await import('../../src/lib/server/db/index.js'); + const handle = db(); mock.mockRestore(); + expect(rebuilt && migrated).toBe(true); + expect(handle.pragma('temp_store', { simple: true })).toBe(1); + expect(handle.pragma('cache_size', { simple: true })).toBe(-131072); + expect(handle.pragma('mmap_size', { simple: true })).toBe(256 * 1024 * 1024); +});