josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Keep startup database rebuilds out of memory

Browse Source
This commit is contained in:
josh
2026-10-06 15:18:13 -04:00
parent 8f8897b86b
commit 6999a33aef
12 changed files with 184 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/db/connection.ts
+25 -1
View File
@@ -2,7 +2,9 @@ import type Database from 'better-sqlite3';
export function tuneConnection(handle: Database.Database, reader = false): void {
handle.pragma(`cache_size = -${reader ? 32768 : 131072}`);
handle.pragma('temp_store = MEMORY');
// MEMORY also puts VACUUM's transient database in RAM. Disk-backed temps
// keep migrations and large sorts from growing with the entire database.
handle.pragma('temp_store = FILE');
// mmap is opt-in because the controller data directory can be a network mount.
const mmapMiB = Number(process.env.EC_SQLITE_MMAP_MB ?? 0);
if (Number.isSafeInteger(mmapMiB) && mmapMiB >= 0 && mmapMiB <= 32768) {
@@ -10,3 +12,25 @@ export function tuneConnection(handle: Database.Database, reader = false): void
}
if (!reader) handle.pragma('journal_size_limit = 67108864');
}
/** Startup needs two page caches during VACUUM, plus buffers for legacy files. */
export function tuneMigrationConnection(handle: Database.Database): void {
handle.pragma('cache_size = -16384');
handle.pragma('temp_store = FILE');
handle.pragma('mmap_size = 0');
handle.pragma('journal_size_limit = 67108864');
}
export function migrationStep<T>(name: string, work: () => T): T {
const started = Date.now();
console.info(`[database] migration starting: ${name}`);
try {
const result = work();
console.info(`[database] migration finished: ${name}`, { elapsedMs: Date.now() - started,
rssMiB: Math.round(process.memoryUsage().rss / 1048576) });
return result;
} catch (error) {
console.error(`[database] migration failed: ${name}`, error);
throw error;
}
}
src/lib/server/db/file-blobs.ts
+5 -4
View File
@@ -1,6 +1,7 @@
import { createHash } from 'node:crypto';
import type Database from 'better-sqlite3';
import { db } from './index.js';
import { migrationStep } from './connection.js';
export function ensureFileBlobSchema(handle: Database.Database): void {
handle.exec(`CREATE TABLE IF NOT EXISTS file_blobs(sha256 TEXT PRIMARY KEY,size INTEGER NOT NULL,content BLOB NOT NULL);
@@ -84,18 +85,18 @@ export function* migrationRows<T>(handle: Database.Database, sql: string): Gener
export function migrateFileBlobs(handle: Database.Database): void {
for (const [table, column] of blobColumns) {
if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue;
handle.transaction(() => {
migrationStep(`shared files: ${table}.${column}`, () => handle.transaction(() => {
handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`);
const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) {
update.run(putFileBlob(row.content, handle), row.rid);
}
handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
})();
})());
}
for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`);
if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) {
handle.transaction(() => {
migrationStep('shared files: launch recipes', () => handle.transaction(() => {
for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle,
'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) {
const packed = packFileJson(JSON.parse(row.recipe), handle);
@@ -103,7 +104,7 @@ export function migrateFileBlobs(handle: Database.Database): void {
referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
}
handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
})();
})());
}
handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN
DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`);
src/lib/server/db/index.ts
+12 -7
View File
@@ -11,10 +11,9 @@ import { initializeFleetSettings } from './fleet-settings.js';
import { migrateServerHistory } from './server-history.js';
import { migrateNativeInstances } from '../backends/schema.js';
import { installPerformanceIndexes } from './performance-indexes.js';
import { migrateVersionLayout } from './version-layout.js';
import { migrateFileBlobs } from './file-blobs.js';
import { enableIncrementalVacuum, retentionClocks } from './retention-clocks.js';
import { tuneConnection } from './connection.js';
import { migrationStep, tuneConnection, tuneMigrationConnection } from './connection.js';
import { timeStatement } from './slow-statements.js';
let _db: Database.Database | null = null;
@@ -318,10 +317,11 @@ export function db(): Database.Database {
// singleton once migrations succeed. A half-migrated handle that's cached
// silently breaks every subsequent query without a chance to retry.
try {
console.info('[database] startup migrations starting');
handle.pragma('journal_mode = WAL');
handle.pragma('auto_vacuum = INCREMENTAL');
handle.pragma('foreign_keys = ON');
tuneConnection(handle);
tuneMigrationConnection(handle);
dropLegacyBetaTables(handle);
handle.exec(schemaSql);
migrateProvisioningActions(handle);
@@ -394,8 +394,9 @@ export function db(): Database.Database {
]);
backfillLeaderboard(handle);
initializeFleetSettings(handle);
migrateNativeInstances(handle);
migrateVersionLayout(handle);
migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
// File-blob migration removes the legacy content column altogether.
// Reordering it first would unnecessarily copy every artifact twice.
migrateFileBlobs(handle);
// Lifecycle history is retained indefinitely. Admission and status reads
// must narrow by server or active state instead of scanning that history.
@@ -416,16 +417,20 @@ export function db(): Database.Database {
ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`);
installPerformanceIndexes(handle);
installReportRevisions(handle);
retentionClocks(handle, {
migrationStep('retention clocks and indexes', () => retentionClocks(handle, {
deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'],
maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'],
leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at']
});
}));
// Existing databases require one rebuild to gain the incremental-vacuum page map.
enableIncrementalVacuum(handle);
handle.pragma('optimize = 0x10002');
handle.pragma('shrink_memory');
tuneConnection(handle);
console.info('[database] startup migrations complete');
} catch (err) {
console.error('[database] startup migrations failed', err);
handle.close();
throw err;
}
src/lib/server/db/retention-clocks.ts
+19 -1
View File
@@ -1,4 +1,5 @@
import type Database from 'better-sqlite3';
import { migrationStep, tuneMigrationConnection } from './connection.js';
/** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */
export function retentionClocks(handle: Database.Database, clocks: Record<string, string[]>): void {
@@ -31,7 +32,24 @@ export function enableIncrementalVacuum(handle: Database.Database): void {
AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`);
}
handle.pragma('auto_vacuum = INCREMENTAL');
handle.exec('VACUUM');
// Guard this operation itself as well as startup: future callers must
// never inherit MEMORY temp storage or an oversized rebuild cache.
const previous = {
temp: handle.pragma('temp_store', { simple: true }),
cache: handle.pragma('cache_size', { simple: true }),
mmap: handle.pragma('mmap_size', { simple: true }),
journal: handle.pragma('journal_size_limit', { simple: true })
};
try {
tuneMigrationConnection(handle);
migrationStep('incremental vacuum page map (disk-backed rebuild)', () => handle.exec('VACUUM'));
} finally {
handle.pragma('shrink_memory');
handle.pragma(`temp_store = ${previous.temp}`);
handle.pragma(`cache_size = ${previous.cache}`);
if (previous.mmap !== undefined) handle.pragma(`mmap_size = ${previous.mmap}`);
handle.pragma(`journal_size_limit = ${previous.journal}`);
}
}
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) {
handle.transaction(() => {
src/lib/server/hosts/schema.ts
+6 -1
View File
@@ -2,6 +2,7 @@ import { createHash } from 'node:crypto';
import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js';
import { retentionClocks } from '../db/retention-clocks.js';
import { installHistoryCounts } from './history-counts.js';
import { migrationStep } from '../db/connection.js';
import type Database from 'better-sqlite3';
import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
@@ -10,6 +11,11 @@ export function ensureHostSchema(handle: Database.Database): void {
if (initialized.has(handle)) return;
// Background readers open only after controller migrations have completed.
if (handle.readonly) { initialized.add(handle); return; }
migrationStep('host schema and saved command files', () => migrateHostSchema(handle));
initialized.add(handle);
}
function migrateHostSchema(handle: Database.Database): void {
ensureFileBlobSchema(handle);
handle.exec(`
CREATE TABLE IF NOT EXISTS ec_hosts (
@@ -189,5 +195,4 @@ export function ensureHostSchema(handle: Database.Database): void {
DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=NEW.id; END;`);
retentionClocks(handle, { ec_host_operations: ['created_at', 'updated_at'] });
installHistoryCounts(handle);
initialized.add(handle);
}