Compare commits

...
2 Commits
Author SHA1 Message Date
josh bd16737dd8 Remove shared prepared statement cache (Refs #76)
CI / checks (push) Successful in 4m40s
2026-10-09 17:05:06 -04:00
josh 4c1f2cdb8f Speed up large buffer assertions (Refs #101) 2026-10-09 16:58:41 -04:00
5 changed files with 71 additions and 30 deletions
+8 -14
View File
@@ -645,21 +645,15 @@ export function db(): Database.Database {
throw err; throw err;
} }
// Cache only runtime queries, after migrations have finished. Statements // Instrument runtime queries after migrations. Each prepare must return a
// belong to this connection, so reopening the DB starts with an empty cache. // fresh statement: modes, permanent bindings, and active iterators are mutable
// Callers share statement options (e.g. raw()); use consistent options for // state owned by the caller, even when another caller uses identical SQL.
// identical SQL and pass values to run/get/all rather than binding permanently.
const statements = new Map<string, Database.Statement>();
const slowMs = Number(process.env.EC_SLOW_SQL_MS ?? 0); const slowMs = Number(process.env.EC_SLOW_SQL_MS ?? 0);
const prepare = handle.prepare.bind(handle); if (Number.isFinite(slowMs) && slowMs > 0) {
handle.prepare = ((sql: string) => { const prepare = handle.prepare.bind(handle);
let statement = statements.get(sql); handle.prepare = ((sql: string) =>
if (!statement) { timeStatement(prepare(sql), sql, slowMs)) as typeof handle.prepare;
statement = timeStatement(prepare(sql), sql, slowMs); }
statements.set(sql, statement);
}
return statement;
}) as typeof handle.prepare;
_db = handle; _db = handle;
optimizeTimer = setInterval(() => { optimizeTimer = setInterval(() => {
+56 -5
View File
@@ -8,9 +8,10 @@ beforeEach(async () => {
}); });
afterEach(async () => { afterEach(async () => {
await fixture.cleanup(); await fixture.cleanup();
vi.unstubAllEnvs();
}); });
describe('prepared statement caching', () => { describe('database statements', () => {
it('tunes the controller connection and stops periodic optimization when closed', async () => { it('tunes the controller connection and stops periodic optimization when closed', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const { db, closeDb } = await import('../../src/lib/server/db/index.js');
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] }); vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
@@ -30,7 +31,7 @@ describe('prepared statement caching', () => {
vi.useRealTimers(); vi.useRealTimers();
} }
}); });
it('reuses statements with fresh parameters and remains usable after rollback', async () => { it('keeps caller-owned statements reusable with fresh parameters after rollback', async () => {
const { db } = await import('../../src/lib/server/db/index.js'); const { db } = await import('../../src/lib/server/db/index.js');
const insertSql = 'INSERT INTO settings (key, value) VALUES (?, ?)'; const insertSql = 'INSERT INTO settings (key, value) VALUES (?, ?)';
const selectSql = 'SELECT value FROM settings WHERE key = ?'; const selectSql = 'SELECT value FROM settings WHERE key = ?';
@@ -45,8 +46,8 @@ describe('prepared statement caching', () => {
).toThrow('rollback'); ).toThrow('rollback');
db().prepare(insertSql).run('cache-second', 'two'); db().prepare(insertSql).run('cache-second', 'two');
expect(db().prepare(insertSql)).toBe(insert); expect(db().prepare(insertSql)).not.toBe(insert);
expect(db().prepare(selectSql)).toBe(select); expect(db().prepare(selectSql)).not.toBe(select);
expect(select.get('cache-first')).toEqual({ value: 'one' }); expect(select.get('cache-first')).toEqual({ value: 'one' });
expect(select.get('cache-second')).toEqual({ value: 'two' }); expect(select.get('cache-second')).toEqual({ value: 'two' });
expect(select.get('cache-rollback')).toBeUndefined(); expect(select.get('cache-rollback')).toBeUndefined();
@@ -65,6 +66,56 @@ describe('prepared statement caching', () => {
expect(statement).not.toBe(oldStatement); expect(statement).not.toBe(oldStatement);
expect(statement.database).not.toBe(oldStatement.database); expect(statement.database).not.toBe(oldStatement.database);
expect(statement.get('cache-persisted')).toEqual({ value: 'saved' }); expect(statement.get('cache-persisted')).toEqual({ value: 'saved' });
expect(db().prepare(sql)).toBe(statement); expect(db().prepare(sql)).not.toBe(statement);
});
describe.each([0, 20])('statement isolation with EC_SLOW_SQL_MS=%i', (slowMs) => {
beforeEach(() => vi.stubEnv('EC_SLOW_SQL_MS', String(slowMs)));
it.each([
['raw', [1]],
['pluck', 1],
['expand', { $: { value: 1 } }],
['safeIntegers', { value: 1n }]
] as const)('keeps %s mode local to its statement', async (mode, expected) => {
const { db } = await import('../../src/lib/server/db/index.js');
const sql = 'SELECT 1 AS value';
const customized = db().prepare(sql)[mode]();
const standard = db().prepare(sql);
expect(standard.get()).toEqual({ value: 1 });
expect(customized.get()).toEqual(expected);
customized[mode](false);
standard[mode]();
expect(customized.get()).toEqual({ value: 1 });
expect(standard.get()).toEqual(expected);
expect(db().prepare(sql).get()).toEqual({ value: 1 });
});
it('keeps permanent bindings local to their statement', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const sql = 'SELECT ? AS value';
const bound = db().prepare(sql).bind('first');
expect(db().prepare(sql).get('second')).toEqual({ value: 'second' });
expect(bound.get()).toEqual({ value: 'first' });
expect(db().prepare(sql).bind('third').get()).toEqual({ value: 'third' });
expect(bound.get()).toEqual({ value: 'first' });
});
it('allows identical queries while another statement is being iterated', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const sql = 'SELECT 1 AS value UNION ALL SELECT 2 AS value';
const iterator = db().prepare(sql).iterate();
try {
expect(iterator.next()).toEqual({ value: { value: 1 }, done: false });
expect(db().prepare(sql).get()).toEqual({ value: 1 });
expect([...db().prepare(sql).iterate()]).toEqual([{ value: 1 }, { value: 2 }]);
expect(iterator.next()).toEqual({ value: { value: 2 }, done: false });
expect(iterator.next().done).toBe(true);
} finally {
iterator.return?.();
}
});
}); });
}); });
+2 -2
View File
@@ -77,8 +77,8 @@ it('shares bytes across versions, staged copies and commands, and scopes downloa
expect( expect(
Buffer.from( Buffer.from(
await GET({ request: request(credential), params: { sha256: hash } }).arrayBuffer() await GET({ request: request(credential), params: { sha256: hash } }).arrayBuffer()
) ).equals(content)
).toEqual(content); ).toBe(true);
expect(() => GET({ request: request(otherCredential), params: { sha256: hash } })).toThrow(); expect(() => GET({ request: request(otherCredential), params: { sha256: hash } })).toThrow();
closeDb(); closeDb();
expect(hosts.getHostOperation(op.id)?.payload).toEqual(payload); expect(hosts.getHostOperation(op.id)?.payload).toEqual(payload);
@@ -328,9 +328,9 @@ describe('routine host observation retention', () => {
).toEqual({ count: 2000 }); ).toEqual({ count: 2000 });
}); });
it('reuses the same three delete statements across different batch sizes', async () => { it('uses the same three delete query shapes across different batch sizes', async () => {
const api = await setup(); const api = await setup();
const statements = new Map<string, ReturnType<typeof api.handle.prepare>>(); const deleteQueries = new Set<string>();
for (const count of [1, 2, 7, 50, 500]) { for (const count of [1, 2, 7, 50, 500]) {
api.handle.transaction(() => { api.handle.transaction(() => {
const insert = api.handle.prepare(`INSERT INTO ec_host_operations const insert = api.handle.prepare(`INSERT INTO ec_host_operations
@@ -346,13 +346,9 @@ describe('routine host observation retention', () => {
.filter((sql) => sql.startsWith('DELETE FROM ec_host_')); .filter((sql) => sql.startsWith('DELETE FROM ec_host_'));
prepare.mockRestore(); prepare.mockRestore();
expect(deletes).toHaveLength(3); expect(deletes).toHaveLength(3);
for (const sql of deletes) { for (const sql of deletes) deleteQueries.add(sql);
const statement = api.handle.prepare(sql);
if (statements.has(sql)) expect(statement).toBe(statements.get(sql));
statements.set(sql, statement);
}
} }
expect(statements.size).toBe(3); expect(deleteQueries.size).toBe(3);
expect(api.handle.pragma('foreign_key_check')).toEqual([]); expect(api.handle.pragma('foreign_key_check')).toEqual([]);
}); });
+1 -1
View File
@@ -28,7 +28,7 @@ it('returns metadata without plugin bytes and lists many artifacts in two querie
targetRelPath: 'plugin.dll', targetRelPath: 'plugin.dll',
content content
}).version.id; }).version.id;
expect(getVersionContent(id)).toEqual(content); expect(getVersionContent(id)?.equals(content)).toBe(true);
expect(getVersion(id)).not.toHaveProperty('content'); expect(getVersion(id)).not.toHaveProperty('content');
expect(getLatestVersion(getVersion(id)!.script_id)).not.toHaveProperty('content'); expect(getLatestVersion(getVersion(id)!.script_id)).not.toHaveProperty('content');
expect(listVersions(getVersion(id)!.script_id)[0]).not.toHaveProperty('content'); expect(listVersions(getVersion(id)!.script_id)[0]).not.toHaveProperty('content');