From 703d414c7a0cd660aaa9dd67ad85e5b4f4b8480c Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Tue, 6 Oct 2026 14:53:51 -0400 Subject: [PATCH] Maintain scoped operation history totals without table scans --- src/lib/server/hosts/history-counts.ts | 42 +++++++++++++++++++ src/lib/server/hosts/index.ts | 4 +- src/lib/server/hosts/schema.ts | 2 + tests/unit/host-observation-retention.test.ts | 3 ++ tests/unit/host-operation-history.test.ts | 39 +++++++++++++++++ 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 src/lib/server/hosts/history-counts.ts diff --git a/src/lib/server/hosts/history-counts.ts b/src/lib/server/hosts/history-counts.ts new file mode 100644 index 0000000..a68407a --- /dev/null +++ b/src/lib/server/hosts/history-counts.ts @@ -0,0 +1,42 @@ +import type Database from 'better-sqlite3'; +import { importantOperationSql } from './observations.js'; + +const important = (alias: string) => importantOperationSql.replaceAll('o.', `${alias}.`); +const visible = (alias: string) => `${alias}.history_expired_at IS NULL`; + +/** Durable totals for global, host, server, and host/server filters; receipts update only changed membership. */ +export function installHistoryCounts(handle: Database.Database): void { + if (handle.prepare("SELECT 1 FROM ec_host_schema_migrations WHERE name='history-counts-v1'").get()) return; + handle.transaction(() => { + handle.exec(`CREATE TABLE ec_host_operation_history_counts ( + host_id TEXT NOT NULL, server_name TEXT NOT NULL, total INTEGER NOT NULL, + routine INTEGER NOT NULL, important INTEGER NOT NULL, PRIMARY KEY(host_id,server_name) + ) WITHOUT ROWID; + DROP INDEX ec_host_operations_important_history; + CREATE INDEX ec_host_operations_important_history ON ec_host_operations(created_at) + WHERE history_expired_at IS NULL AND ${importantOperationSql.replaceAll('o.', '')};`); + for (const [host, server] of [['', ''], ['o.host_id', ''], ['', 'o.server_name'], ['o.host_id', 'o.server_name']]) { + const groups = [host, server].filter(Boolean); + handle.exec(`INSERT INTO ec_host_operation_history_counts + SELECT ${host || "''"},${server || "''"},COUNT(*),COALESCE(SUM(o.routine=1),0),COALESCE(SUM(${important('o')}),0) + FROM ec_host_operations o WHERE ${visible('o')} ${server ? 'AND o.server_name IS NOT NULL' : ''} + ${groups.length ? 'GROUP BY ' + groups.join(',') : ''}`); + } + const adjust = (alias: string, direction: 1 | -1) => ` + INSERT INTO ec_host_operation_history_counts(host_id,server_name,total,routine,important) + SELECT host_id,server_name,${direction},${direction}*(${alias}.routine=1),${direction}*${important(alias)} FROM ( + SELECT '' host_id,'' server_name UNION ALL SELECT ${alias}.host_id,'' + UNION ALL SELECT '',${alias}.server_name WHERE ${alias}.server_name IS NOT NULL + UNION ALL SELECT ${alias}.host_id,${alias}.server_name WHERE ${alias}.server_name IS NOT NULL + ) WHERE ${visible(alias)} + ON CONFLICT(host_id,server_name) DO UPDATE SET total=total+excluded.total, + routine=routine+excluded.routine,important=important+excluded.important;`; + handle.exec(`CREATE TRIGGER operation_history_count_insert AFTER INSERT ON ec_host_operations BEGIN ${adjust('NEW', 1)} END; + CREATE TRIGGER operation_history_count_delete AFTER DELETE ON ec_host_operations BEGIN ${adjust('OLD', -1)} END; + CREATE TRIGGER operation_history_count_update AFTER UPDATE OF host_id,server_name,routine,state,observation_success,error,history_expired_at ON ec_host_operations + WHEN OLD.host_id IS NOT NEW.host_id OR OLD.server_name IS NOT NEW.server_name OR OLD.routine IS NOT NEW.routine + OR (${visible('OLD')}) IS NOT (${visible('NEW')}) OR ${important('OLD')} IS NOT ${important('NEW')} + BEGIN ${adjust('OLD', -1)} ${adjust('NEW', 1)} END;`); + handle.prepare("INSERT INTO ec_host_schema_migrations(name) VALUES('history-counts-v1')").run(); + })(); +} diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index 880fe98..b6b437e 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -426,7 +426,9 @@ export function listHostOperationHistory(page = 1, filter: HostOperationFilter = if (scope.server) { conditions.push('o.server_name=?'); parameters.push(scope.server); } const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : ''; return handle.transaction((): HostOperationHistoryPage => { - const { total } = handle.prepare(`SELECT COUNT(*) AS total FROM ec_host_operations o ${where}`).get(...parameters) as { total: number }; + const countColumn = filter === 'all' ? 'total' : filter; + const total = (handle.prepare(`SELECT ${countColumn} AS total FROM ec_host_operation_history_counts WHERE host_id=? AND server_name=?`) + .get(scope.host ?? '', scope.server ?? '') as { total: number } | undefined)?.total ?? 0; const totalPages = Math.max(1, Math.ceil(total / pageSize)); const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1); const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind, diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts index 1865a0d..a00edbd 100644 --- a/src/lib/server/hosts/schema.ts +++ b/src/lib/server/hosts/schema.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js'; import { retentionClocks } from '../db/retention-clocks.js'; +import { installHistoryCounts } from './history-counts.js'; import type Database from 'better-sqlite3'; import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js'; @@ -187,5 +188,6 @@ export function ensureHostSchema(handle: Database.Database): void { WHEN NEW.history_expired_at IS NOT NULL BEGIN DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=NEW.id; END;`); retentionClocks(handle, { ec_host_operations: ['created_at', 'updated_at'] }); + installHistoryCounts(handle); initialized.add(handle); } diff --git a/tests/unit/host-observation-retention.test.ts b/tests/unit/host-observation-retention.test.ts index a93a131..07b9c18 100644 --- a/tests/unit/host-observation-retention.test.ts +++ b/tests/unit/host-observation-retention.test.ts @@ -179,6 +179,9 @@ describe('routine host observation retention', () => { retained.push(malformed); api.handle.exec('DROP INDEX ec_host_operations_observation_retention'); api.handle.exec('DROP INDEX ec_host_operations_important_history'); + api.handle.exec(`DROP TRIGGER operation_history_count_insert; DROP TRIGGER operation_history_count_delete; + DROP TRIGGER operation_history_count_update; DROP TABLE ec_host_operation_history_counts; + DELETE FROM ec_host_schema_migrations WHERE name='history-counts-v1'`); api.handle.exec('ALTER TABLE ec_host_operations DROP COLUMN observation_success'); api.closeDb(); diff --git a/tests/unit/host-operation-history.test.ts b/tests/unit/host-operation-history.test.ts index 819634c..d3d6f80 100644 --- a/tests/unit/host-operation-history.test.ts +++ b/tests/unit/host-operation-history.test.ts @@ -43,6 +43,40 @@ async function operationHistory() { } describe('host operation history', () => { + it('maintains exact scoped totals through receipts, expiration, rollback, deletion and reopening without recounting history', async () => { + const api = await operationHistory(); + const { importantOperationSql } = await import('../../src/lib/server/hosts/observations.js'); + const { closeDb, db } = await import('../../src/lib/server/db/index.js'); + const ids = Array.from({ length: 35 }, (_, i) => api.add(i % 2 ? 'instance.inspect' : 'instance.restart', { state: 'pending' })); + const verify = () => { + for (const host of [undefined, api.first.id, api.second.id]) for (const server of [undefined, 'server-one']) { + for (const filter of ['all', 'routine', 'important'] as const) { + const where = ['o.history_expired_at IS NULL']; const args: string[] = []; + if (host) { where.push('o.host_id=?'); args.push(host); } + if (server) { where.push('o.server_name=?'); args.push(server); } + if (filter !== 'all') where.push(filter === 'routine' ? 'o.routine=1' : importantOperationSql); + const expected = db().prepare(`SELECT COUNT(*) total FROM ec_host_operations o WHERE ${where.join(' AND ')}`).get(...args) as { total: number }; + expect(api.listHostOperationHistory(1, filter, { host, server }).total).toBe(expected.total); + } + } + }; + verify(); + for (const [index, id] of ids.entries()) api.reportHostOperationStatus(api.first.id, id, + index % 3 ? { state: 'completed', result: { healthy: index % 5 !== 0 } } : { state: 'failed', error: 'failure' }); + api.handle.prepare("UPDATE ec_host_operations SET server_name='server-one',host_id=? WHERE rowid%2=0").run(api.second.id); + api.handle.prepare('UPDATE ec_host_operations SET history_expired_at=? WHERE rowid%4=0').run(new Date().toISOString()); + api.handle.exec('DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT id FROM ec_host_operations WHERE rowid%7=0); DELETE FROM ec_host_operations WHERE rowid%7=0'); + expect(() => api.handle.transaction(() => { api.handle.exec('DELETE FROM ec_host_operation_events; DELETE FROM ec_host_operations'); throw new Error('rollback'); })()).toThrow('rollback'); + verify(); closeDb(); verify(); + const prepare = vi.spyOn(db(), 'prepare'); + api.listHostOperationHistory(1); api.listHostOperationHistory(2); + expect(prepare.mock.calls.some(([sql]) => /COUNT\(/i.test(sql))).toBe(false); + const count = prepare.mock.calls.find(([sql]) => sql.includes('FROM ec_host_operation_history_counts'))![0]; + expect(db().prepare('EXPLAIN QUERY PLAN ' + count).all('', '')).toEqual(expect.arrayContaining([ + expect.objectContaining({ detail: expect.stringContaining('USING PRIMARY KEY') }) + ])); + }); + it('scopes counts and pages to one host or server and preserves deleted-host history', async () => { const api = await seedHistory(90); api.handle.prepare("UPDATE ec_host_operations SET server_name='server-one' WHERE host_id=?").run(api.first.id); @@ -223,6 +257,11 @@ describe('host operation history', () => { const manual = api.add('rcon.execute', { payload: { command: 'say command-secret' } }); const unsafe = api.add('files.read', { payload: { path: '../private-secret' } }); api.handle.exec(`DROP INDEX ec_host_operations_routine_history; + DROP TRIGGER operation_history_count_insert; + DROP TRIGGER operation_history_count_delete; + DROP TRIGGER operation_history_count_update; + DROP TABLE ec_host_operation_history_counts; + DELETE FROM ec_host_schema_migrations WHERE name='history-counts-v1'; DROP INDEX ec_host_operations_important_history; DROP INDEX ec_host_operations_server_history; ALTER TABLE ec_host_operations DROP COLUMN routine;