josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Maintain scoped operation history totals without table scans

Browse Source
This commit is contained in:
josh
2026-10-06 14:53:51 -04:00
parent b17673720b
commit 703d414c7a
5 changed files with 89 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/history-counts.ts
+42
View File
@@ -0,0 +1,42 @@
import type Database from 'better-sqlite3';
import { importantOperationSql } from './observations.js';
const important = (alias: string) => importantOperationSql.replaceAll('o.', `${alias}.`);
const visible = (alias: string) => `${alias}.history_expired_at IS NULL`;
/** Durable totals for global, host, server, and host/server filters; receipts update only changed membership. */
export function installHistoryCounts(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM ec_host_schema_migrations WHERE name='history-counts-v1'").get()) return;
handle.transaction(() => {
handle.exec(`CREATE TABLE ec_host_operation_history_counts (
host_id TEXT NOT NULL, server_name TEXT NOT NULL, total INTEGER NOT NULL,
routine INTEGER NOT NULL, important INTEGER NOT NULL, PRIMARY KEY(host_id,server_name)
) WITHOUT ROWID;
DROP INDEX ec_host_operations_important_history;
CREATE INDEX ec_host_operations_important_history ON ec_host_operations(created_at)
WHERE history_expired_at IS NULL AND ${importantOperationSql.replaceAll('o.', '')};`);
for (const [host, server] of [['', ''], ['o.host_id', ''], ['', 'o.server_name'], ['o.host_id', 'o.server_name']]) {
const groups = [host, server].filter(Boolean);
handle.exec(`INSERT INTO ec_host_operation_history_counts
SELECT ${host || "''"},${server || "''"},COUNT(*),COALESCE(SUM(o.routine=1),0),COALESCE(SUM(${important('o')}),0)
FROM ec_host_operations o WHERE ${visible('o')} ${server ? 'AND o.server_name IS NOT NULL' : ''}
${groups.length ? 'GROUP BY ' + groups.join(',') : ''}`);
}
const adjust = (alias: string, direction: 1 | -1) => `
INSERT INTO ec_host_operation_history_counts(host_id,server_name,total,routine,important)
SELECT host_id,server_name,${direction},${direction}*(${alias}.routine=1),${direction}*${important(alias)} FROM (
SELECT '' host_id,'' server_name UNION ALL SELECT ${alias}.host_id,''
UNION ALL SELECT '',${alias}.server_name WHERE ${alias}.server_name IS NOT NULL
UNION ALL SELECT ${alias}.host_id,${alias}.server_name WHERE ${alias}.server_name IS NOT NULL
) WHERE ${visible(alias)}
ON CONFLICT(host_id,server_name) DO UPDATE SET total=total+excluded.total,
routine=routine+excluded.routine,important=important+excluded.important;`;
handle.exec(`CREATE TRIGGER operation_history_count_insert AFTER INSERT ON ec_host_operations BEGIN ${adjust('NEW', 1)} END;
CREATE TRIGGER operation_history_count_delete AFTER DELETE ON ec_host_operations BEGIN ${adjust('OLD', -1)} END;
CREATE TRIGGER operation_history_count_update AFTER UPDATE OF host_id,server_name,routine,state,observation_success,error,history_expired_at ON ec_host_operations
WHEN OLD.host_id IS NOT NEW.host_id OR OLD.server_name IS NOT NEW.server_name OR OLD.routine IS NOT NEW.routine
OR (${visible('OLD')}) IS NOT (${visible('NEW')}) OR ${important('OLD')} IS NOT ${important('NEW')}
BEGIN ${adjust('OLD', -1)} ${adjust('NEW', 1)} END;`);
handle.prepare("INSERT INTO ec_host_schema_migrations(name) VALUES('history-counts-v1')").run();
})();
}
src/lib/server/hosts/index.ts
+3 -1
View File
@@ -426,7 +426,9 @@ export function listHostOperationHistory(page = 1, filter: HostOperationFilter =
if (scope.server) { conditions.push('o.server_name=?'); parameters.push(scope.server); }
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
return handle.transaction((): HostOperationHistoryPage => {
const { total } = handle.prepare(`SELECT COUNT(*) AS total FROM ec_host_operations o ${where}`).get(...parameters) as { total: number };
const countColumn = filter === 'all' ? 'total' : filter;
const total = (handle.prepare(`SELECT ${countColumn} AS total FROM ec_host_operation_history_counts WHERE host_id=? AND server_name=?`)
.get(scope.host ?? '', scope.server ?? '') as { total: number } | undefined)?.total ?? 0;
const totalPages = Math.max(1, Math.ceil(total / pageSize));
const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1);
const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind,
src/lib/server/hosts/schema.ts
+2
View File
@@ -1,6 +1,7 @@
import { createHash } from 'node:crypto';
import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js';
import { retentionClocks } from '../db/retention-clocks.js';
import { installHistoryCounts } from './history-counts.js';
import type Database from 'better-sqlite3';
import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
@@ -187,5 +188,6 @@ export function ensureHostSchema(handle: Database.Database): void {
WHEN NEW.history_expired_at IS NOT NULL BEGIN
DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=NEW.id; END;`);
retentionClocks(handle, { ec_host_operations: ['created_at', 'updated_at'] });
installHistoryCounts(handle);
initialized.add(handle);
}
tests/unit/host-observation-retention.test.ts
+3
View File
@@ -179,6 +179,9 @@ describe('routine host observation retention', () => {
retained.push(malformed);
api.handle.exec('DROP INDEX ec_host_operations_observation_retention');
api.handle.exec('DROP INDEX ec_host_operations_important_history');
api.handle.exec(`DROP TRIGGER operation_history_count_insert; DROP TRIGGER operation_history_count_delete;
DROP TRIGGER operation_history_count_update; DROP TABLE ec_host_operation_history_counts;
DELETE FROM ec_host_schema_migrations WHERE name='history-counts-v1'`);
api.handle.exec('ALTER TABLE ec_host_operations DROP COLUMN observation_success');
api.closeDb();
tests/unit/host-operation-history.test.ts
+39
View File
@@ -43,6 +43,40 @@ async function operationHistory() {
}
describe('host operation history', () => {
it('maintains exact scoped totals through receipts, expiration, rollback, deletion and reopening without recounting history', async () => {
const api = await operationHistory();
const { importantOperationSql } = await import('../../src/lib/server/hosts/observations.js');
const { closeDb, db } = await import('../../src/lib/server/db/index.js');
const ids = Array.from({ length: 35 }, (_, i) => api.add(i % 2 ? 'instance.inspect' : 'instance.restart', { state: 'pending' }));
const verify = () => {
for (const host of [undefined, api.first.id, api.second.id]) for (const server of [undefined, 'server-one']) {
for (const filter of ['all', 'routine', 'important'] as const) {
const where = ['o.history_expired_at IS NULL']; const args: string[] = [];
if (host) { where.push('o.host_id=?'); args.push(host); }
if (server) { where.push('o.server_name=?'); args.push(server); }
if (filter !== 'all') where.push(filter === 'routine' ? 'o.routine=1' : importantOperationSql);
const expected = db().prepare(`SELECT COUNT(*) total FROM ec_host_operations o WHERE ${where.join(' AND ')}`).get(...args) as { total: number };
expect(api.listHostOperationHistory(1, filter, { host, server }).total).toBe(expected.total);
}
}
};
verify();
for (const [index, id] of ids.entries()) api.reportHostOperationStatus(api.first.id, id,
index % 3 ? { state: 'completed', result: { healthy: index % 5 !== 0 } } : { state: 'failed', error: 'failure' });
api.handle.prepare("UPDATE ec_host_operations SET server_name='server-one',host_id=? WHERE rowid%2=0").run(api.second.id);
api.handle.prepare('UPDATE ec_host_operations SET history_expired_at=? WHERE rowid%4=0').run(new Date().toISOString());
api.handle.exec('DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT id FROM ec_host_operations WHERE rowid%7=0); DELETE FROM ec_host_operations WHERE rowid%7=0');
expect(() => api.handle.transaction(() => { api.handle.exec('DELETE FROM ec_host_operation_events; DELETE FROM ec_host_operations'); throw new Error('rollback'); })()).toThrow('rollback');
verify(); closeDb(); verify();
const prepare = vi.spyOn(db(), 'prepare');
api.listHostOperationHistory(1); api.listHostOperationHistory(2);
expect(prepare.mock.calls.some(([sql]) => /COUNT\(/i.test(sql))).toBe(false);
const count = prepare.mock.calls.find(([sql]) => sql.includes('FROM ec_host_operation_history_counts'))![0];
expect(db().prepare('EXPLAIN QUERY PLAN ' + count).all('', '')).toEqual(expect.arrayContaining([
expect.objectContaining({ detail: expect.stringContaining('USING PRIMARY KEY') })
]));
});
it('scopes counts and pages to one host or server and preserves deleted-host history', async () => {
const api = await seedHistory(90);
api.handle.prepare("UPDATE ec_host_operations SET server_name='server-one' WHERE host_id=?").run(api.first.id);
@@ -223,6 +257,11 @@ describe('host operation history', () => {
const manual = api.add('rcon.execute', { payload: { command: 'say command-secret' } });
const unsafe = api.add('files.read', { payload: { path: '../private-secret' } });
api.handle.exec(`DROP INDEX ec_host_operations_routine_history;
DROP TRIGGER operation_history_count_insert;
DROP TRIGGER operation_history_count_delete;
DROP TRIGGER operation_history_count_update;
DROP TABLE ec_host_operation_history_counts;
DELETE FROM ec_host_schema_migrations WHERE name='history-counts-v1';
DROP INDEX ec_host_operations_important_history;
DROP INDEX ec_host_operations_server_history;
ALTER TABLE ec_host_operations DROP COLUMN routine;