josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add configurable retention for successful routine operations

Browse Source
This commit is contained in:
josh
2026-10-05 13:09:44 -04:00
parent 2d069907f2
commit e6912ceb57
14 changed files with 590 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/fleet-settings.ts
+10 -4
View File
@@ -1,5 +1,6 @@
export interface FleetSettings {
deployKeepDays: number;
observationKeepDays: number;
restartConcurrency: number;
createConcurrency: number;
destroyConcurrency: number;
@@ -7,6 +8,7 @@ export interface FleetSettings {
export const defaultFleetSettings: FleetSettings = {
deployKeepDays: 7,
observationKeepDays: 1,
restartConcurrency: 1,
createConcurrency: 1,
destroyConcurrency: 1
@@ -14,6 +16,7 @@ export const defaultFleetSettings: FleetSettings = {
export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
deployKeepDays: 'Campaign history retention',
observationKeepDays: 'Routine operation history retention',
restartConcurrency: 'Concurrent restarts',
createConcurrency: 'Concurrent server startups',
destroyConcurrency: 'Concurrent server destructions'
@@ -27,11 +30,14 @@ export function validateFleetSettings(input: unknown): FleetSettings {
}
for (const key of Object.keys(defaultFleetSettings) as (keyof FleetSettings)[]) {
const value = values[key];
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) {
throw new Error(`${fleetSettingLabels[key]} must be a positive whole number.`);
const minimum = key === 'observationKeepDays' ? 0 : 1;
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) {
throw new Error(`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`);
}
if ((key === 'deployKeepDays' || key === 'observationKeepDays') && value > 36500) {
throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`);
}
if (key === 'deployKeepDays' && value > 36500) throw new Error('Campaign history retention must be at most 36500 days.');
}
return { deployKeepDays: values.deployKeepDays as number, restartConcurrency: values.restartConcurrency as number,
return { deployKeepDays: values.deployKeepDays as number, observationKeepDays: values.observationKeepDays as number, restartConcurrency: values.restartConcurrency as number,
createConcurrency: values.createConcurrency as number, destroyConcurrency: values.destroyConcurrency as number };
}
src/lib/server/db/fleet-settings.ts
+3 -2
View File
@@ -8,8 +8,9 @@ export function initializeFleetSettings(handle: Database.Database): void {
const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
if (saved) {
const settings = JSON.parse(saved.value);
if (!Object.hasOwn(settings, 'destroyConcurrency')) {
const upgraded = validateFleetSettings({ ...settings, destroyConcurrency: defaultFleetSettings.destroyConcurrency });
if (!Object.hasOwn(settings, 'destroyConcurrency') || !Object.hasOwn(settings, 'observationKeepDays')) {
const upgraded = validateFleetSettings({ destroyConcurrency: defaultFleetSettings.destroyConcurrency,
observationKeepDays: defaultFleetSettings.observationKeepDays, ...settings });
handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
}
return;
src/lib/server/hosts/index.ts
+24 -1
View File
@@ -4,6 +4,8 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt
import { isIP } from 'node:net';
import { db } from '../db/index.js';
import { ensureHostSchema } from './schema.js';
import { successfulObservation } from './observations.js';
import { fleetSettings } from '../settings.js';
import { recordInstanceResources, validateInstanceResources } from './calibration.js';
import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationHistoryPage, HostOperationState } from '../../hosts.js';
@@ -372,6 +374,24 @@ export function listHostOperationHistory(page = 1): HostOperationHistoryPage {
return { operations, page: currentPage, pageSize, total, totalPages };
})();
}
/** Bound each cleanup pass so retained history cannot stall monitoring or receipts. */
export function pruneHostObservations(): number {
const days = fleetSettings().observationKeepDays;
if (!Number.isSafeInteger(days) || days <= 0 || days > 36500) return 0;
const handle = database();
const cutoff = new Date(Date.now() - days * 86_400_000).toISOString();
return handle.transaction(() => {
const rows = handle.prepare(`SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed'
AND updated_at<? ORDER BY updated_at LIMIT 500`).all(cutoff) as { id: string }[];
if (!rows.length) return 0;
const ids = rows.map(row => row.id);
const placeholders = ids.map(() => '?').join(',');
handle.prepare(`DELETE FROM ec_host_operation_events WHERE operation_id IN (${placeholders})`).run(...ids);
handle.prepare(`DELETE FROM ec_host_operation_payloads WHERE operation_id IN (${placeholders})`).run(...ids);
return handle.prepare(`DELETE FROM ec_host_operations WHERE id IN (${placeholders})`).run(...ids).changes;
})();
}
/** Heartbeats can tell an executing create to stop at its next safe command boundary. */
export function pendingCreationCancellations(hostId: string): string[] {
return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
@@ -479,7 +499,10 @@ export function reportHostOperationStatus(hostId: string, id: string, input: { s
const error = input.error === undefined || input.error === null ? null : String(input.error).slice(0, 4000);
const result = input.result === undefined ? null : JSON.stringify(input.result);
if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id);
const observation = state === 'completed' && !error && successfulObservation(operation.kind,
operation.kind === 'rcon.execute' ? JSON.parse(getHostOperationPayload(hostId, id).toString('utf8')) : {}, input.result);
database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=?,observation_success=? WHERE id=?')
.run(state, result, error, now(), Number(observation), id);
if (operation.kind === 'host.decommission') {
if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id);
else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id);
src/lib/server/hosts/observations.ts
+18
View File
@@ -0,0 +1,18 @@
export const observationKinds = ['host.inspect', 'instance.inspect', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read'];
const commands = ['status', 'getstatus', 'version'];
export function successfulObservation(kind: string, payload: Record<string, unknown>, result: unknown): boolean {
const routine = observationKinds.includes(kind) || kind === 'rcon.execute' && commands.includes(String(payload.command));
const value = result && typeof result === 'object' ? result as Record<string, unknown> : {};
return routine && value.ok !== false && value.healthy !== false && (value.error == null || value.error === '');
}
/** Uses the operation alias o; serialized payloads may live outside the routing row. */
export const routineObservationSql = `(o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')})
OR (o.kind='rcon.execute' AND json_extract(COALESCE(
(SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')))`;
export const successfulObservationSql = `(o.state='completed' AND COALESCE(o.error,'')='' AND ${routineObservationSql}
AND CASE WHEN o.result IS NULL THEN 1 WHEN json_valid(o.result)
THEN json_extract(o.result,'$.ok') IS NOT 0 AND json_extract(o.result,'$.healthy') IS NOT 0
AND COALESCE(json_extract(o.result,'$.error'),'')='' ELSE 0 END)`;
src/lib/server/hosts/schema.ts
+11
View File
@@ -1,4 +1,5 @@
import type Database from 'better-sqlite3';
import { successfulObservationSql } from './observations.js';
const initialized = new WeakSet<Database.Database>();
export function ensureHostSchema(handle: Database.Database): void {
@@ -28,6 +29,7 @@ export function ensureHostSchema(handle: Database.Database): void {
payload TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending','accepted','running','completed','failed','unknown')),
result TEXT, error TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, last_delivered_at TEXT,
observation_success INTEGER NOT NULL DEFAULT 0,
UNIQUE(host_id, idempotency_key)
);
CREATE INDEX IF NOT EXISTS ec_host_operations_queue ON ec_host_operations(host_id, state, created_at);
@@ -45,6 +47,7 @@ export function ensureHostSchema(handle: Database.Database): void {
id INTEGER PRIMARY KEY AUTOINCREMENT, operation_id TEXT NOT NULL REFERENCES ec_host_operations(id),
state TEXT NOT NULL, created_at TEXT NOT NULL, error TEXT
);
CREATE INDEX IF NOT EXISTS ec_host_operation_events_operation ON ec_host_operation_events(operation_id);
CREATE TABLE IF NOT EXISTS ec_host_ports (
host_id TEXT NOT NULL REFERENCES ec_hosts(id), instance_id TEXT NOT NULL,
bind_address TEXT NOT NULL, protocol TEXT NOT NULL CHECK (protocol IN ('udp','tcp')),
@@ -82,6 +85,14 @@ export function ensureHostSchema(handle: Database.Database): void {
`);
const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
if (!columns.some(column => column.name === 'observation_success')) {
handle.transaction(() => {
handle.exec('ALTER TABLE ec_host_operations ADD COLUMN observation_success INTEGER NOT NULL DEFAULT 0');
handle.exec(`UPDATE ec_host_operations AS o SET observation_success=1 WHERE ${successfulObservationSql}`);
})();
}
handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_observation_retention ON ec_host_operations(updated_at)
WHERE observation_success=1 AND state='completed'`);
const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[];
src/lib/server/pipeline/sweep.ts
+2
View File
@@ -2,6 +2,7 @@ import { fleetSettings } from '../settings.js';
import { db } from '../db/index.js';
import { emit } from './bus.js';
import { requestRestart } from '../provisioning/index.js';
import { pruneHostObservations } from '../hosts/index.js';
interface PendingRow {
deployment_id: string; server: string; restart_state: string;
@@ -18,6 +19,7 @@ export function sweep(): Promise<SweepResult> {
return sweepInFlight;
}
async function runSweep(): Promise<SweepResult> {
pruneHostObservations();
const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };