304 lines
28 KiB
TypeScript
304 lines
28 KiB
TypeScript
import { blobSize, pruneFileBlobs, referenceBlobs } from './db/file-blobs.js';
|
|
import { createHash } from 'node:crypto';
|
|
import { db } from './db/index.js';
|
|
import { fleetSettings } from './settings.js';
|
|
import { runtimeLaunchSql } from './fleet/runtime-config.js';
|
|
import { ensureHostSchema } from './hosts/schema.js';
|
|
import { ensureGameFileSchema } from './game-files/schema.js';
|
|
import { ensureAuthSchema } from './auth/schema.js';
|
|
import type { FleetSettings } from '../fleet-settings.js';
|
|
|
|
interface Policy {
|
|
category: string; label: string; table: string; id: string; clock: string;
|
|
days: keyof FleetSettings; eligible: string; bytes?: string; failed?: string; minimum?: boolean;
|
|
}
|
|
const terminal = "('done','warning','failed','cancelled')";
|
|
// Cancellation stops observation, not an already-delivered command. Its source
|
|
// history and before images must survive until the receipt has been reconciled.
|
|
const unsettledDeployment = `d.state NOT IN ${terminal}
|
|
OR EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=d.id AND r.healthy_at IS NULL)
|
|
OR EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id)
|
|
LEFT JOIN native_instances i ON i.id=n.instance_id WHERE n.deployment_id=d.id AND
|
|
(a.state NOT IN ('completed','failed') OR (a.id=n.operation_id AND a.state='completed' AND (
|
|
(n.applied_at IS NULL AND COALESCE(json_array_length(n.files_json),0)>0)
|
|
OR (d.action='destroy' AND i.retired_at IS NULL)
|
|
OR (d.action='rebuild' AND i.retired_at IS NULL AND i.generation=n.generation
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id))))))
|
|
OR EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id
|
|
WHERE m.deployment_id=d.id AND (a.state NOT IN ('completed','failed') OR (a.state='completed' AND m.removed_at IS NULL)))`;
|
|
const unreferencedOperation = `NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE o.id IN (n.operation_id,n.stage_operation_id))
|
|
AND NOT EXISTS (SELECT 1 FROM native_instances i WHERE i.retired_at IS NULL AND ${runtimeLaunchSql})
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.remove_operation_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_decommissions d WHERE d.operation_id=o.id)
|
|
AND o.kind NOT IN ('runtime.pull','library.download','library.import','library.remove','host.decommission','host.rotate-credential')
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_operations dependent WHERE dependent.state IN ('pending','accepted','running','unknown')
|
|
AND json_extract(dependent.payload,'$.createOperationId')=o.id)`;
|
|
const campaignSafe = `NOT EXISTS (SELECT 1 FROM deployment_servers s WHERE s.deployment_id=o.id AND s.server_deleted_at IS NULL
|
|
AND s.restart_state IN ('pending','held','manual'))
|
|
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id
|
|
WHERE s.kind='campaign' AND s.source_id=o.id AND (${unsettledDeployment}))
|
|
AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.rolled_back_by=o.id)`;
|
|
const failedCampaign = `EXISTS (SELECT 1 FROM deployment_servers s WHERE s.deployment_id=o.id AND s.server_deleted_at IS NULL AND s.restart_state='failed')`;
|
|
|
|
function policies(): Policy[] {
|
|
return [
|
|
{ category: 'campaign', label: 'Campaigns', table: 'deployments', id: 'id', clock: 'started_at', days: 'deployKeepDays',
|
|
eligible: `${campaignSafe} AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='file-content' AND c.record_id=o.id AND c.protected=1)`, failed: failedCampaign,
|
|
bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
|
|
{ category: 'provisioning', label: 'Provisioning workflows', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'provisioningKeepDays', failed: "o.state='failed'", eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
|
|
AND NOT EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=o.id AND r.healthy_at IS NULL)
|
|
AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id) WHERE n.deployment_id=o.id AND a.state NOT IN ('completed','failed'))
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id WHERE m.deployment_id=o.id AND a.state NOT IN ('completed','failed'))
|
|
AND NOT EXISTS (SELECT 1 FROM deployment_servers s WHERE s.provisioning_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM beta_servers s WHERE s.provisioning_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs s WHERE s.provisioning_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM watchdog_actions s WHERE s.operation_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE o.id IN (c.creation_id,c.cleanup_id))
|
|
AND NOT EXISTS (SELECT 1 FROM server_creation_dependencies c WHERE c.removal_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM server_collection_members m JOIN server_collection_members sibling ON sibling.collection_id=m.collection_id
|
|
JOIN server_deployments d ON d.id=sibling.deployment_id WHERE m.deployment_id=o.id AND (${unsettledDeployment}))`,
|
|
bytes: "COALESCE(length(o.observation_json),0)+(SELECT COALESCE(SUM(length(recipe)),0) FROM server_launch_recipes WHERE deployment_id=o.id)" },
|
|
{ category: 'operation', label: 'Agent actions and resolved failures', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'actionKeepDays',
|
|
failed: "(o.state='failed' OR (o.routine=1 AND o.observation_success=0))",
|
|
eligible: `o.history_expired_at IS NULL AND o.state IN ('completed','failed') AND (o.routine=0 OR o.observation_success=0) AND ${unreferencedOperation}`,
|
|
bytes: "COALESCE(length(o.payload),0)+COALESCE(length(o.result),0)+COALESCE((SELECT length(payload) FROM ec_host_operation_payloads WHERE operation_id=o.id),0)" },
|
|
{ category: 'observation', label: 'Routine operations', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'observationKeepDays', eligible: "o.state='completed' AND o.observation_success=1", bytes: 'length(o.payload)+COALESCE(length(o.result),0)' },
|
|
{ category: 'maintenance', label: 'Maintenance jobs', table: 'maintenance_jobs', id: 'id', clock: 'finished_at', days: 'maintenanceKeepDays',
|
|
eligible: `o.kind!='update' AND o.state IN ('done','failed','cancelled') AND NOT EXISTS (SELECT 1 FROM server_deployments d WHERE d.id=o.provisioning_id AND (${unsettledDeployment}))` },
|
|
{ category: 'recovery', label: 'Resolved recovery incidents', table: 'watchdog_incidents', id: 'id', clock: 'finished_at', days: 'recoveryKeepDays',
|
|
eligible: `o.state='resolved' AND NOT EXISTS (SELECT 1 FROM watchdog_actions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.incident_id=o.id AND (${unsettledDeployment}))
|
|
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a JOIN restart_rollout r ON r.operation_id=a.operation_id WHERE a.incident_id=o.id AND r.healthy_at IS NULL)` },
|
|
{ category: 'match', label: 'Matches', table: 'leaderboard_matches', id: 'id', clock: 'last_received_at', days: 'matchKeepDays', eligible: "o.status IN ('completed','interrupted')", bytes: '0' },
|
|
{ category: 'beta', label: 'Completed betas', table: 'betas', id: 'id', clock: 'closed_at', days: 'betaKeepDays',
|
|
eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta-file-content' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
|
|
bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` },
|
|
{ category: 'file-content', label: 'Campaign file bodies', table: 'deployments', id: 'id', clock: 'finished_at', days: 'fileContentKeepDays',
|
|
eligible: `${campaignSafe} AND NOT (${failedCampaign})
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=o.id AND c.protected=1)
|
|
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`,
|
|
bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
|
|
{ category: 'beta-file-content', label: 'Closed beta file snapshots', table: 'betas', id: 'id', clock: 'closed_at', days: 'fileContentKeepDays',
|
|
eligible: `o.state IN ('promoted','rolled-back')
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1)
|
|
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT))
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
|
|
bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` },
|
|
{ category: 'workflow-payload', label: 'Completed launch file bodies', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'payloadKeepDays',
|
|
failed: "o.state='failed'",
|
|
eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=o.id AND c.protected=1)
|
|
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='workflow-payload' AND e.record_id=o.id)`,
|
|
bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type='recipe' AND r.owner_id=o.id)" },
|
|
{ category: 'operation-payload', label: 'Completed command file bodies', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'payloadKeepDays',
|
|
failed: "o.state='failed'",
|
|
eligible: `o.state IN ('completed','failed') AND o.kind IN ('instance.create','instance.rebuild','instance.restart','instance.start','files.stage','files.read')
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='operation' AND c.record_id=o.id AND c.protected=1)
|
|
AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
|
WHERE o.id IN (n.operation_id,n.stage_operation_id) AND ((${unsettledDeployment}) OR EXISTS (
|
|
SELECT 1 FROM retention_controls c WHERE c.category IN ('provisioning','workflow-payload') AND c.record_id=d.id AND c.protected=1)))
|
|
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='operation-payload' AND e.record_id=o.id)`,
|
|
bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type IN ('operation','operation-result') AND r.owner_id=o.id)" },
|
|
{ category: 'version', label: 'Unused artifact versions', table: 'script_versions', id: 'id', clock: 'uploaded_at', days: 'versionKeepDays', minimum: false,
|
|
eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id)
|
|
AND ${versionReferences()}`, bytes: `${blobSize('o.content_hash')}` },
|
|
{ category: 'audit', label: 'Authentication audit events', table: 'auth_audit', id: 'id', clock: 'created_at', days: 'auditKeepDays', eligible: '1', bytes: 'length(o.details)' }
|
|
];
|
|
}
|
|
function versionReferences(): string {
|
|
// Even cascading references pin versions: automatic cleanup must never erase
|
|
// another feature's history as a side effect of deleting an artifact.
|
|
const tables = db().prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as { name: string }[];
|
|
const clauses: string[] = [];
|
|
for (const { name } of tables) {
|
|
if (name === 'plugin_imports') continue;
|
|
const quote = (s: string) => '"' + s.replaceAll('"','""') + '"';
|
|
for (const fk of db().prepare(`PRAGMA foreign_key_list(${quote(name)})`).all() as { table: string; from: string }[]) {
|
|
if (fk.table === 'script_versions') clauses.push(`NOT EXISTS (SELECT 1 FROM ${quote(name)} v WHERE v.${quote(fk.from)}=o.id)`);
|
|
}
|
|
}
|
|
return clauses.join(' AND ') || '1';
|
|
}
|
|
function initialize() { ensureHostSchema(db()); ensureAuthSchema(db()); }
|
|
|
|
function withoutFileBodies(value: unknown): unknown {
|
|
if (Array.isArray(value)) return value.map(withoutFileBodies);
|
|
if (!value || typeof value !== 'object') return value;
|
|
return Object.fromEntries(Object.entries(value).filter(([key]) => !['contentHash','contentSha256','contentBase64'].includes(key))
|
|
.map(([key, item]) => [key, withoutFileBodies(item)]));
|
|
}
|
|
|
|
function expirePayload(category: string, id: string): void {
|
|
if (category === 'workflow-payload') {
|
|
const row = db().prepare('SELECT recipe FROM server_launch_recipes WHERE deployment_id=?').get(id) as { recipe: string } | undefined;
|
|
if (row) db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(withoutFileBodies(JSON.parse(row.recipe))), id);
|
|
db().prepare('UPDATE native_provisioning SET files_json=NULL WHERE deployment_id=?').run(id);
|
|
referenceBlobs('recipe', id, []);
|
|
return;
|
|
}
|
|
const row = db().prepare(`SELECT COALESCE(p.payload,o.payload) payload,o.result FROM ec_host_operations o
|
|
LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(id) as { payload: string; result: string | null };
|
|
const payload = JSON.stringify(withoutFileBodies(JSON.parse(row.payload)));
|
|
db().prepare(`UPDATE ec_host_operations SET payload=?,result=?,request_hash=COALESCE(request_hash,?) WHERE id=?`)
|
|
.run(payload, row.result === null ? null : JSON.stringify(withoutFileBodies(JSON.parse(row.result))),
|
|
createHash('sha256').update(row.payload).digest('hex'), id);
|
|
db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(id);
|
|
referenceBlobs('operation', id, []);
|
|
referenceBlobs('operation-result', id, []);
|
|
}
|
|
function selection(policy: Policy, settings = fleetSettings()) {
|
|
const days = settings[policy.days];
|
|
const failureDays = settings.failureKeepDays;
|
|
const cut = (n: number) => new Date(Date.now() - n * 86400_000).toISOString();
|
|
const age = policy.category === 'audit' ? 'o.created_at < @numericCutoff' : `o.${policy.clock} < @cutoff`;
|
|
const failed = policy.failed ?? '0';
|
|
const enabled = days > 0 || policy.category === 'version' && settings.versionKeepCount > 0;
|
|
// Keep age outside the correlated failure checks so SQLite can seek the
|
|
// clock index before examining dependencies. Resolved failures start from
|
|
// their controls, not another scan of the entire history table.
|
|
const candidates: string[] = [];
|
|
if (enabled) candidates.push(`SELECT o.rowid FROM ${policy.table} o
|
|
WHERE ${policy.category === 'version' && !days ? '1' : age} AND NOT (${failed})`);
|
|
if (failureDays > 0 && policy.failed) candidates.push(`SELECT o.rowid FROM retention_controls c
|
|
JOIN ${policy.table} o ON o.${policy.id}=c.record_id
|
|
WHERE c.category=@category AND c.resolved_at<@failureCutoff
|
|
AND c.resolved_at>=o.${policy.clock} AND (${failed})`);
|
|
const minimum = policy.minimum === false ? 0 : settings.historyKeepCount;
|
|
const minimumScope = policy.category === 'observation' ? "WHERE observation_success=1 AND state='completed'"
|
|
: policy.category === 'operation' ? "WHERE history_expired_at IS NULL AND (routine=0 OR observation_success=0)" : '';
|
|
const versionCount = policy.category === 'version' ? `AND o.id NOT IN (SELECT id FROM script_versions v WHERE v.script_id=o.script_id ORDER BY version DESC LIMIT ${settings.versionKeepCount})` : '';
|
|
return { sql: `FROM ${policy.table} o WHERE ${candidates.length ? `o.rowid IN (${candidates.join(' UNION ALL ')})` : '0'} AND (${policy.eligible})
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category=@category AND c.record_id=CAST(o.${policy.id} AS TEXT) AND c.protected=1)
|
|
AND o.rowid NOT IN (SELECT rowid FROM ${policy.table} ${minimumScope} ORDER BY ${policy.clock} DESC,rowid DESC LIMIT ${minimum}) ${versionCount}`,
|
|
params: { category: policy.category, cutoff: cut(days), numericCutoff: Date.now() - days * 86400_000, failureCutoff: cut(failureDays) } };
|
|
}
|
|
// better-sqlite3 accepts unused named bindings, keeping selection reusable.
|
|
export function retentionPreview(settings = fleetSettings()) {
|
|
initialize();
|
|
return policies().map(policy => {
|
|
const query = selection(policy, settings);
|
|
const summary = db().prepare(`SELECT COUNT(*) AS eligible,COALESCE(SUM(COALESCE(${policy.bytes ?? '0'},0)),0) AS payloadBytes ${query.sql}`).get(query.params) as { eligible: number; payloadBytes: number };
|
|
const records = db().prepare(`SELECT CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 20`).all(query.params) as { id: string }[];
|
|
return { category: policy.category, label: policy.label, ...summary, records };
|
|
});
|
|
}
|
|
export function retentionControls() {
|
|
return db().prepare('SELECT category,record_id AS id,protected,resolved_at AS resolvedAt FROM retention_controls WHERE protected=1 OR resolved_at IS NOT NULL ORDER BY category,record_id').all();
|
|
}
|
|
export function setRetentionControl(input: { category: string; id: string; protected?: boolean; resolve?: boolean }) {
|
|
initialize();
|
|
ensureGameFileSchema(db());
|
|
const policy = policies().find(policy => policy.category === input.category)
|
|
?? (['game-file','upload'].includes(input.category) ? { category: input.category, table: 'game_file_versions',id: 'id',failed: undefined } : undefined);
|
|
if (!policy || typeof input.id !== 'string' || !input.id || input.id.length > 200
|
|
|| (input.protected !== undefined && typeof input.protected !== 'boolean') || (input.resolve !== undefined && typeof input.resolve !== 'boolean')) throw new Error('Choose a valid history category and record ID.');
|
|
const row = db().prepare(`SELECT 1 FROM ${policy.table} o WHERE CAST(o.${policy.id} AS TEXT)=? ${policy.table === 'ec_host_operations' ? 'AND o.history_expired_at IS NULL' : ''} ${input.resolve ? `AND (${policy.failed ?? '0'})` : ''}`).get(input.id);
|
|
if (!row) throw new Error(input.resolve ? 'Only a failed record can be marked resolved.' : 'History record was not found.');
|
|
if (input.protected && (db().prepare('SELECT 1 FROM retention_expirations WHERE category=? AND record_id=?').get(input.category,input.id)
|
|
|| ['game-file','upload'].includes(input.category) && db().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(input.id))) {
|
|
throw new Error('These files have expired or cleanup has already started. Protection cannot restore them.');
|
|
}
|
|
db().prepare(`INSERT INTO retention_controls(category,record_id,protected,resolved_at) VALUES(?,?,?,?)
|
|
ON CONFLICT(category,record_id) DO UPDATE SET protected=COALESCE(?,protected),resolved_at=COALESCE(?,resolved_at)`)
|
|
.run(input.category,input.id,Number(input.protected ?? false),input.resolve ? new Date().toISOString() : null,
|
|
input.protected === undefined ? null : Number(input.protected),input.resolve ? new Date().toISOString() : null);
|
|
}
|
|
|
|
/** Each policy removes at most 500 eligible records per pass; dependencies are checked again under the transaction. */
|
|
export function pruneHistory(only?: string): number {
|
|
initialize();
|
|
if (only === 'observation') return pruneRoutineObservations();
|
|
let removed = only ? 0 : pruneRoutineObservations();
|
|
for (const policy of policies().filter(policy => policy.category !== 'observation' && (!only || policy.category === only))) {
|
|
removed += db().transaction(() => {
|
|
const query = selection(policy);
|
|
const rows = db().prepare(`SELECT o.rowid AS rid,CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 500`).all(query.params) as { rid: number; id: string }[];
|
|
for (const row of rows) {
|
|
if (policy.category === 'provisioning') {
|
|
db().prepare('DELETE FROM native_provisioning WHERE deployment_id=?').run(row.id);
|
|
db().prepare('DELETE FROM ec_host_migrations WHERE deployment_id=?').run(row.id);
|
|
db().prepare('DELETE FROM restart_rollout WHERE operation_id=? AND healthy_at IS NOT NULL').run(row.id);
|
|
}
|
|
if (policy.category === 'operation' || policy.category === 'observation') {
|
|
db().prepare('DELETE FROM ec_host_operation_events WHERE operation_id=?').run(row.id);
|
|
if (policy.category === 'operation') {
|
|
const payload = db().prepare(`SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(row.id) as { payload: string };
|
|
db().prepare(`UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?),payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`)
|
|
.run(createHash('sha256').update(payload.payload).digest('hex'),new Date().toISOString(),row.id);
|
|
}
|
|
db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(row.id);
|
|
}
|
|
if (policy.category === 'workflow-payload' || policy.category === 'operation-payload') {
|
|
expirePayload(policy.category, row.id);
|
|
db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
|
|
} else if (policy.category === 'beta-file-content') {
|
|
db().prepare("UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?").run(row.id);
|
|
db().prepare('UPDATE beta_file_snapshots SET prior_content_hash=NULL WHERE beta_id=?').run(row.id);
|
|
db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
|
|
} else if (policy.category === 'file-content') {
|
|
db().prepare('UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?').run(row.id);
|
|
db().prepare("UPDATE staged_file_changes SET content_hash=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')").run(row.id);
|
|
db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
|
|
} else if (policy.category !== 'operation') {
|
|
db().prepare(`DELETE FROM ${policy.table} WHERE rowid=?`).run(row.rid);
|
|
db().prepare('DELETE FROM retention_controls WHERE category=? AND record_id=?').run(policy.category,row.id);
|
|
}
|
|
}
|
|
return rows.length;
|
|
})();
|
|
}
|
|
const blobs = pruneFileBlobs();
|
|
if ((removed || blobs) && Number(db().pragma('freelist_count', { simple: true })) >= 256) db().pragma('incremental_vacuum(256)');
|
|
return removed;
|
|
}
|
|
|
|
function samplePolicies(s = fleetSettings()) {
|
|
const iso = (days: number) => new Date(Date.now() - days * 86400_000).toISOString();
|
|
return [
|
|
{ category: 'host-resources', label: 'Host resource samples', table: 'ec_host_resource_samples', condition: 'sampled_at < ?', value: iso(s.hostResourceKeepDays), enabled: s.hostResourceKeepDays > 0 },
|
|
{ category: 'instance-resources', label: 'Instance resource samples', table: 'ec_instance_resource_samples', condition: 'bucket < ?', value: iso(Math.max(1,s.instanceResourceKeepDays)), enabled: s.instanceResourceKeepDays > 0 },
|
|
{ category: 'metrics', label: 'Fleet metrics samples', table: 'metric_samples', condition: 'at < ?', value: Date.now() - s.metricsKeepDays * 86400_000, enabled: s.metricsKeepDays > 0 },
|
|
{ category: 'telemetry', label: 'Telemetry diagnostics (age expiry)', table: 'telemetry_requests', condition: 'received_at < ?', value: iso(s.telemetryKeepDays), enabled: s.telemetryKeepDays > 0 }
|
|
];
|
|
}
|
|
export function sampleRetentionPreview(settings = fleetSettings()) {
|
|
initialize();
|
|
const result = samplePolicies(settings).filter(p => p.category !== 'telemetry').map(p => ({ category: p.category,label: p.label,records: [],payloadBytes: 0,
|
|
eligible: p.enabled ? (db().prepare(`SELECT COUNT(*) AS n FROM ${p.table} WHERE ${p.condition}`).get(p.value) as { n: number }).n : 0 }));
|
|
const telemetry = db().prepare(`SELECT COUNT(*) AS eligible,COALESCE(SUM(bytes),0) AS payloadBytes FROM (
|
|
SELECT received_at,length(payload) AS bytes,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS per_server,
|
|
ROW_NUMBER() OVER (ORDER BY id DESC) AS overall FROM telemetry_requests)
|
|
WHERE (? > 0 AND received_at < ?) OR (? > 0 AND per_server > ?) OR (? > 0 AND overall > ?)`)
|
|
.get(settings.telemetryKeepDays,new Date(Date.now()-settings.telemetryKeepDays*86400_000).toISOString(),settings.telemetryMaxPerServer,
|
|
settings.telemetryMaxPerServer,settings.telemetryMaxTotal,settings.telemetryMaxTotal) as {eligible:number;payloadBytes:number};
|
|
result.push({category:'telemetry',label:'Telemetry diagnostics (age and count limits)',records:[],...telemetry});
|
|
return result;
|
|
}
|
|
export function pruneSampleHistory() {
|
|
initialize();
|
|
for (const p of samplePolicies().filter(p => p.enabled)) db().prepare(`DELETE FROM ${p.table} WHERE rowid IN (SELECT rowid FROM ${p.table} WHERE ${p.condition} LIMIT 5000)`).run(p.value);
|
|
const s = fleetSettings();
|
|
if (s.telemetryMaxPerServer) db().prepare(`DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM
|
|
(SELECT id,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS rank FROM telemetry_requests) WHERE rank>? LIMIT 5000)`).run(s.telemetryMaxPerServer);
|
|
if (s.telemetryMaxTotal) db().prepare('DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 5000 OFFSET ?)').run(s.telemetryMaxTotal);
|
|
}
|
|
|
|
function pruneRoutineObservations(): number {
|
|
const settings = fleetSettings();
|
|
if (!settings.observationKeepDays) return 0;
|
|
const handle = db();
|
|
return handle.transaction(() => {
|
|
const rows = handle.prepare(`SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed'
|
|
AND updated_at<? AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='observation' AND c.record_id=ec_host_operations.id AND c.protected=1)
|
|
${settings.historyKeepCount ? `AND id NOT IN (SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed' ORDER BY updated_at DESC LIMIT ${settings.historyKeepCount})` : ''}
|
|
ORDER BY updated_at LIMIT 500`).all(new Date(Date.now()-settings.observationKeepDays*86400_000).toISOString()) as { id: string }[];
|
|
if (!rows.length) return 0;
|
|
const ids = JSON.stringify(rows.map(row=>row.id));
|
|
handle.prepare('DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT value FROM json_each(?))').run(ids);
|
|
handle.prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id IN (SELECT value FROM json_each(?))').run(ids);
|
|
return handle.prepare('DELETE FROM ec_host_operations WHERE id IN (SELECT value FROM json_each(?))').run(ids).changes;
|
|
})();
|
|
}
|