736 lines
37 KiB
TypeScript
736 lines
37 KiB
TypeScript
import { blobSize, pruneFileBlobs, referenceBlobs } from './db/file-blobs.js';
|
|
import { createHash } from 'node:crypto';
|
|
import { db } from './db/index.js';
|
|
import { fleetSettings } from './settings.js';
|
|
import { runtimeLaunchSql } from './fleet/runtime-config.js';
|
|
import { logResultMetadata } from './hosts/log-results.js';
|
|
import type { FleetSettings } from '../fleet-settings.js';
|
|
|
|
interface Policy {
|
|
category: string;
|
|
label: string;
|
|
table: string;
|
|
id: string;
|
|
clock: string;
|
|
days: keyof FleetSettings;
|
|
eligible: string;
|
|
bytes?: string;
|
|
failed?: string;
|
|
minimum?: boolean;
|
|
}
|
|
const terminal = "('done','warning','failed','cancelled')";
|
|
// Cancellation stops observation, not an already-delivered command. Its source
|
|
// history and before images must survive until the receipt has been reconciled.
|
|
const unsettledDeployment = `d.state NOT IN ${terminal}
|
|
OR EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=d.id AND r.healthy_at IS NULL)
|
|
OR EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id)
|
|
LEFT JOIN native_instances i ON i.id=n.instance_id WHERE n.deployment_id=d.id AND
|
|
(a.state NOT IN ('completed','failed')
|
|
OR (a.state='failed' AND a.kind='instance.rebuild' AND a.id=n.operation_id
|
|
AND json_type(a.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL)
|
|
OR (a.id=n.operation_id AND a.state='completed' AND (
|
|
(n.applied_at IS NULL AND COALESCE(json_array_length(n.files_json),0)>0)
|
|
OR (d.action='destroy' AND i.retired_at IS NULL)
|
|
OR (a.kind='instance.rebuild' AND i.retired_at IS NULL AND i.generation=n.generation)))))
|
|
OR EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id
|
|
WHERE m.deployment_id=d.id AND (a.state NOT IN ('completed','failed') OR (a.state='completed' AND m.removed_at IS NULL)))`;
|
|
// Recovery replaces operation_id with its latest command. The original launch
|
|
// still supplies rebuild files and the ownership label used by creation cleanup.
|
|
const recoveryLaunch = `EXISTS (SELECT 1 FROM native_provisioning n
|
|
JOIN server_deployments d ON d.id=n.deployment_id
|
|
JOIN native_instances i ON i.id=n.instance_id AND i.retired_at IS NULL
|
|
WHERE n.recovery_step IS NOT NULL AND d.server_deleted_at IS NULL AND o.instance_id=i.id AND o.host_id=i.host_id
|
|
AND ((o.kind='instance.create' AND o.generation=n.generation)
|
|
OR (o.kind='instance.rebuild' AND o.generation+1=n.generation))
|
|
AND ((${unsettledDeployment}) OR (d.action='create' AND d.state IN ('failed','cancelled')
|
|
AND i.generation BETWEEN n.generation AND n.generation+1
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server
|
|
AND newer.server_deleted_at IS NULL AND newer.rowid>d.rowid AND newer.state!='cancelled'
|
|
AND NOT (newer.action='destroy' AND newer.state='waiting'
|
|
AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=newer.id AND p.operation_id IS NOT NULL))))))`;
|
|
const unreferencedOperation = `NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE o.id IN (n.operation_id,n.stage_operation_id))
|
|
AND NOT EXISTS (SELECT 1 FROM native_instances i WHERE i.retired_at IS NULL AND ${runtimeLaunchSql})
|
|
AND NOT (${recoveryLaunch})
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.remove_operation_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_decommissions d WHERE d.operation_id=o.id)
|
|
AND o.kind NOT IN ('runtime.pull','library.download','library.import','library.remove','host.decommission','host.rotate-credential')
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_operations dependent WHERE dependent.state IN ('pending','accepted','running','unknown')
|
|
AND json_extract(dependent.payload,'$.createOperationId')=o.id)`;
|
|
const campaignSafe = `NOT EXISTS (SELECT 1 FROM deployment_servers s WHERE s.deployment_id=o.id AND s.server_deleted_at IS NULL
|
|
AND s.restart_state IN ('pending','held','manual'))
|
|
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id
|
|
WHERE s.kind='campaign' AND s.source_id=o.id AND (${unsettledDeployment}))
|
|
AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.rolled_back_by=o.id)`;
|
|
const failedCampaign = `EXISTS (SELECT 1 FROM deployment_servers s WHERE s.deployment_id=o.id AND s.server_deleted_at IS NULL AND s.restart_state='failed')`;
|
|
|
|
function policies(): Policy[] {
|
|
return [
|
|
{
|
|
category: 'campaign',
|
|
label: 'Campaigns',
|
|
table: 'deployments',
|
|
id: 'id',
|
|
clock: 'started_at',
|
|
days: 'deployKeepDays',
|
|
eligible: `${campaignSafe} AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='file-content' AND c.record_id=o.id AND c.protected=1)`,
|
|
failed: failedCampaign,
|
|
bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)`
|
|
},
|
|
{
|
|
category: 'provisioning',
|
|
label: 'Provisioning workflows',
|
|
table: 'server_deployments',
|
|
id: 'id',
|
|
clock: 'finished_at',
|
|
days: 'provisioningKeepDays',
|
|
failed: "o.state='failed'",
|
|
eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
|
|
AND NOT EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=o.id AND r.healthy_at IS NULL)
|
|
AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id) WHERE n.deployment_id=o.id AND a.state NOT IN ('completed','failed'))
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id WHERE m.deployment_id=o.id AND a.state NOT IN ('completed','failed'))
|
|
AND NOT EXISTS (SELECT 1 FROM deployment_servers s WHERE s.provisioning_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM beta_servers s WHERE s.provisioning_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs s WHERE s.provisioning_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM watchdog_actions s WHERE s.operation_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE o.id IN (c.creation_id,c.cleanup_id))
|
|
AND NOT EXISTS (SELECT 1 FROM server_creation_dependencies c WHERE c.removal_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM rebalancing_moves m WHERE m.operation_id=o.id AND m.state IN ('moving','failed'))
|
|
AND NOT EXISTS (SELECT 1 FROM server_collection_members m JOIN server_collection_members sibling ON sibling.collection_id=m.collection_id
|
|
JOIN server_deployments d ON d.id=sibling.deployment_id WHERE m.deployment_id=o.id AND (${unsettledDeployment}))`,
|
|
bytes: 'COALESCE(length(o.observation_json),0)+(SELECT COALESCE(SUM(length(recipe)),0) FROM server_launch_recipes WHERE deployment_id=o.id)'
|
|
},
|
|
{
|
|
category: 'operation',
|
|
label: 'Agent actions and resolved failures',
|
|
table: 'ec_host_operations',
|
|
id: 'id',
|
|
clock: 'updated_at',
|
|
days: 'actionKeepDays',
|
|
failed: "(o.state='failed' OR (o.routine=1 AND o.observation_success=0))",
|
|
eligible: `o.history_expired_at IS NULL AND o.state IN ('completed','failed') AND (o.routine=0 OR o.observation_success=0) AND ${unreferencedOperation}`,
|
|
bytes: 'COALESCE(length(o.payload),0)+COALESCE(length(o.result),0)+COALESCE((SELECT length(payload) FROM ec_host_operation_payloads WHERE operation_id=o.id),0)'
|
|
},
|
|
{
|
|
category: 'observation',
|
|
label: 'Routine operations',
|
|
table: 'ec_host_operations',
|
|
id: 'id',
|
|
clock: 'updated_at',
|
|
days: 'observationKeepDays',
|
|
eligible: "o.state='completed' AND o.observation_success=1",
|
|
bytes: 'length(o.payload)+COALESCE(length(o.result),0)'
|
|
},
|
|
{
|
|
category: 'maintenance',
|
|
label: 'Maintenance jobs',
|
|
table: 'maintenance_jobs',
|
|
id: 'id',
|
|
clock: 'finished_at',
|
|
days: 'maintenanceKeepDays',
|
|
eligible: `o.kind!='update' AND o.state IN ('done','failed','cancelled') AND NOT EXISTS (SELECT 1 FROM server_deployments d WHERE d.id=o.provisioning_id AND (${unsettledDeployment}))`
|
|
},
|
|
{
|
|
category: 'recovery',
|
|
label: 'Resolved recovery incidents',
|
|
table: 'watchdog_incidents',
|
|
id: 'id',
|
|
clock: 'finished_at',
|
|
days: 'recoveryKeepDays',
|
|
eligible: `o.state='resolved' AND NOT EXISTS (SELECT 1 FROM watchdog_actions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.incident_id=o.id AND (${unsettledDeployment}))
|
|
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a JOIN restart_rollout r ON r.operation_id=a.operation_id WHERE a.incident_id=o.id AND r.healthy_at IS NULL)`
|
|
},
|
|
{
|
|
category: 'match',
|
|
label: 'Matches',
|
|
table: 'leaderboard_matches',
|
|
id: 'id',
|
|
clock: 'last_received_at',
|
|
days: 'matchKeepDays',
|
|
eligible: "o.status IN ('completed','interrupted','end-report-missing')",
|
|
bytes: '0'
|
|
},
|
|
{
|
|
category: 'beta',
|
|
label: 'Completed betas',
|
|
table: 'betas',
|
|
id: 'id',
|
|
clock: 'closed_at',
|
|
days: 'betaKeepDays',
|
|
eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta-file-content' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
|
|
bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)`
|
|
},
|
|
{
|
|
category: 'file-content',
|
|
label: 'Campaign file bodies',
|
|
table: 'deployments',
|
|
id: 'id',
|
|
clock: 'finished_at',
|
|
days: 'fileContentKeepDays',
|
|
eligible: `${campaignSafe} AND NOT (${failedCampaign})
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=o.id AND c.protected=1)
|
|
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`,
|
|
bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)`
|
|
},
|
|
{
|
|
category: 'beta-file-content',
|
|
label: 'Closed beta file snapshots',
|
|
table: 'betas',
|
|
id: 'id',
|
|
clock: 'closed_at',
|
|
days: 'fileContentKeepDays',
|
|
eligible: `o.state IN ('promoted','rolled-back')
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1)
|
|
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT))
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
|
|
bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)`
|
|
},
|
|
{
|
|
category: 'workflow-payload',
|
|
label: 'Completed launch file bodies',
|
|
table: 'server_deployments',
|
|
id: 'id',
|
|
clock: 'finished_at',
|
|
days: 'payloadKeepDays',
|
|
failed: "o.state='failed'",
|
|
eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
|
|
AND NOT EXISTS (SELECT 1 FROM rebalancing_moves m WHERE m.operation_id=o.id AND m.state IN ('moving','failed'))
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=o.id AND c.protected=1)
|
|
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='workflow-payload' AND e.record_id=o.id)`,
|
|
bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type='recipe' AND r.owner_id=o.id)"
|
|
},
|
|
{
|
|
category: 'operation-payload',
|
|
label: 'Completed command file bodies',
|
|
table: 'ec_host_operations',
|
|
id: 'id',
|
|
clock: 'updated_at',
|
|
days: 'payloadKeepDays',
|
|
failed: "o.state='failed'",
|
|
eligible: `o.state IN ('completed','failed') AND o.kind IN ('instance.create','instance.rebuild','instance.restart','instance.start','files.stage','files.read')
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='operation' AND c.record_id=o.id AND c.protected=1)
|
|
AND NOT (${recoveryLaunch})
|
|
AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
|
WHERE o.id IN (n.operation_id,n.stage_operation_id) AND ((${unsettledDeployment}) OR EXISTS (
|
|
SELECT 1 FROM retention_controls c WHERE c.category IN ('provisioning','workflow-payload') AND c.record_id=d.id AND c.protected=1)))
|
|
AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='operation-payload' AND e.record_id=o.id)`,
|
|
bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type IN ('operation','operation-result') AND r.owner_id=o.id)"
|
|
},
|
|
{
|
|
category: 'version',
|
|
label: 'Unused artifact versions',
|
|
table: 'script_versions',
|
|
id: 'id',
|
|
clock: 'uploaded_at',
|
|
days: 'versionKeepDays',
|
|
minimum: false,
|
|
eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id)
|
|
AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id)
|
|
AND ${versionReferences()}`,
|
|
bytes: `${blobSize('o.content_hash')}`
|
|
},
|
|
{
|
|
category: 'audit',
|
|
label: 'Authentication audit events',
|
|
table: 'auth_audit',
|
|
id: 'id',
|
|
clock: 'created_at',
|
|
days: 'auditKeepDays',
|
|
eligible: '1',
|
|
bytes: 'length(o.details)'
|
|
}
|
|
];
|
|
}
|
|
function versionReferences(): string {
|
|
// Even cascading references pin versions: automatic cleanup must never erase
|
|
// another feature's history as a side effect of deleting an artifact.
|
|
const tables = db().prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as {
|
|
name: string;
|
|
}[];
|
|
const clauses: string[] = [];
|
|
for (const { name } of tables) {
|
|
if (name === 'plugin_imports') continue;
|
|
const quote = (s: string) => '"' + s.replaceAll('"', '""') + '"';
|
|
for (const fk of db()
|
|
.prepare(`PRAGMA foreign_key_list(${quote(name)})`)
|
|
.all() as { table: string; from: string }[]) {
|
|
if (fk.table === 'script_versions')
|
|
clauses.push(
|
|
`NOT EXISTS (SELECT 1 FROM ${quote(name)} v WHERE v.${quote(fk.from)}=o.id)`
|
|
);
|
|
}
|
|
}
|
|
return clauses.join(' AND ') || '1';
|
|
}
|
|
|
|
function withoutFileBodies(value: unknown): unknown {
|
|
if (Array.isArray(value)) return value.map(withoutFileBodies);
|
|
if (!value || typeof value !== 'object') return value;
|
|
return Object.fromEntries(
|
|
Object.entries(value)
|
|
.filter(([key]) => !['contentHash', 'contentSha256', 'contentBase64'].includes(key))
|
|
.map(([key, item]) => [key, withoutFileBodies(item)])
|
|
);
|
|
}
|
|
|
|
function expirePayload(category: string, id: string): void {
|
|
if (category === 'workflow-payload') {
|
|
const row = db()
|
|
.prepare('SELECT recipe FROM server_launch_recipes WHERE deployment_id=?')
|
|
.get(id) as { recipe: string } | undefined;
|
|
if (row)
|
|
db()
|
|
.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?')
|
|
.run(JSON.stringify(withoutFileBodies(JSON.parse(row.recipe))), id);
|
|
db()
|
|
.prepare('UPDATE native_provisioning SET files_json=NULL WHERE deployment_id=?')
|
|
.run(id);
|
|
referenceBlobs('recipe', id, []);
|
|
return;
|
|
}
|
|
const row = db()
|
|
.prepare(
|
|
`SELECT COALESCE(p.payload,o.payload) payload,o.result FROM ec_host_operations o
|
|
LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`
|
|
)
|
|
.get(id) as { payload: string; result: string | null };
|
|
const payload = JSON.stringify(withoutFileBodies(JSON.parse(row.payload)));
|
|
db()
|
|
.prepare(
|
|
`UPDATE ec_host_operations SET payload=?,result=?,request_hash=COALESCE(request_hash,?) WHERE id=?`
|
|
)
|
|
.run(
|
|
payload,
|
|
row.result === null ? null : JSON.stringify(withoutFileBodies(JSON.parse(row.result))),
|
|
createHash('sha256').update(row.payload).digest('hex'),
|
|
id
|
|
);
|
|
db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(id);
|
|
referenceBlobs('operation', id, []);
|
|
referenceBlobs('operation-result', id, []);
|
|
}
|
|
function selection(policy: Policy, settings = fleetSettings()) {
|
|
const days = settings[policy.days];
|
|
const failureDays = settings.failureKeepDays;
|
|
const cut = (n: number) => new Date(Date.now() - n * 86400_000).toISOString();
|
|
const age =
|
|
policy.category === 'audit'
|
|
? 'o.created_at < @numericCutoff'
|
|
: `o.${policy.clock} < @cutoff`;
|
|
const failed = policy.failed ?? '0';
|
|
const enabled = days > 0 || (policy.category === 'version' && settings.versionKeepCount > 0);
|
|
// Keep age outside the correlated failure checks so SQLite can seek the
|
|
// clock index before examining dependencies. Resolved failures start from
|
|
// their controls, not another scan of the entire history table.
|
|
const candidates: string[] = [];
|
|
if (enabled)
|
|
candidates.push(`SELECT o.rowid FROM ${policy.table} o
|
|
WHERE ${policy.category === 'version' && !days ? '1' : age} AND NOT (${failed})`);
|
|
if (failureDays > 0 && policy.failed)
|
|
candidates.push(`SELECT o.rowid FROM retention_controls c
|
|
JOIN ${policy.table} o ON o.${policy.id}=c.record_id
|
|
WHERE c.category=@category AND c.resolved_at<@failureCutoff
|
|
AND c.resolved_at>=o.${policy.clock} AND (${failed})`);
|
|
const minimum = policy.minimum === false ? 0 : settings.historyKeepCount;
|
|
const minimumScope =
|
|
policy.category === 'observation'
|
|
? "WHERE observation_success=1 AND state='completed'"
|
|
: policy.category === 'operation'
|
|
? 'WHERE history_expired_at IS NULL AND (routine=0 OR observation_success=0)'
|
|
: '';
|
|
const versionCount =
|
|
policy.category === 'version'
|
|
? `AND o.id NOT IN (SELECT id FROM script_versions v WHERE v.script_id=o.script_id ORDER BY version DESC LIMIT ${settings.versionKeepCount})`
|
|
: '';
|
|
return {
|
|
sql: `FROM ${policy.table} o WHERE ${candidates.length ? `o.rowid IN (${candidates.join(' UNION ALL ')})` : '0'} AND (${policy.eligible})
|
|
AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category=@category AND c.record_id=CAST(o.${policy.id} AS TEXT) AND c.protected=1)
|
|
AND o.rowid NOT IN (SELECT rowid FROM ${policy.table} ${minimumScope} ORDER BY ${policy.clock} DESC,rowid DESC LIMIT ${minimum}) ${versionCount}`,
|
|
params: {
|
|
category: policy.category,
|
|
cutoff: cut(days),
|
|
numericCutoff: Date.now() - days * 86400_000,
|
|
failureCutoff: cut(failureDays)
|
|
}
|
|
};
|
|
}
|
|
// better-sqlite3 accepts unused named bindings, keeping selection reusable.
|
|
export function retentionPreview(settings = fleetSettings()) {
|
|
return policies().map((policy) => {
|
|
const query = selection(policy, settings);
|
|
const summary = db()
|
|
.prepare(
|
|
`SELECT COUNT(*) AS eligible,COALESCE(SUM(COALESCE(${policy.bytes ?? '0'},0)),0) AS payloadBytes ${query.sql}`
|
|
)
|
|
.get(query.params) as { eligible: number; payloadBytes: number };
|
|
const records = db()
|
|
.prepare(
|
|
`SELECT CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 20`
|
|
)
|
|
.all(query.params) as { id: string }[];
|
|
return { category: policy.category, label: policy.label, ...summary, records };
|
|
});
|
|
}
|
|
export function retentionControls() {
|
|
return db()
|
|
.prepare(
|
|
'SELECT category,record_id AS id,protected,resolved_at AS resolvedAt FROM retention_controls WHERE protected=1 OR resolved_at IS NOT NULL ORDER BY category,record_id'
|
|
)
|
|
.all();
|
|
}
|
|
export function setRetentionControl(input: {
|
|
category: string;
|
|
id: string;
|
|
protected?: boolean;
|
|
resolve?: boolean;
|
|
}) {
|
|
const policy =
|
|
policies().find((policy) => policy.category === input.category) ??
|
|
(['game-file', 'upload'].includes(input.category)
|
|
? { category: input.category, table: 'game_file_versions', id: 'id', failed: undefined }
|
|
: undefined);
|
|
if (
|
|
!policy ||
|
|
typeof input.id !== 'string' ||
|
|
!input.id ||
|
|
input.id.length > 200 ||
|
|
(input.protected !== undefined && typeof input.protected !== 'boolean') ||
|
|
(input.resolve !== undefined && typeof input.resolve !== 'boolean')
|
|
)
|
|
throw new Error('Choose a valid history category and record ID.');
|
|
const row = db()
|
|
.prepare(
|
|
`SELECT 1 FROM ${policy.table} o WHERE CAST(o.${policy.id} AS TEXT)=? ${policy.table === 'ec_host_operations' ? 'AND o.history_expired_at IS NULL' : ''} ${input.resolve ? `AND (${policy.failed ?? '0'})` : ''}`
|
|
)
|
|
.get(input.id);
|
|
if (!row)
|
|
throw new Error(
|
|
input.resolve
|
|
? 'Only a failed record can be marked resolved.'
|
|
: 'History record was not found.'
|
|
);
|
|
if (
|
|
input.protected &&
|
|
(db()
|
|
.prepare('SELECT 1 FROM retention_expirations WHERE category=? AND record_id=?')
|
|
.get(input.category, input.id) ||
|
|
(['game-file', 'upload'].includes(input.category) &&
|
|
db().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(input.id)))
|
|
) {
|
|
throw new Error(
|
|
'These files have expired or cleanup has already started. Protection cannot restore them.'
|
|
);
|
|
}
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO retention_controls(category,record_id,protected,resolved_at) VALUES(?,?,?,?)
|
|
ON CONFLICT(category,record_id) DO UPDATE SET protected=COALESCE(?,protected),resolved_at=COALESCE(?,resolved_at)`
|
|
)
|
|
.run(
|
|
input.category,
|
|
input.id,
|
|
Number(input.protected ?? false),
|
|
input.resolve ? new Date().toISOString() : null,
|
|
input.protected === undefined ? null : Number(input.protected),
|
|
input.resolve ? new Date().toISOString() : null
|
|
);
|
|
}
|
|
|
|
/** Each policy removes at most 500 eligible records per pass; dependencies are checked again under the transaction. */
|
|
export function pruneHistory(only?: string): number {
|
|
if (only === 'observation') return pruneRoutineObservations();
|
|
let removed = only ? 0 : pruneRoutineObservations();
|
|
for (const policy of policies().filter(
|
|
(policy) => policy.category !== 'observation' && (!only || policy.category === only)
|
|
)) {
|
|
removed += db().transaction(() => {
|
|
const query = selection(policy);
|
|
const rows = db()
|
|
.prepare(
|
|
`SELECT o.rowid AS rid,CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 500`
|
|
)
|
|
.all(query.params) as { rid: number; id: string }[];
|
|
for (const row of rows) {
|
|
if (policy.category === 'provisioning') {
|
|
db()
|
|
.prepare('DELETE FROM native_provisioning WHERE deployment_id=?')
|
|
.run(row.id);
|
|
db()
|
|
.prepare('DELETE FROM ec_host_migrations WHERE deployment_id=?')
|
|
.run(row.id);
|
|
db()
|
|
.prepare(
|
|
'DELETE FROM restart_rollout WHERE operation_id=? AND healthy_at IS NOT NULL'
|
|
)
|
|
.run(row.id);
|
|
}
|
|
if (policy.category === 'operation' || policy.category === 'observation') {
|
|
db()
|
|
.prepare('DELETE FROM ec_host_operation_events WHERE operation_id=?')
|
|
.run(row.id);
|
|
if (policy.category === 'operation') {
|
|
const payload = db()
|
|
.prepare(
|
|
`SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`
|
|
)
|
|
.get(row.id) as { payload: string };
|
|
db()
|
|
.prepare(
|
|
`UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?),payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`
|
|
)
|
|
.run(
|
|
createHash('sha256').update(payload.payload).digest('hex'),
|
|
new Date().toISOString(),
|
|
row.id
|
|
);
|
|
}
|
|
db()
|
|
.prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?')
|
|
.run(row.id);
|
|
}
|
|
if (
|
|
policy.category === 'workflow-payload' ||
|
|
policy.category === 'operation-payload'
|
|
) {
|
|
expirePayload(policy.category, row.id);
|
|
db()
|
|
.prepare(
|
|
'INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)'
|
|
)
|
|
.run(policy.category, row.id, new Date().toISOString());
|
|
} else if (policy.category === 'beta-file-content') {
|
|
db()
|
|
.prepare('UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?')
|
|
.run(row.id);
|
|
db()
|
|
.prepare(
|
|
'UPDATE beta_file_snapshots SET prior_content_hash=NULL WHERE beta_id=?'
|
|
)
|
|
.run(row.id);
|
|
db()
|
|
.prepare(
|
|
'INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)'
|
|
)
|
|
.run(policy.category, row.id, new Date().toISOString());
|
|
} else if (policy.category === 'file-content') {
|
|
db()
|
|
.prepare(
|
|
'UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?'
|
|
)
|
|
.run(row.id);
|
|
db()
|
|
.prepare(
|
|
"UPDATE staged_file_changes SET content_hash=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')"
|
|
)
|
|
.run(row.id);
|
|
db()
|
|
.prepare(
|
|
'INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)'
|
|
)
|
|
.run(policy.category, row.id, new Date().toISOString());
|
|
} else if (policy.category !== 'operation') {
|
|
db().prepare(`DELETE FROM ${policy.table} WHERE rowid=?`).run(row.rid);
|
|
db()
|
|
.prepare('DELETE FROM retention_controls WHERE category=? AND record_id=?')
|
|
.run(policy.category, row.id);
|
|
}
|
|
}
|
|
return rows.length;
|
|
})();
|
|
}
|
|
const blobs = pruneFileBlobs();
|
|
if ((removed || blobs) && Number(db().pragma('freelist_count', { simple: true })) >= 256)
|
|
db().pragma('incremental_vacuum(256)');
|
|
return removed;
|
|
}
|
|
|
|
function samplePolicies(s = fleetSettings()) {
|
|
const iso = (days: number) => new Date(Date.now() - days * 86400_000).toISOString();
|
|
return [
|
|
{
|
|
category: 'host-resources',
|
|
label: 'Host resource samples',
|
|
table: 'ec_host_resource_samples',
|
|
condition: 'sampled_at < ?',
|
|
value: iso(s.hostResourceKeepDays),
|
|
enabled: s.hostResourceKeepDays > 0
|
|
},
|
|
{
|
|
category: 'host-temperatures',
|
|
label: 'Host temperature samples',
|
|
table: 'ec_host_temperature_samples',
|
|
condition: 'bucket < ?',
|
|
value: iso(s.hostResourceKeepDays),
|
|
enabled: s.hostResourceKeepDays > 0
|
|
},
|
|
{
|
|
category: 'instance-resources',
|
|
label: 'Instance resource samples',
|
|
table: 'ec_instance_resource_samples',
|
|
condition: 'bucket < ?',
|
|
value: iso(Math.max(1, s.instanceResourceKeepDays)),
|
|
enabled: s.instanceResourceKeepDays > 0
|
|
},
|
|
{
|
|
category: 'metrics',
|
|
label: 'Fleet metrics samples',
|
|
table: 'metric_samples',
|
|
condition: 'at < ?',
|
|
value: Date.now() - s.metricsKeepDays * 86400_000,
|
|
enabled: s.metricsKeepDays > 0
|
|
},
|
|
{
|
|
category: 'telemetry',
|
|
label: 'Telemetry diagnostics (age expiry)',
|
|
table: 'telemetry_requests',
|
|
condition: 'received_at < ?',
|
|
value: iso(s.telemetryKeepDays),
|
|
enabled: s.telemetryKeepDays > 0
|
|
}
|
|
];
|
|
}
|
|
export function sampleRetentionPreview(settings = fleetSettings()) {
|
|
const result = samplePolicies(settings)
|
|
.filter((p) => p.category !== 'telemetry')
|
|
.map((p) => ({
|
|
category: p.category,
|
|
label: p.label,
|
|
records: [],
|
|
payloadBytes: 0,
|
|
eligible: p.enabled
|
|
? (
|
|
db()
|
|
.prepare(`SELECT COUNT(*) AS n FROM ${p.table} WHERE ${p.condition}`)
|
|
.get(p.value) as { n: number }
|
|
).n
|
|
: 0
|
|
}));
|
|
const telemetry = db()
|
|
.prepare(
|
|
`SELECT COUNT(*) AS eligible,COALESCE(SUM(bytes),0) AS payloadBytes FROM (
|
|
SELECT received_at,length(payload) AS bytes,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS per_server,
|
|
ROW_NUMBER() OVER (ORDER BY id DESC) AS overall FROM telemetry_requests)
|
|
WHERE (? > 0 AND received_at < ?) OR (? > 0 AND per_server > ?) OR (? > 0 AND overall > ?)`
|
|
)
|
|
.get(
|
|
settings.telemetryKeepDays,
|
|
new Date(Date.now() - settings.telemetryKeepDays * 86400_000).toISOString(),
|
|
settings.telemetryMaxPerServer,
|
|
settings.telemetryMaxPerServer,
|
|
settings.telemetryMaxTotal,
|
|
settings.telemetryMaxTotal
|
|
) as { eligible: number; payloadBytes: number };
|
|
result.push({
|
|
category: 'telemetry',
|
|
label: 'Telemetry diagnostics (age and count limits)',
|
|
records: [],
|
|
...telemetry
|
|
});
|
|
return result;
|
|
}
|
|
export function pruneSampleHistory() {
|
|
let removed = 0;
|
|
for (const p of samplePolicies().filter((p) => p.enabled))
|
|
removed += db()
|
|
.prepare(
|
|
`DELETE FROM ${p.table} WHERE rowid IN (SELECT rowid FROM ${p.table} WHERE ${p.condition} LIMIT 5000)`
|
|
)
|
|
.run(p.value).changes;
|
|
const s = fleetSettings();
|
|
if (s.telemetryMaxPerServer)
|
|
removed += db()
|
|
.prepare(
|
|
`DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM
|
|
(SELECT id,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS rank FROM telemetry_requests) WHERE rank>? LIMIT 5000)`
|
|
)
|
|
.run(s.telemetryMaxPerServer).changes;
|
|
if (s.telemetryMaxTotal)
|
|
removed += db()
|
|
.prepare(
|
|
'DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 5000 OFFSET ?)'
|
|
)
|
|
.run(s.telemetryMaxTotal).changes;
|
|
return removed;
|
|
}
|
|
|
|
/** Drain every eligible batch, yielding between transactions so reads and receipts can run. */
|
|
export async function drainRetention(): Promise<number> {
|
|
let total = 0,
|
|
logCursor = 0;
|
|
while (true) {
|
|
const removed = pruneHistory() + pruneSampleHistory();
|
|
total += removed;
|
|
if (!removed) break;
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
}
|
|
// Remove expired rows first; only compact bodies in history that will survive.
|
|
if (db().prepare("SELECT 1 FROM settings WHERE key='log_history_metadata_v1'").get())
|
|
return total;
|
|
while (true) {
|
|
const rows = db()
|
|
.prepare(
|
|
`SELECT rowid AS rid,id,result FROM ec_host_operations
|
|
WHERE rowid>? AND kind='logs.read' AND json_valid(result)
|
|
AND (json_type(result,'$.lines')='array' OR json_type(result,'$.contentBase64')='text'
|
|
OR json_type(result,'$.contentSha256')='text')
|
|
ORDER BY rowid LIMIT 50`
|
|
)
|
|
.all(logCursor) as { rid: number; id: string; result: string }[];
|
|
db().transaction(() => {
|
|
for (const row of rows) {
|
|
db()
|
|
.prepare('UPDATE ec_host_operations SET result=? WHERE id=?')
|
|
.run(JSON.stringify(logResultMetadata(JSON.parse(row.result))), row.id);
|
|
db()
|
|
.prepare(
|
|
"DELETE FROM file_blob_references WHERE owner_type='operation-result' AND owner_id=?"
|
|
)
|
|
.run(row.id);
|
|
}
|
|
if (!rows.length)
|
|
db()
|
|
.prepare(
|
|
"INSERT OR IGNORE INTO settings(key,value) VALUES('log_history_metadata_v1','1')"
|
|
)
|
|
.run();
|
|
})();
|
|
if (!rows.length) return total;
|
|
logCursor = rows.at(-1)!.rid;
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
}
|
|
}
|
|
|
|
function pruneRoutineObservations(): number {
|
|
const settings = fleetSettings();
|
|
if (!settings.observationKeepDays) return 0;
|
|
const handle = db();
|
|
return handle.transaction(() => {
|
|
const rows = handle
|
|
.prepare(
|
|
`SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed'
|
|
AND updated_at<? AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='observation' AND c.record_id=ec_host_operations.id AND c.protected=1)
|
|
${settings.historyKeepCount ? `AND id NOT IN (SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed' ORDER BY updated_at DESC LIMIT ${settings.historyKeepCount})` : ''}
|
|
ORDER BY updated_at LIMIT 500`
|
|
)
|
|
.all(new Date(Date.now() - settings.observationKeepDays * 86400_000).toISOString()) as {
|
|
id: string;
|
|
}[];
|
|
if (!rows.length) return 0;
|
|
const ids = JSON.stringify(rows.map((row) => row.id));
|
|
handle
|
|
.prepare(
|
|
'DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT value FROM json_each(?))'
|
|
)
|
|
.run(ids);
|
|
handle
|
|
.prepare(
|
|
'DELETE FROM ec_host_operation_payloads WHERE operation_id IN (SELECT value FROM json_each(?))'
|
|
)
|
|
.run(ids);
|
|
return handle
|
|
.prepare('DELETE FROM ec_host_operations WHERE id IN (SELECT value FROM json_each(?))')
|
|
.run(ids).changes;
|
|
})();
|
|
}
|