From 3952c4c938875af7d03f366146b25d9e7ef0b715 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 5 Oct 2026 22:48:04 -0400 Subject: [PATCH] Add configurable history and storage retention policies --- README.md | 5 +- docs/maintenance.md | 2 +- docs/metrics.md | 6 +- docs/native-hosts.md | 8 +- docs/settings.md | 115 ++++++++ src/lib/fleet-settings.ts | 100 ++++++- src/lib/server/auth/access.ts | 1 + src/lib/server/db/schema.sql | 10 + src/lib/server/fleet/chat.ts | 6 +- src/lib/server/fleet/metrics.ts | 4 +- src/lib/server/fleet/rollback.ts | 5 +- src/lib/server/fleet/telemetry-requests.ts | 23 +- src/lib/server/game-files/index.ts | 78 +++++- src/lib/server/game-files/schema.ts | 2 + src/lib/server/game-files/uploads.ts | 1 + src/lib/server/hosts/calibration.ts | 5 +- src/lib/server/hosts/index.ts | 32 +-- src/lib/server/hosts/schema.ts | 2 + src/lib/server/maintenance/runtime.ts | 4 +- src/lib/server/pipeline/sweep.ts | 10 +- src/lib/server/pipeline/worker.ts | 6 +- src/lib/server/retention.ts | 246 ++++++++++++++++++ src/lib/ui/components/RetentionTools.svelte | 67 +++++ .../ui/components/TelemetryDiagnostics.svelte | 2 +- src/routes/api/campaigns/[id]/+server.ts | 3 +- .../campaigns/[id]/file/[fileId]/+server.ts | 3 + .../[name]/telemetry-requests/+server.ts | 5 +- src/routes/api/settings/retention/+server.ts | 11 + .../api/settings/retention/preview/+server.ts | 10 + src/routes/campaigns/[id]/+page.svelte | 6 +- src/routes/settings/+page.svelte | 8 +- tests/helpers/settings.ts | 8 + tests/unit/fleet-settings.test.ts | 4 +- tests/unit/game-file-browser-upload.test.ts | 56 +++- tests/unit/game-file-library.test.ts | 5 +- tests/unit/retention.test.ts | 184 +++++++++++++ tests/unit/telemetry-diagnostics.test.ts | 2 +- 37 files changed, 963 insertions(+), 82 deletions(-) create mode 100644 docs/settings.md create mode 100644 src/lib/server/retention.ts create mode 100644 src/lib/ui/components/RetentionTools.svelte create mode 100644 src/routes/api/settings/retention/+server.ts create mode 100644 src/routes/api/settings/retention/preview/+server.ts create mode 100644 tests/helpers/settings.ts create mode 100644 tests/unit/retention.test.ts diff --git a/README.md b/README.md index 8ae1609..a0e8f83 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,8 @@ time remaining for the current step. Interrupted transfers resume. Failed prepar retries automatically with backoff; the host page reports any blocking condition. Set `PLUTAINER_IMAGE` to change the default runtime prepared on hosts. Existing servers retain their existing files and runtime without a forced restart. -Previous uploads, verified host copies and server references are preserved. +Upload history and server references are preserved; unused version files follow the +configured retention policy. Existing named libraries appear together under their game, with no renaming or reupload required. Multiple instances on one host share read-only base files. @@ -139,6 +140,8 @@ affect occupancy or game history. Existing JSON payloads are unchanged. ## Documentation +- [Fleet concurrency and retention settings](docs/settings.md) + - [Native hosts, storage and operation recovery](docs/native-hosts.md) - [Users, roles and authentication](docs/authentication.md) - [Provisioning and safety queues](docs/provisioning.md) diff --git a/docs/maintenance.md b/docs/maintenance.md index fd5b8d2..dbb40bb 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -98,7 +98,7 @@ Schedules survive application restarts. Missed days, including while globally paused, become one run on resume. Pending work is deduplicated per server. The worker starts on the first application request, so keep the app running and reachable. Finished scheduled and legacy maintenance activity is retained for -30 days; update attempt records remain durable. +30 days by default (configurable in Administration → Settings); update attempt records remain durable. ## Health recovery and configuration drift diff --git a/docs/metrics.md b/docs/metrics.md index 3dd1c23..177b74d 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -45,8 +45,8 @@ represents the current registration rather than every instance in the history. identical filters. Utilization changes are percentage points; player-hours changes are relative percentages. A previous observed zero can show **New**. Missing evidence remains unavailable. Partial retained history is labelled; - a 30-day comparison usually has little previous history because retention is - only 31 days. Both periods' known time is shown. Bounds cannot establish that + a 30-day comparison usually has little previous history because default retention is + 31 days. Both periods' known time is shown. Bounds cannot establish that collection was uninterrupted, and changes are not adjusted for fleet size. - **Activity heatmap** groups captured time by weekday and hour in its displayed timezone. Color represents occupied / known server time. Exact cell values @@ -82,7 +82,7 @@ Observations are recorded by the existing status worker. Each remains valid unti Capacity comes from `sv_maxclients` / `sv_maxClients` in the status response. When unavailable, the UI displays unknown rather than assuming a four-player limit. Capacity coverage is the map time for which every tracked lobby reported both occupancy and capacity. -Collection begins after this feature is installed; previous occupancy cannot be reconstructed. Samples are retained for 31 days and pruned during collection. They survive removal of a server and contain no player identities. The additive database table is created automatically on startup. +Collection begins after this feature is installed; previous occupancy cannot be reconstructed. Samples default to 31 days of retention; Administration → Settings can change this period. Cleanup runs during collection and in the background. They survive removal of a server and contain no player identities. The additive database table is created automatically on startup. `GET /api/metrics` supplies the same report as the page and disables response caching. Query parameters are `days` (1, 7, or 30), `game`, `map` (requires game), diff --git a/docs/native-hosts.md b/docs/native-hosts.md index 567fded..6c23a4b 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -140,7 +140,7 @@ host-wide busy time across all cores on a 0–100% scale between inventory sampl (normally about 15 seconds). Idle and I/O wait are excluded; stolen CPU time counts as unavailable capacity. The first sample after startup, unavailable counters, and older agents show **Not reported**. Offline hosts retain their -last reported value. The controller retains CPU and memory samples for seven days (missing measurements +last reported value. The controller defaults to retaining CPU and memory samples for seven days (missing measurements remain unknown). Scheduling checks capacity against the latest heartbeat and uses the last five minutes to rank eligible hosts; the Inventory panel shows the latest reading. @@ -171,7 +171,7 @@ Sampling failures appear on the host's Learned server capacity panel. Existing containers can begin contributing running samples as soon as the agent updates; startup samples require observing the first five minutes of a container's run. -The controller retains per-instance minute peaks for 24 hours and caches learned +The controller defaults to retaining per-instance minute peaks for 24 hours and caches learned CPU/memory allowances. The first five minutes after a container starts count as startup; later samples train the running allowance. Startup uses the observed maximum and running uses the 95th percentile of minute peaks, with **no per-server @@ -471,7 +471,9 @@ controller round trips do not stall every transferred chunk. Corrupt or incomplete content never becomes Ready. Cache inventory is reconciled on heartbeats, including vanished caches. The latest published version of each game is always retained. With agent 1.4.7 or newer, an older host copy is automatically -removed once no live instance reservation on that host references it. Stopped, +eligible for removal once no live instance reservation on that host references it. +The game-file minimum count, supersession grace period and explicit protections in +Administration → Settings also apply. Stopped, provisioning, and rebuilding instances still pin their version. The agent also refuses deletion if any Docker container (including stopped containers) mounts those files. Cleanup includes partial downloads and retries after interruption. diff --git a/docs/settings.md b/docs/settings.md new file mode 100644 index 0000000..dd30821 --- /dev/null +++ b/docs/settings.md @@ -0,0 +1,115 @@ +# Fleet settings + +Administration → Settings has independent **Concurrency** and **Retention** +sections. Settings persist in SQLite, apply without restarting the controller, +and are writable only by administrators. Saving one section preserves the other +section's saved values. Upgrades preserve existing values and add new defaults. + +## Concurrency + +The existing fleet restart, startup and destruction limits default to 1. Additional +limits default to 0 (no additional cap): + +- Restarts per host, including starts and rebuild recovery. +- Startups per host, including creation and rebuilds. +- Destructions per host, including migration source removal. +- Combined lifecycle work across create/start/stop/restart/rebuild/destroy. +- Rebuilds across the fleet, including host drain migrations. +- Hosts downloading or verifying game files simultaneously, including manual imports. + +Limits are checked again before command submission after asynchronous preflight. +Launch-file preparation reserves capacity. A lifecycle workflow counts once in the +combined pool; unresolved commands and recovery keep capacity occupied. Migrations +keep rebuild capacity through source removal and destination creation. Lower limits +do not cancel admitted work. Game-file preparation slots are assigned on agent +command delivery and survive interrupted transfers. Runtime pulls, the four file +transfers per host and the two health probes per host remain unchanged. + +## Retention + +For age settings, 0 means no expiry. An explicit version-count policy can still +expire unused artifact versions when their age policy is disabled. Game-file grace +is a delay: 0 adds no delay. A positive age and version count together preserve the +minimum count as well as versions younger than the age threshold. + +| History or storage | Default | +| --- | --- | +| Campaigns | 7 days | +| Successful routine operations | 1 day | +| Provisioning workflows | Indefinite | +| Agent actions | Indefinite | +| Explicitly resolved failures | Indefinite | +| Non-update maintenance jobs | 30 days | +| Resolved recovery incidents | Indefinite | +| Host resource history | 7 days | +| Instance resource samples | 1 day | +| Fleet occupancy metrics | 31 days | +| Telemetry diagnostics | 7 days; 200 per server; 10,000 overall | +| Completed/interrupted matches | Indefinite | +| Closed betas and test evidence | Indefinite | +| Campaign file bodies and closed beta snapshots | No separate expiry | +| Unused artifact versions | No automatic expiry; no minimum count | +| Unused game files | No grace period; keep latest 1 version per game | +| Incomplete browser uploads | Indefinite | +| Authentication audit events | Indefinite | +| Minimum history count | 0 additional records | + +Telemetry age and count policies are independent; 0 disables each individual cap. +Diagnostic responses remain bounded to 10,000 rows when the per-server retention +cap is disabled. Metric and capacity chart time ranges are unchanged; increasing +storage retention does not automatically add new chart ranges. Capacity learning +continues to use its 24-hour window independently of longer sample retention. + +### Cleanup and dependencies + +Background cleanup runs in bounded batches. New age policies for previously +permanent history default to indefinite. Pending work, unresolved failures, +recovery dependencies and explicitly protected records are never removed merely +because they are old. Referenced artifact versions are retained, including versions +referenced by historical campaigns and betas. Collection workflows with unfinished +members, cancellation links and recovery slots can keep provisioning history longer +than its configured age. + +Update attempts and dismissed watchdog incidents remain durable suppression evidence. +Host setup, credential/decommission operations and referenced agent commands are +protected. Expired unreferenced action history drops payloads, results and events +but keeps compact terminal receipts and request fingerprints. Reusing an idempotency +key cannot execute an expired command again. Agent mutation journals remain intact; +this policy controls controller history, not the agent's execution recovery journal. + +Expiring campaign file bodies preserves file paths, hashes and summary records but +makes file comparisons and rollback unavailable. The detail page identifies this +state and rollback requests fail explicitly. Active beta rollback images stay +protected. Expiring matches removes their player participation rows and changes +leaderboard/player totals; lifetime aggregates are not retained separately. + +Game-file minimums apply per game. Grace starts when a version is superseded. +Live, stopped, rebuilding and provisioning instances pin their files; unresolved +operations and offline host copies also prevent controller deletion. Controller +cleanup still waits for every host to report its copy absent. Incomplete browser +uploads expire based on their last activity, including resume checks; active chunk +writes and publication hold locks against cleanup. Metadata remains after files +expire. Raising retention cannot restore previously deleted bytes. + +### Preview and record protection + +Expand **Cleanup preview and protected records** to preview the values currently +entered without saving them. Preview lists eligible records and known payload +sizes; SQLite file space may be reused rather than immediately returned to the OS. +Policies can overlap, so payload sizes should not be added together. Unknown row, +index, upload and agent-disk overhead is not included. + +Use a record's ID from its detail URL or the preview to protect it. Protection can +be removed in the same panel. Protecting a campaign or beta also protects its file +bodies; protecting those bodies prevents deletion of their parent history. +Game-file and incomplete-upload protections apply to storage cleanup. + +The minimum history count preserves recent records per history category. It does +not apply to telemetry caps, metric/resource samples or game-file storage, which +have their own policies. Explicit holds protect individual history/storage records, +not individual resource samples or telemetry diagnostics. + +Failed campaigns, provisioning workflows and agent commands require explicit +resolution before the resolved-failure period applies. Resolution does not bypass +recovery/dependency checks. Resolving a failed campaign stops automatic campaign +retry requests; it does not cancel commands already submitted. diff --git a/src/lib/fleet-settings.ts b/src/lib/fleet-settings.ts index 8e64790..091fe70 100644 --- a/src/lib/fleet-settings.ts +++ b/src/lib/fleet-settings.ts @@ -1,4 +1,26 @@ export interface FleetSettings { + provisioningKeepDays: number; + actionKeepDays: number; + failureKeepDays: number; + maintenanceKeepDays: number; + recoveryKeepDays: number; + hostResourceKeepDays: number; + instanceResourceKeepDays: number; + metricsKeepDays: number; + telemetryKeepDays: number; + telemetryMaxPerServer: number; + telemetryMaxTotal: number; + matchKeepDays: number; + betaKeepDays: number; + fileContentKeepDays: number; + versionKeepDays: number; + versionKeepCount: number; + gameFileGraceDays: number; + gameFileKeepCount: number; + uploadKeepDays: number; + auditKeepDays: number; + historyKeepCount: number; + deployKeepDays: number; observationKeepDays: number; restartConcurrency: number; @@ -14,6 +36,28 @@ export interface FleetSettings { } export const defaultFleetSettings: FleetSettings = { + provisioningKeepDays: 0, + actionKeepDays: 0, + failureKeepDays: 0, + maintenanceKeepDays: 30, + recoveryKeepDays: 0, + hostResourceKeepDays: 7, + instanceResourceKeepDays: 1, + metricsKeepDays: 31, + telemetryKeepDays: 7, + telemetryMaxPerServer: 200, + telemetryMaxTotal: 10000, + matchKeepDays: 0, + betaKeepDays: 0, + fileContentKeepDays: 0, + versionKeepDays: 0, + versionKeepCount: 0, + gameFileGraceDays: 0, + gameFileKeepCount: 1, + uploadKeepDays: 0, + auditKeepDays: 0, + historyKeepCount: 0, + deployKeepDays: 7, observationKeepDays: 1, restartConcurrency: 1, @@ -28,6 +72,28 @@ export const defaultFleetSettings: FleetSettings = { }; export const fleetSettingLabels: Record = { + provisioningKeepDays: 'Provisioning history retention', + actionKeepDays: 'Action operation history retention', + failureKeepDays: 'Resolved failure history retention', + maintenanceKeepDays: 'Maintenance history retention', + recoveryKeepDays: 'Resolved recovery incident retention', + hostResourceKeepDays: 'Host resource history retention', + instanceResourceKeepDays: 'Instance resource sample retention', + metricsKeepDays: 'Fleet metrics history retention', + telemetryKeepDays: 'Telemetry diagnostic history retention', + telemetryMaxPerServer: 'Maximum telemetry records per server', + telemetryMaxTotal: 'Maximum telemetry records overall', + matchKeepDays: 'Match history retention', + betaKeepDays: 'Completed beta history retention', + fileContentKeepDays: 'File diffs and rollback snapshot retention', + versionKeepDays: 'Unused artifact version retention', + versionKeepCount: 'Minimum artifact versions to keep', + gameFileGraceDays: 'Unused game-file grace period', + gameFileKeepCount: 'Minimum game-file versions to keep', + uploadKeepDays: 'Abandoned upload retention', + auditKeepDays: 'Authentication audit history retention', + historyKeepCount: 'Minimum history records to keep', + deployKeepDays: 'Campaign history retention', observationKeepDays: 'Routine operation history retention', restartConcurrency: 'Concurrent restarts', @@ -50,11 +116,11 @@ export function validateFleetSettings(input: unknown): FleetSettings { } for (const key of Object.keys(defaultFleetSettings) as (keyof FleetSettings)[]) { const value = values[key]; - const minimum = key === 'observationKeepDays' || additionalConcurrencyKeys.includes(key) ? 0 : 1; + const minimum = settingMinimum(key); if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) { throw new Error(`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`); } - if ((key === 'deployKeepDays' || key === 'observationKeepDays') && value > 36500) { + if (key.endsWith('Days') && value > 36500) { throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`); } } @@ -63,8 +129,34 @@ export function validateFleetSettings(input: unknown): FleetSettings { export const additionalConcurrencyKeys: (keyof FleetSettings)[] = ["restartConcurrencyPerHost", "createConcurrencyPerHost", "destroyConcurrencyPerHost", "lifecycleConcurrency", "rebuildConcurrency", "preparationConcurrency"]; export const concurrencyKeys: (keyof FleetSettings)[] = ['restartConcurrency', 'createConcurrency', 'destroyConcurrency', ...additionalConcurrencyKeys]; -export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays']; +export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays', 'provisioningKeepDays', 'actionKeepDays', 'failureKeepDays', 'maintenanceKeepDays', 'recoveryKeepDays', 'hostResourceKeepDays', 'instanceResourceKeepDays', 'metricsKeepDays', 'telemetryKeepDays', 'telemetryMaxPerServer', 'telemetryMaxTotal', 'matchKeepDays', 'betaKeepDays', 'fileContentKeepDays', 'versionKeepDays', 'versionKeepCount', 'gameFileGraceDays', 'gameFileKeepCount', 'uploadKeepDays', 'auditKeepDays', 'historyKeepCount']; +export function settingMinimum(key: keyof FleetSettings): number { + return ['restartConcurrency','createConcurrency','destroyConcurrency','gameFileKeepCount'].includes(key) ? 1 : 0; +} + export const settingHelp: Partial> = { + provisioningKeepDays: 'Completed lifecycle history. Recovery, source workflows and cancellation dependencies remain protected.', + actionKeepDays: 'Completed agent actions. Referenced commands and retry evidence remain protected. Expired actions keep compact receipts to prevent duplicate execution.', + failureKeepDays: 'Explicitly resolved failed operations and campaigns, measured from resolution. Unresolved failures remain protected.', + maintenanceKeepDays: 'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.', + recoveryKeepDays: 'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.', + hostResourceKeepDays: 'Host CPU and memory history.', + instanceResourceKeepDays: 'Minute resource samples. Capacity learning continues to use its independent 24-hour window.', + metricsKeepDays: 'Historical fleet occupancy and utilization samples.', + telemetryKeepDays: 'Raw received telemetry diagnostics. Independent of saved matches. Count limits also apply.', + telemetryMaxPerServer: 'Count cap per server, in addition to the age limit. 0 disables this count cap.', + telemetryMaxTotal: 'Fleet-wide count cap for telemetry diagnostics. 0 disables this count cap.', + matchKeepDays: 'Completed and interrupted matches and player details. Expiry changes leaderboards and player totals; no lifetime aggregates are preserved.', + betaKeepDays: 'Closed betas, revisions and test evidence, after linked campaigns and recovery work no longer need them.', + fileContentKeepDays: 'Expired campaign file bodies are removed while summaries remain. Rollback becomes unavailable for those campaigns. Active beta snapshots remain protected.', + versionKeepDays: 'Unused script, plugin, configuration and baseline versions. Current and referenced versions are always protected.', + versionKeepCount: 'Keep at least this many recent versions per artifact. With no age expiry, a positive count removes older unused versions.', + gameFileGraceDays: 'Delay removal after a version becomes superseded. Latest files, referenced versions and offline host copies remain protected. 0 adds no delay.', + gameFileKeepCount: 'Keep this many recent published versions per game, in addition to every referenced or protected version.', + uploadKeepDays: 'Remove incomplete browser uploads after this many days without activity. Active transfers and publication are protected.', + auditKeepDays: 'Recorded authentication and account-administration events.', + historyKeepCount: 'Keep at least this many recent records per history category. Does not apply to metrics, resource samples, telemetry count caps or game-file storage.', + restartConcurrency: 'Fleet recovery slots stay occupied until container and game health are verified.', createConcurrency: 'Fleet startup slots cover creation and rebuilds through health and selected-map verification.', destroyConcurrency: 'Fleet removals hold capacity until teardown and cleanup finish.', @@ -74,6 +166,6 @@ export const settingHelp: Partial> = { lifecycleConcurrency: 'Combined fleet limit for creates, starts, stops, restarts, rebuilds and destructions. Each workflow counts once.', rebuildConcurrency: 'Additional fleet limit for rebuilds, including host drain migrations.', preparationConcurrency: 'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.', - deployKeepDays: 'Eligible campaign records and file diffs. Pending, failed, held and recovery-linked campaigns are protected.', + deployKeepDays: 'Eligible campaign records and file diffs. Pending, unresolved failed, held and recovery-linked campaigns are protected.', observationKeepDays: 'Successful monitoring and file-read records. Failures and actions are retained. 0 keeps records indefinitely.' }; diff --git a/src/lib/server/auth/access.ts b/src/lib/server/auth/access.ts index b57197e..7396fe9 100644 --- a/src/lib/server/auth/access.ts +++ b/src/lib/server/auth/access.ts @@ -29,6 +29,7 @@ export function routePermission(routeId: string | null, method: string): RoutePe || within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin'; if (!reading && ['/api/hosts', '/api/settings', '/api/game-files'].some(base => within(routeId, base))) return 'admin'; + if (within(routeId, '/api/settings/retention')) return 'admin'; if (routeId === '/api/provisioning/[id]/override') return 'admin'; // These endpoints mix operational actions with global policy and schedule // changes. Per-incident/job actions remain available to operators. diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index e59c301..dd955bf 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -730,3 +730,13 @@ CREATE TABLE IF NOT EXISTS watchdog_actions ( cycle INTEGER NOT NULL DEFAULT 1, UNIQUE(incident_id,cycle,action) ); + +-- Explicit history holds and failure resolution never bypass dependency checks. +CREATE TABLE IF NOT EXISTS retention_controls ( + category TEXT NOT NULL, record_id TEXT NOT NULL, protected INTEGER NOT NULL DEFAULT 0, + resolved_at TEXT, PRIMARY KEY(category,record_id) +); +CREATE TABLE IF NOT EXISTS retention_expirations ( + category TEXT NOT NULL, record_id TEXT NOT NULL, expired_at TEXT NOT NULL, + PRIMARY KEY(category,record_id) +); diff --git a/src/lib/server/fleet/chat.ts b/src/lib/server/fleet/chat.ts index 9da4395..794aefb 100644 --- a/src/lib/server/fleet/chat.ts +++ b/src/lib/server/fleet/chat.ts @@ -1,7 +1,7 @@ import { db } from '../db/index.js'; import { enrichGameChatTimeline, type ChatMatch, type ChatRoundReport, type ContextualChatEntry } from '../../game-chat.js'; import { parseGameEvent } from '../../telemetry.js'; -import { TELEMETRY_REQUEST_RETENTION } from '../../telemetry-diagnostics.js'; +import { telemetryRetention, telemetryCutoff } from './telemetry-requests.js'; import type { ServerRow } from './servers.js'; import type { TailResult } from './logs.js'; @@ -15,7 +15,7 @@ export function serverChatTimeline(server: ServerRow, tail: TailResult): Context // Diagnostic history survives deletion. Its receipt time must belong to the // current registration so a reused server name cannot inherit old rounds. - const cutoff = new Date(Date.now() - TELEMETRY_REQUEST_RETENTION.maxAgeDays * 86_400_000).toISOString(); + const cutoff = telemetryCutoff(); const reports = handle.prepare(`SELECT received_at AS receivedAt, event, round, player_count AS playerCount FROM telemetry_requests WHERE server = ? AND status = 200 AND received_at >= ? @@ -23,7 +23,7 @@ export function serverChatTimeline(server: ServerRow, tail: TailResult): Context AND event IN ('game_start', 'round_change', 'heartbeat', 'game_end') AND player_count >= 0 AND (event = 'game_start' OR round >= 0) ORDER BY id DESC LIMIT ?`).all(server.name, cutoff, server.created_at, - TELEMETRY_REQUEST_RETENTION.maxPerServer) as ChatRoundReport[]; + (telemetryRetention().maxPerServer || 10000)) as ChatRoundReport[]; // Saving gameplay does not depend on diagnostic logging succeeding. Keep the // surviving accepted snapshot useful when its request log was not recorded. diff --git a/src/lib/server/fleet/metrics.ts b/src/lib/server/fleet/metrics.ts index 9ace2d1..16d579d 100644 --- a/src/lib/server/fleet/metrics.ts +++ b/src/lib/server/fleet/metrics.ts @@ -1,3 +1,4 @@ +import { fleetSettings } from '../settings.js'; import { db } from '../db/index.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; import { parseMetricsQuery, type HeatmapCell, type MapMetrics, type MetricPoint, type MetricsData, @@ -18,7 +19,8 @@ export function recordMetrics(samples: MetricSample[], at = Date.now()): void { handle.transaction(() => { const insert = handle.prepare('INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)'); for (const s of samples) insert.run(at, s.server, s.game, s.map, s.players, s.capacity); - handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - 31 * 86400_000); + const days = fleetSettings().metricsKeepDays; + if (days) handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - days * 86400_000); })(); } diff --git a/src/lib/server/fleet/rollback.ts b/src/lib/server/fleet/rollback.ts index 67089cb..b0e9a13 100644 --- a/src/lib/server/fleet/rollback.ts +++ b/src/lib/server/fleet/rollback.ts @@ -11,7 +11,7 @@ import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } // pre-multi-script legacy rows. export class RollbackError extends Error { - constructor(message: string, public readonly kind: 'no-such-deployment' | 'not-promote' | 'already-rolled-back') { + constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back') { super(message); } } @@ -72,6 +72,9 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise new Date(Date.now() - TELEMETRY_REQUEST_RETENTION.maxAgeDays * 86_400_000).toISOString(); +export function telemetryRetention() { + const settings = fleetSettings(); + return { maxAgeDays: settings.telemetryKeepDays, maxPerServer: settings.telemetryMaxPerServer, maxTotal: settings.telemetryMaxTotal }; +} +export const telemetryCutoff = () => telemetryRetention().maxAgeDays ? new Date(Date.now() - telemetryRetention().maxAgeDays * 86_400_000).toISOString() : ''; +const cutoff = telemetryCutoff; const textField = (value: unknown) => typeof value === 'string' ? value.slice(0, 255) : null; export function recordTelemetryRequest(input: { @@ -31,13 +36,13 @@ export function recordTelemetryRequest(input: { Array.isArray(body.players) ? body.players.length : null, textField(input.contentType), payload.slice(0, MAX_PAYLOAD_LENGTH), Number(payload.length > MAX_PAYLOAD_LENGTH) ); - handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff()); - handle.prepare(`DELETE FROM telemetry_requests WHERE server = ? AND id <= ( + if (telemetryRetention().maxAgeDays) handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff()); + if (telemetryRetention().maxPerServer) handle.prepare(`DELETE FROM telemetry_requests WHERE server = ? AND id <= ( SELECT id FROM telemetry_requests WHERE server = ? ORDER BY id DESC LIMIT 1 OFFSET ? - )`).run(body.server, body.server, TELEMETRY_REQUEST_RETENTION.maxPerServer); - handle.prepare(`DELETE FROM telemetry_requests WHERE id <= ( + )`).run(body.server, body.server, telemetryRetention().maxPerServer); + if (telemetryRetention().maxTotal) handle.prepare(`DELETE FROM telemetry_requests WHERE id <= ( SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 1 OFFSET ? - )`).run(MAX_REQUESTS); + )`).run(telemetryRetention().maxTotal); })(); } @@ -45,6 +50,6 @@ export function listTelemetryRequests(server: string): TelemetryRequestLogEntry[ const rows = db().prepare(`SELECT id, server, received_at AS receivedAt, status, error, event, map, round, player_count AS playerCount, content_type AS contentType, payload, payload_truncated AS payloadTruncated FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`) - .all(server, cutoff(), TELEMETRY_REQUEST_RETENTION.maxPerServer) as Array & { payloadTruncated: number }>; + .all(server, cutoff(), telemetryRetention().maxPerServer || 10000) as Array & { payloadTruncated: number }>; return rows.map(row => ({ ...row, payloadTruncated: !!row.payloadTruncated })); } diff --git a/src/lib/server/game-files/index.ts b/src/lib/server/game-files/index.ts index e40c7b7..165e00c 100644 --- a/src/lib/server/game-files/index.ts +++ b/src/lib/server/game-files/index.ts @@ -1,3 +1,4 @@ +import { fleetSettings } from '../settings.js'; import { randomUUID } from 'node:crypto'; import { mkdir, rename, stat, rm, lstat } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; @@ -7,7 +8,7 @@ import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hos import { ensureGameFileSchema } from './schema.js'; import { ensureHostSchema } from '../hosts/schema.js'; import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js'; -import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js'; +import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock, uploadIsActive } from './uploads.js'; import { GAME_FILE_GAMES } from '../../game-files.js'; import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState, LibraryTransferProgress } from '../../game-files.js'; @@ -137,6 +138,7 @@ export function createLibraryUpload(input: Record): GameLibrary type UploadEntry = { path: string; size: number; lastModified: number }; function requireBrowserUpload(id: string): GameLibraryVersion { const version = getLibraryVersion(id); + if (database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(id)) throw Object.assign(new Error('Upload files expired under the retention policy.'), { status: 410 }); if (!version || version.sourceHostId !== null || !database().prepare('SELECT 1 FROM game_file_upload_entries WHERE version_id=? LIMIT 1').get(id)) { throw Object.assign(new Error('Folder upload not found.'), { status: 404 }); } @@ -153,6 +155,7 @@ async function uploadStorageRoot(version: GameLibraryVersion): Promise { } export async function browserUploadStatus(id: string) { const version = requireBrowserUpload(id); + touchUpload(id); const status = version.state === 'ready' ? 'published' as const : publishing.has(id) ? 'verifying' as const : version.state === 'failed' ? 'failed' as const : 'pending' as const; const root = await uploadStorageRoot(version); const entries = uploadEntries(id); @@ -169,15 +172,18 @@ export async function uploadBrowserFile(id: string, path: string, chunk: { offse if (version.state === 'ready' || version.manifestSha256 || publishing.has(id)) throw Object.assign(new Error('This upload is no longer writable.'), { status: 409 }); }; checkWritable(); + touchUpload(id); const file = database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? AND path=?').get(id, path) as UploadEntry | undefined; if (!file) throw new Error('File is not listed in this upload.'); const progress = await receiveUploadChunk(stagingRoot(id), file, chunk, body, checkWritable); database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id); + touchUpload(id); return progress; } export async function publishBrowserUpload(id: string): Promise { const version = requireBrowserUpload(id); if (version.state === 'ready') return version; + touchUpload(id); const existing = publishing.get(id); if (existing) return existing; const task = withUploadLock(stagingRoot(id), async () => { const root = await uploadStorageRoot(version); @@ -241,6 +247,7 @@ export function createLibraryImport(input: Record): GameLibrary export function requireImportOwner(hostId: string, id: string): GameLibraryVersion { const version = getLibraryVersion(id); + if (database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(id)) throw Object.assign(new Error('Upload files expired under the retention policy.'), { status: 410 }); if (!version || version.sourceHostId !== hostId) throw new Error('Import not found for this host.'); return version; } @@ -473,7 +480,7 @@ export function reconcileHostLibraryCleanup(hostId: string): void { const host = getHost(hostId); if (!host?.connected || !host.credentialActive || host.mode === 'disabled' || host.decommission || !host.inventory.capabilities?.includes('library.remove')) return; - const latest = new Set(latestPublishedLibraries().map(version => version.id)); + const latest = retainedGameFileIds(); const caches = handle.prepare('SELECT * FROM host_library_caches WHERE host_id=?').all(hostId) as CacheRow[]; for (const cache of caches) { if (latest.has(cache.version_id) || (cache.status === 'missing' && cache.bytes_downloaded === 0) || libraryInUse(cache.version_id, hostId)) continue; @@ -493,7 +500,8 @@ export function cleanupControllerLibraries(): Promise { const handle = database(); const current = cleaning.get(handle); if (current) return current; const task = (async () => { - const latest = new Set(latestPublishedLibraries().map(version => version.id)); + await cleanupAbandonedUploads(); + const latest = retainedGameFileIds(); const versions = handle.prepare(`SELECT v.id,v.storage_location,l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id LEFT JOIN game_file_cleanup c ON c.version_id=v.id WHERE v.state='ready' AND c.removed_at IS NULL`).all() as { id: string; storage_location: string; game: string }[]; @@ -521,3 +529,67 @@ export function cleanupControllerLibraries(): Promise { void task.finally(() => cleaning.delete(handle)).catch(() => {}); return task; } + +function touchUpload(id: string) { + database().prepare("UPDATE game_file_versions SET last_activity_at=? WHERE id=? AND state!='ready'").run(new Date().toISOString(),id); +} + +/** Latest N published versions are retained per game; grace starts at supersession. */ +export function retainedGameFileIds(settings = fleetSettings()): Set { + const versions = database().prepare(`SELECT v.id,l.game,v.published_at FROM game_file_versions v + JOIN game_file_libraries l ON l.id=v.library_id WHERE v.state='ready' + ORDER BY v.published_at DESC,v.created_at DESC,v.rowid DESC`).all() as { id: string; game: string; published_at: string }[]; + const kept = new Set(); + const seen = new Map(); + for (const v of versions) { + const prior = seen.get(v.game); + if (!prior || prior.count < settings.gameFileKeepCount + || Date.parse(prior.newerAt) > Date.now() - settings.gameFileGraceDays * 86400_000) kept.add(v.id); + seen.set(v.game,{ count: (prior?.count ?? 0) + 1, newerAt: v.published_at }); + } + for (const row of database().prepare("SELECT record_id FROM retention_controls WHERE category='game-file' AND protected=1").all() as { record_id: string }[]) kept.add(row.record_id); + return kept; +} + +function abandonedUploads(settings = fleetSettings()) { + const days = settings.uploadKeepDays; + if (!days) return []; + return database().prepare(`SELECT v.id,v.storage_location,l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id + WHERE v.state IN ('importing','failed') AND v.source_host_id='' AND COALESCE(v.last_activity_at,v.created_at) { + // A chunk or resume request may have refreshed the timestamp before lock admission. + if (!abandonedUploads().some(row => row.id === upload.id)) return; + if (!/^[a-zA-Z0-9_-]+$/.test(upload.id)) return; + const expected = join(gameFilesRoot(),upload.game,upload.id); + if (resolve(upload.storage_location) !== expected) return; + database().prepare('INSERT OR IGNORE INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(upload.id,new Date().toISOString()); + // Paths are controller-derived. Validate parents before recursive removal. + for (const path of [gameFilesRoot(),join(gameFilesRoot(),'.imports'),join(gameFilesRoot(),upload.game),expected,stagingRoot(upload.id),`${stagingRoot(upload.id)}.chunks`]) { + const info = await lstat(path).catch(cause => { if (cause.code === 'ENOENT') return null; throw cause; }); + if (info?.isSymbolicLink()) throw new Error('Refusing upload cleanup through a symlink.'); + } + for (const path of [expected,stagingRoot(upload.id),`${stagingRoot(upload.id)}.chunks`]) await rm(path,{ recursive: true,force: true }); + database().prepare("UPDATE game_file_versions SET state='failed',error='Upload expired under the retention policy.' WHERE id=? AND state!='ready'").run(upload.id); + database().prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(),upload.id); + }); + } +} + +export function gameFileRetentionPreview(settings = fleetSettings()) { + const kept = retainedGameFileIds(settings); + const versions = database().prepare(`SELECT v.id,v.size_bytes FROM game_file_versions v LEFT JOIN game_file_cleanup c ON c.version_id=v.id + WHERE v.state='ready' AND c.removed_at IS NULL`).all() as { id: string; size_bytes: number }[]; + const eligible = versions.filter(v => !kept.has(v.id) && !libraryInUse(v.id) && !database().prepare("SELECT 1 FROM host_library_caches WHERE version_id=? AND (status!='missing' OR bytes_downloaded>0)").get(v.id)); + const uploads = abandonedUploads(settings).filter(v => !publishing.has(v.id) && !uploadIsActive(stagingRoot(v.id))); + return [ + { category: 'game-file',label: 'Unused controller game files',eligible: eligible.length,payloadBytes: eligible.reduce((n,v)=>n+v.size_bytes,0),records: eligible.slice(0,20).map(v=>({ id: v.id })) }, + { category: 'upload',label: 'Abandoned uploads',eligible: uploads.length,payloadBytes: 0,records: uploads.slice(0,20).map(v=>({ id: v.id })) } + ]; +} diff --git a/src/lib/server/game-files/schema.ts b/src/lib/server/game-files/schema.ts index 65c5e66..12146d7 100644 --- a/src/lib/server/game-files/schema.ts +++ b/src/lib/server/game-files/schema.ts @@ -49,6 +49,8 @@ export function ensureGameFileSchema(handle: Database.Database): void { CREATE TRIGGER IF NOT EXISTS game_file_versions_immutable BEFORE UPDATE ON game_file_versions WHEN OLD.state = 'ready' BEGIN SELECT RAISE(ABORT, 'Published game-file versions are immutable'); END; `); + const versionColumns = handle.prepare('PRAGMA table_info(game_file_versions)').all() as { name: string }[]; + if (!versionColumns.some(column => column.name === 'last_activity_at')) handle.exec('ALTER TABLE game_file_versions ADD COLUMN last_activity_at TEXT'); const cacheColumns = handle.prepare('PRAGMA table_info(host_library_caches)').all() as { name: string }[]; if (!cacheColumns.some(column => column.name === 'bytes_verified')) handle.exec('ALTER TABLE host_library_caches ADD COLUMN bytes_verified INTEGER'); if (!cacheColumns.some(column => column.name === 'transfer_progress')) handle.exec('ALTER TABLE host_library_caches ADD COLUMN transfer_progress TEXT'); diff --git a/src/lib/server/game-files/uploads.ts b/src/lib/server/game-files/uploads.ts index 8252ed5..51f8d0c 100644 --- a/src/lib/server/game-files/uploads.ts +++ b/src/lib/server/game-files/uploads.ts @@ -46,6 +46,7 @@ export function validateUploadPlan(input: unknown): UploadFilePlan[] { interface UploadLocks { barrier: Promise; writers: number; files: Map>; active: Set> } const uploadLocks = new Map(); +export function uploadIsActive(root: string): boolean { return uploadLocks.has(resolve(root)); } function locksFor(key: string): UploadLocks { let locks = uploadLocks.get(key); if (!locks) { locks = { barrier: Promise.resolve(), writers: 0, files: new Map(), active: new Set() }; uploadLocks.set(key, locks); } diff --git a/src/lib/server/hosts/calibration.ts b/src/lib/server/hosts/calibration.ts index 6f2a1ce..0a86147 100644 --- a/src/lib/server/hosts/calibration.ts +++ b/src/lib/server/hosts/calibration.ts @@ -1,3 +1,4 @@ +import { fleetSettings } from '../settings.js'; import { db } from '../db/index.js'; import { ensureHostSchema } from './schema.js'; import type { CapacityCalibration, CapacitySample, CapacitySamplePage, HostInventory, ResourceAllowance } from '../../hosts.js'; @@ -89,8 +90,8 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples, window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`) .run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp); - if (scope === '*') handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket 36500) return 0; - const handle = database(); - const cutoff = new Date(Date.now() - days * 86_400_000).toISOString(); - return handle.transaction(() => { - const rows = handle.prepare(`SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed' - AND updated_at row.id)); - handle.prepare('DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT value FROM json_each(?))').run(ids); - handle.prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id IN (SELECT value FROM json_each(?))').run(ids); - return handle.prepare('DELETE FROM ec_host_operations WHERE id IN (SELECT value FROM json_each(?))').run(ids).changes; - })(); -} +export function pruneHostObservations(): number { return pruneHistory('observation'); } /** Heartbeats can tell an executing create to stop at its next safe command boundary. */ export function pendingCreationCancellations(hostId: string): string[] { return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations @@ -464,11 +451,12 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.'); const operation = database().transaction(() => { if (options.idempotencyKey) { - const existing = database().prepare(`SELECT o.id,o.kind,o.instance_id,o.generation,${operationPayloadSql}=CAST(? AS TEXT) AS same_payload + const existing = database().prepare(`SELECT o.id,o.kind,o.instance_id,o.generation,o.history_expired_at,o.request_hash,${operationPayloadSql}=CAST(? AS TEXT) AS same_payload FROM ec_host_operations o WHERE o.host_id=? AND o.idempotency_key=?`).get(payload, hostId, options.idempotencyKey) as - { id: string; kind: string; instance_id: string | null; generation: number | null; same_payload: number } | undefined; + { id: string; kind: string; instance_id: string | null; generation: number | null; history_expired_at: string | null; request_hash: string | null; same_payload: number } | undefined; if (existing) { - if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || !existing.same_payload) throw new Error('Operation idempotency key is already associated with a different request.'); + const samePayload = existing.history_expired_at ? existing.request_hash === createHash('sha256').update(payload).digest('hex') : !!existing.same_payload; + if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || !samePayload) throw new Error('Operation idempotency key is already associated with a different request.'); return getHostOperationStatus(existing.id)!; } } diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts index 72c70d0..1e0cddf 100644 --- a/src/lib/server/hosts/schema.ts +++ b/src/lib/server/hosts/schema.ts @@ -88,6 +88,8 @@ export function ensureHostSchema(handle: Database.Database): void { ); `); const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[]; + if (!columns.some(column => column.name === 'history_expired_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN history_expired_at TEXT'); + if (!columns.some(column => column.name === 'request_hash')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN request_hash TEXT'); if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT'); if (!columns.some(column => column.name === 'observation_success')) { handle.transaction(() => { diff --git a/src/lib/server/maintenance/runtime.ts b/src/lib/server/maintenance/runtime.ts index 59ecfdb..86d3b8c 100644 --- a/src/lib/server/maintenance/runtime.ts +++ b/src/lib/server/maintenance/runtime.ts @@ -120,9 +120,7 @@ async function runTick(): Promise { try { await processJob(job); } catch { hold(job, 'Maintenance check failed; waiting for the next check.'); } } - // Update attempts are also durable suppression evidence; never age them into an automatic retry. - db().prepare(`DELETE FROM maintenance_jobs WHERE kind != 'update' AND finished_at < ? AND state IN ('done','failed','cancelled')`) - .run(new Date(Date.now() - 30 * 24 * HOUR).toISOString()); + } async function processJob(job: MaintenanceJob): Promise { const server = getServer(job.server); diff --git a/src/lib/server/pipeline/sweep.ts b/src/lib/server/pipeline/sweep.ts index a84a3af..e308de0 100644 --- a/src/lib/server/pipeline/sweep.ts +++ b/src/lib/server/pipeline/sweep.ts @@ -1,4 +1,4 @@ -import { fleetSettings } from '../settings.js'; +import { pruneHistory } from '../retention.js'; import { db } from '../db/index.js'; import { emit } from './bus.js'; import { requestRestart } from '../provisioning/index.js'; @@ -22,7 +22,8 @@ export function sweep(): Promise { async function runSweep(): Promise { pruneHostObservations(); const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id - WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[]; + WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed') + AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=ds.deployment_id AND c.resolved_at IS NOT NULL) ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[]; const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 }; for (const row of pending) { if (!row.files_changed) continue; // A failed file write is not a restart request. @@ -46,10 +47,7 @@ async function runSweep(): Promise { } db().prepare(`UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS ( SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`).run(nowIso()); - db().prepare(`DELETE FROM deployments WHERE started_at < ? - AND NOT EXISTS (SELECT 1 FROM watchdog_actions WHERE campaign_id = deployments.id) AND NOT EXISTS ( - SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`) - .run(new Date(Date.now() - fleetSettings().deployKeepDays * 86_400_000).toISOString()); + pruneHistory('campaign'); emit({ type: 'sweep-tick', frozen: false }); return result; } diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index 976dcfe..6c86f01 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -1,3 +1,4 @@ +import { pruneHistory, pruneSampleHistory } from '../retention.js'; import { config } from '../config.js'; import { processProvisioning } from '../provisioning/worker.js'; import { sweep } from './sweep.js'; @@ -51,13 +52,14 @@ function reconcileRequesters(): void { ['campaign', 'Reconciling campaigns', sweep], ['beta', 'Reconciling beta tests', betaTick], ['maintenance', 'Checking maintenance and recovery', runMaintenance], - ['game-files', 'Removing unused game files', cleanupControllerLibraries] + ['game-files', 'Removing unused game files', cleanupControllerLibraries], + ['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); }] ]; for (const [key, label, work] of requesters) { if (requesterFlights.has(key)) continue; const flight = trackActivity(key, label, work) .catch(err => { console.error(`[worker] ${key} error:`, err); }) - .then(async () => { if (key !== 'game-files') await tick(true); }) + .then(async () => { if (key !== 'game-files' && key !== 'retention') await tick(true); }) .finally(() => { requesterFlights.delete(key); }); requesterFlights.set(key, flight); } diff --git a/src/lib/server/retention.ts b/src/lib/server/retention.ts new file mode 100644 index 0000000..d0569d1 --- /dev/null +++ b/src/lib/server/retention.ts @@ -0,0 +1,246 @@ +import { createHash } from 'node:crypto'; +import { db } from './db/index.js'; +import { fleetSettings } from './settings.js'; +import { ensureHostSchema } from './hosts/schema.js'; +import { ensureGameFileSchema } from './game-files/schema.js'; +import { ensureAuthSchema } from './auth/schema.js'; +import type { FleetSettings } from '../fleet-settings.js'; + +interface Policy { + category: string; label: string; table: string; id: string; clock: string; + days: keyof FleetSettings; eligible: string; bytes?: string; failed?: string; minimum?: boolean; +} +const terminal = "('done','warning','failed','cancelled')"; +// Cancellation stops observation, not an already-delivered command. Its source +// history and before images must survive until the receipt has been reconciled. +const unsettledDeployment = `d.state NOT IN ${terminal} + OR EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=d.id AND r.healthy_at IS NULL) + OR EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id) + LEFT JOIN native_instances i ON i.id=n.instance_id WHERE n.deployment_id=d.id AND + (a.state NOT IN ('completed','failed') OR (a.id=n.operation_id AND a.state='completed' AND ( + (n.applied_at IS NULL AND COALESCE(json_array_length(n.files_json),0)>0) + OR (d.action='destroy' AND i.retired_at IS NULL) + OR (d.action='rebuild' AND i.retired_at IS NULL AND i.generation=n.generation + AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id)))))) + OR EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id + WHERE m.deployment_id=d.id AND (a.state NOT IN ('completed','failed') OR (a.state='completed' AND m.removed_at IS NULL)))`; +const unreferencedOperation = `NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE o.id IN (n.operation_id,n.stage_operation_id)) + AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.remove_operation_id=o.id) + AND NOT EXISTS (SELECT 1 FROM ec_host_decommissions d WHERE d.operation_id=o.id) + AND o.kind NOT IN ('runtime.pull','library.download','library.import','library.remove','host.decommission','host.rotate-credential') + AND NOT EXISTS (SELECT 1 FROM ec_host_operations dependent WHERE dependent.state IN ('pending','accepted','running','unknown') + AND json_extract(dependent.payload,'$.createOperationId')=o.id)`; +const campaignSafe = `NOT EXISTS (SELECT 1 FROM deployment_servers s WHERE s.deployment_id=o.id AND s.server_deleted_at IS NULL + AND s.restart_state IN ('pending','held','manual')) + AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=o.id) + AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id + WHERE s.kind='campaign' AND s.source_id=o.id AND (${unsettledDeployment})) + AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.rolled_back_by=o.id)`; +const failedCampaign = `EXISTS (SELECT 1 FROM deployment_servers s WHERE s.deployment_id=o.id AND s.server_deleted_at IS NULL AND s.restart_state='failed')`; + +function policies(): Policy[] { + return [ + { category: 'campaign', label: 'Campaigns', table: 'deployments', id: 'id', clock: 'started_at', days: 'deployKeepDays', + eligible: `${campaignSafe} AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='file-content' AND c.record_id=o.id AND c.protected=1)`, failed: failedCampaign, + bytes: `(SELECT COALESCE(SUM(COALESCE(length(content_before),0)+COALESCE(length(content_after),0)),0) FROM deployment_files WHERE deployment_id=o.id)` }, + { category: 'provisioning', label: 'Provisioning workflows', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'provisioningKeepDays', failed: "o.state='failed'", eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')}) + AND NOT EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=o.id AND r.healthy_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id) WHERE n.deployment_id=o.id AND a.state NOT IN ('completed','failed')) + AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id WHERE m.deployment_id=o.id AND a.state NOT IN ('completed','failed')) + AND NOT EXISTS (SELECT 1 FROM deployment_servers s WHERE s.provisioning_id=o.id) + AND NOT EXISTS (SELECT 1 FROM beta_servers s WHERE s.provisioning_id=o.id) + AND NOT EXISTS (SELECT 1 FROM maintenance_jobs s WHERE s.provisioning_id=o.id) + AND NOT EXISTS (SELECT 1 FROM watchdog_actions s WHERE s.operation_id=o.id) + AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE o.id IN (c.creation_id,c.cleanup_id)) + AND NOT EXISTS (SELECT 1 FROM server_creation_dependencies c WHERE c.removal_id=o.id) + AND NOT EXISTS (SELECT 1 FROM server_collection_members m JOIN server_collection_members sibling ON sibling.collection_id=m.collection_id + JOIN server_deployments d ON d.id=sibling.deployment_id WHERE m.deployment_id=o.id AND (${unsettledDeployment}))`, + bytes: "COALESCE(length(o.observation_json),0)+(SELECT COALESCE(SUM(length(recipe)),0) FROM server_launch_recipes WHERE deployment_id=o.id)" }, + { category: 'operation', label: 'Agent actions and resolved failures', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'actionKeepDays', + failed: "(o.state='failed' OR (o.routine=1 AND o.observation_success=0))", + eligible: `o.history_expired_at IS NULL AND o.state IN ('completed','failed') AND (o.routine=0 OR o.observation_success=0) AND ${unreferencedOperation}`, + bytes: "COALESCE(length(o.payload),0)+COALESCE(length(o.result),0)+COALESCE((SELECT length(payload) FROM ec_host_operation_payloads WHERE operation_id=o.id),0)" }, + { category: 'observation', label: 'Routine operations', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'observationKeepDays', eligible: "o.state='completed' AND o.observation_success=1", bytes: 'length(o.payload)+COALESCE(length(o.result),0)' }, + { category: 'maintenance', label: 'Maintenance jobs', table: 'maintenance_jobs', id: 'id', clock: 'finished_at', days: 'maintenanceKeepDays', + eligible: `o.kind!='update' AND o.state IN ('done','failed','cancelled') AND NOT EXISTS (SELECT 1 FROM server_deployments d WHERE d.id=o.provisioning_id AND (${unsettledDeployment}))` }, + { category: 'recovery', label: 'Resolved recovery incidents', table: 'watchdog_incidents', id: 'id', clock: 'finished_at', days: 'recoveryKeepDays', + eligible: `o.state='resolved' AND NOT EXISTS (SELECT 1 FROM watchdog_actions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.incident_id=o.id AND (${unsettledDeployment})) + AND NOT EXISTS (SELECT 1 FROM watchdog_actions a JOIN restart_rollout r ON r.operation_id=a.operation_id WHERE a.incident_id=o.id AND r.healthy_at IS NULL)` }, + { category: 'match', label: 'Matches', table: 'leaderboard_matches', id: 'id', clock: 'last_received_at', days: 'matchKeepDays', eligible: "o.status IN ('completed','interrupted')", bytes: '0' }, + { category: 'beta', label: 'Completed betas', table: 'betas', id: 'id', clock: 'closed_at', days: 'betaKeepDays', + eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta-file-content' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id) + AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`, + bytes: '(SELECT COALESCE(SUM(length(content)),0) FROM beta_revision_files WHERE beta_id=o.id)' }, + { category: 'file-content', label: 'Campaign file bodies', table: 'deployments', id: 'id', clock: 'finished_at', days: 'fileContentKeepDays', + eligible: `${campaignSafe} AND NOT (${failedCampaign}) + AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=o.id AND c.protected=1) + AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`, + bytes: '(SELECT COALESCE(SUM(COALESCE(length(content_before),0)+COALESCE(length(content_after),0)),0) FROM deployment_files WHERE deployment_id=o.id)' }, + { category: 'beta-file-content', label: 'Closed beta file snapshots', table: 'betas', id: 'id', clock: 'closed_at', days: 'fileContentKeepDays', + eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id) + AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) + AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT)) + AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`, + bytes: '(SELECT COALESCE(SUM(length(content)),0) FROM beta_revision_files WHERE beta_id=o.id)' }, + { category: 'version', label: 'Unused artifact versions', table: 'script_versions', id: 'id', clock: 'uploaded_at', days: 'versionKeepDays', minimum: false, + eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id) + AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id) + AND ${versionReferences()}`, bytes: 'length(o.content)' }, + { category: 'audit', label: 'Authentication audit events', table: 'auth_audit', id: 'id', clock: 'created_at', days: 'auditKeepDays', eligible: '1', bytes: 'length(o.details)' } + ]; +} +function versionReferences(): string { + // Even cascading references pin versions: automatic cleanup must never erase + // another feature's history as a side effect of deleting an artifact. + const tables = db().prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as { name: string }[]; + const clauses: string[] = []; + for (const { name } of tables) { + if (name === 'plugin_imports') continue; + const quote = (s: string) => '"' + s.replaceAll('"','""') + '"'; + for (const fk of db().prepare(`PRAGMA foreign_key_list(${quote(name)})`).all() as { table: string; from: string }[]) { + if (fk.table === 'script_versions') clauses.push(`NOT EXISTS (SELECT 1 FROM ${quote(name)} v WHERE v.${quote(fk.from)}=o.id)`); + } + } + return clauses.join(' AND ') || '1'; +} +function initialize() { ensureHostSchema(db()); ensureAuthSchema(db()); } +function selection(policy: Policy, settings = fleetSettings()) { + const days = settings[policy.days]; + const failureDays = settings.failureKeepDays; + const cut = (n: number) => new Date(Date.now() - n * 86400_000).toISOString(); + const age = ['observation','operation'].includes(policy.category) ? `o.${policy.clock} < @cutoff` : policy.category === 'audit' ? "o.created_at < @numericCutoff" : `julianday(o.${policy.clock}) < julianday(@cutoff)`; + const failed = policy.failed ?? '0'; + const enabled = days > 0 || policy.category === 'version' && settings.versionKeepCount > 0; + const normal = `(${enabled ? 1 : 0} AND NOT (${failed}) AND ${policy.category === 'version' && !days ? '1' : age})`; + const failure = `(${failureDays > 0 && !!policy.failed ? 1 : 0} AND (${failed}) AND EXISTS (SELECT 1 FROM retention_controls c WHERE c.category=@category AND c.record_id=CAST(o.${policy.id} AS TEXT) AND julianday(c.resolved_at)>=julianday(o.${policy.clock}) AND julianday(c.resolved_at) { + const query = selection(policy, settings); + const summary = db().prepare(`SELECT COUNT(*) AS eligible,COALESCE(SUM(COALESCE(${policy.bytes ?? '0'},0)),0) AS payloadBytes ${query.sql}`).get(query.params) as { eligible: number; payloadBytes: number }; + const records = db().prepare(`SELECT CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 20`).all(query.params) as { id: string }[]; + return { category: policy.category, label: policy.label, ...summary, records }; + }); +} +export function retentionControls() { + return db().prepare('SELECT category,record_id AS id,protected,resolved_at AS resolvedAt FROM retention_controls WHERE protected=1 OR resolved_at IS NOT NULL ORDER BY category,record_id').all(); +} +export function setRetentionControl(input: { category: string; id: string; protected?: boolean; resolve?: boolean }) { + initialize(); + ensureGameFileSchema(db()); + const policy = policies().find(policy => policy.category === input.category) + ?? (['game-file','upload'].includes(input.category) ? { category: input.category, table: 'game_file_versions',id: 'id',failed: undefined } : undefined); + if (!policy || typeof input.id !== 'string' || !input.id || input.id.length > 200 + || (input.protected !== undefined && typeof input.protected !== 'boolean') || (input.resolve !== undefined && typeof input.resolve !== 'boolean')) throw new Error('Choose a valid history category and record ID.'); + const row = db().prepare(`SELECT 1 FROM ${policy.table} o WHERE CAST(o.${policy.id} AS TEXT)=? ${policy.table === 'ec_host_operations' ? 'AND o.history_expired_at IS NULL' : ''} ${input.resolve ? `AND (${policy.failed ?? '0'})` : ''}`).get(input.id); + if (!row) throw new Error(input.resolve ? 'Only a failed record can be marked resolved.' : 'History record was not found.'); + if (input.protected && (db().prepare('SELECT 1 FROM retention_expirations WHERE category=? AND record_id=?').get(input.category,input.id) + || ['game-file','upload'].includes(input.category) && db().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(input.id))) { + throw new Error('These files have expired or cleanup has already started. Protection cannot restore them.'); + } + db().prepare(`INSERT INTO retention_controls(category,record_id,protected,resolved_at) VALUES(?,?,?,?) + ON CONFLICT(category,record_id) DO UPDATE SET protected=COALESCE(?,protected),resolved_at=COALESCE(?,resolved_at)`) + .run(input.category,input.id,Number(input.protected ?? false),input.resolve ? new Date().toISOString() : null, + input.protected === undefined ? null : Number(input.protected),input.resolve ? new Date().toISOString() : null); +} + +/** Each policy removes at most 500 eligible records per pass; dependencies are checked again under the transaction. */ +export function pruneHistory(only?: string): number { + initialize(); + if (only === 'observation') return pruneRoutineObservations(); + let removed = only ? 0 : pruneRoutineObservations(); + for (const policy of policies().filter(policy => policy.category !== 'observation' && (!only || policy.category === only))) { + removed += db().transaction(() => { + const query = selection(policy); + const rows = db().prepare(`SELECT o.rowid AS rid,CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 500`).all(query.params) as { rid: number; id: string }[]; + for (const row of rows) { + if (policy.category === 'provisioning') { + db().prepare('DELETE FROM native_provisioning WHERE deployment_id=?').run(row.id); + db().prepare('DELETE FROM ec_host_migrations WHERE deployment_id=?').run(row.id); + db().prepare('DELETE FROM restart_rollout WHERE operation_id=? AND healthy_at IS NOT NULL').run(row.id); + } + if (policy.category === 'operation' || policy.category === 'observation') { + db().prepare('DELETE FROM ec_host_operation_events WHERE operation_id=?').run(row.id); + if (policy.category === 'operation') { + const payload = db().prepare(`SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(row.id) as { payload: string }; + db().prepare(`UPDATE ec_host_operations SET request_hash=?,payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`) + .run(createHash('sha256').update(payload.payload).digest('hex'),new Date().toISOString(),row.id); + } + db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(row.id); + } + if (policy.category === 'beta-file-content') { + db().prepare("UPDATE beta_revision_files SET content=X'' WHERE beta_id=?").run(row.id); + db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id=?').run(row.id); + db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString()); + } else if (policy.category === 'file-content') { + db().prepare('UPDATE deployment_files SET content_before=NULL,content_after=NULL WHERE deployment_id=?').run(row.id); + db().prepare("UPDATE staged_file_changes SET content=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')").run(row.id); + db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString()); + } else if (policy.category !== 'operation') { + db().prepare(`DELETE FROM ${policy.table} WHERE rowid=?`).run(row.rid); + db().prepare('DELETE FROM retention_controls WHERE category=? AND record_id=?').run(policy.category,row.id); + } + } + return rows.length; + })(); + } + return removed; +} + +function samplePolicies(s = fleetSettings()) { + const iso = (days: number) => new Date(Date.now() - days * 86400_000).toISOString(); + return [ + { category: 'host-resources', label: 'Host resource samples', table: 'ec_host_resource_samples', condition: 'sampled_at < ?', value: iso(s.hostResourceKeepDays), enabled: s.hostResourceKeepDays > 0 }, + { category: 'instance-resources', label: 'Instance resource samples', table: 'ec_instance_resource_samples', condition: 'bucket < ?', value: iso(Math.max(1,s.instanceResourceKeepDays)), enabled: s.instanceResourceKeepDays > 0 }, + { category: 'metrics', label: 'Fleet metrics samples', table: 'metric_samples', condition: 'at < ?', value: Date.now() - s.metricsKeepDays * 86400_000, enabled: s.metricsKeepDays > 0 }, + { category: 'telemetry', label: 'Telemetry diagnostics (age expiry)', table: 'telemetry_requests', condition: 'received_at < ?', value: iso(s.telemetryKeepDays), enabled: s.telemetryKeepDays > 0 } + ]; +} +export function sampleRetentionPreview(settings = fleetSettings()) { + initialize(); + const result = samplePolicies(settings).filter(p => p.category !== 'telemetry').map(p => ({ category: p.category,label: p.label,records: [],payloadBytes: 0, + eligible: p.enabled ? (db().prepare(`SELECT COUNT(*) AS n FROM ${p.table} WHERE ${p.condition}`).get(p.value) as { n: number }).n : 0 })); + const telemetry = db().prepare(`SELECT COUNT(*) AS eligible,COALESCE(SUM(bytes),0) AS payloadBytes FROM ( + SELECT received_at,length(payload) AS bytes,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS per_server, + ROW_NUMBER() OVER (ORDER BY id DESC) AS overall FROM telemetry_requests) + WHERE (? > 0 AND received_at < ?) OR (? > 0 AND per_server > ?) OR (? > 0 AND overall > ?)`) + .get(settings.telemetryKeepDays,new Date(Date.now()-settings.telemetryKeepDays*86400_000).toISOString(),settings.telemetryMaxPerServer, + settings.telemetryMaxPerServer,settings.telemetryMaxTotal,settings.telemetryMaxTotal) as {eligible:number;payloadBytes:number}; + result.push({category:'telemetry',label:'Telemetry diagnostics (age and count limits)',records:[],...telemetry}); + return result; +} +export function pruneSampleHistory() { + initialize(); + for (const p of samplePolicies().filter(p => p.enabled)) db().prepare(`DELETE FROM ${p.table} WHERE rowid IN (SELECT rowid FROM ${p.table} WHERE ${p.condition} LIMIT 5000)`).run(p.value); + const s = fleetSettings(); + if (s.telemetryMaxPerServer) db().prepare(`DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM + (SELECT id,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS rank FROM telemetry_requests) WHERE rank>? LIMIT 5000)`).run(s.telemetryMaxPerServer); + if (s.telemetryMaxTotal) db().prepare('DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 5000 OFFSET ?)').run(s.telemetryMaxTotal); +} + +function pruneRoutineObservations(): number { + const settings = fleetSettings(); + if (!settings.observationKeepDays) return 0; + const handle = db(); + return handle.transaction(() => { + const rows = handle.prepare(`SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed' + AND updated_atrow.id)); + handle.prepare('DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT value FROM json_each(?))').run(ids); + handle.prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id IN (SELECT value FROM json_each(?))').run(ids); + return handle.prepare('DELETE FROM ec_host_operations WHERE id IN (SELECT value FROM json_each(?))').run(ids).changes; + })(); +} diff --git a/src/lib/ui/components/RetentionTools.svelte b/src/lib/ui/components/RetentionTools.svelte new file mode 100644 index 0000000..98a56e8 --- /dev/null +++ b/src/lib/ui/components/RetentionTools.svelte @@ -0,0 +1,67 @@ + +
+ Cleanup preview and protected records +

Preview uses the values currently entered above, without saving them. Background cleanup runs in batches. Known payload sizes are estimates; database file size may not shrink immediately. Expired data cannot be restored by increasing retention.

+ + {#if error}{/if} + {#if notice}

{notice}

{/if} + {#if preview} +
+ {#each preview.policies as policy}{/each} +
CategoryEligible recordsKnown payload size
{policy.label}{policy.eligible}{policy.payloadBytes ? `${(policy.payloadBytes / 1024 / 1024).toFixed(2)} MiB` : '—'}
+ {#each preview.policies.filter(policy => policy.records.length) as policy} +
{policy.label}: eligible IDs (up to 20) + {#each policy.records as record}
{record.id}
{/each} +
+ {/each} +

Protected records and resolved failures

+ {#each preview.controls as record}
{record.category} · {record.id}{record.resolvedAt ? ' · Resolved' : ''} + {#if record.protected}{/if}
{:else}

No explicit record protections or resolutions.

{/each} + {/if} +

Protect a record or resolve a failure

+

Use the ID from its detail URL or the preview. Resolving a failed campaign stops automatic retry requests for that campaign; unfinished recovery still prevents cleanup.

+
+ + + + {#if ['campaign','provisioning','operation'].includes(category)}{/if} +
+
+ diff --git a/src/lib/ui/components/TelemetryDiagnostics.svelte b/src/lib/ui/components/TelemetryDiagnostics.svelte index 8f9e57d..dfee4d6 100644 --- a/src/lib/ui/components/TelemetryDiagnostics.svelte +++ b/src/lib/ui/components/TelemetryDiagnostics.svelte @@ -84,7 +84,7 @@ The scoreboard uses telemetry; the fleet player count comes from RCON separately.

- Newest first · up to {retention.maxPerServer} requests from the last {retention.maxAgeDays} days. + Newest first · {retention.maxPerServer ? `up to ${retention.maxPerServer} requests per server` : 'no per-server retention cap'} · {retention.maxAgeDays ? `last ${retention.maxAgeDays} days` : 'no age expiry'}. Fleet-wide limits also apply. Refreshes every 5 seconds while visible. {#if refreshedAt}Last refreshed .{/if}

diff --git a/src/routes/api/campaigns/[id]/+server.ts b/src/routes/api/campaigns/[id]/+server.ts index 4962bff..137be9f 100644 --- a/src/routes/api/campaigns/[id]/+server.ts +++ b/src/routes/api/campaigns/[id]/+server.ts @@ -41,5 +41,6 @@ export async function GET({ params }: { params: { id: string } }) { const events = db().prepare('SELECT id, created_at, message FROM campaign_events WHERE deployment_id = ? ORDER BY id').all(params.id); const watchdog = db().prepare('SELECT incident_id FROM watchdog_actions WHERE campaign_id = ? LIMIT 1').get(params.id) as { incident_id: string } | undefined; - return json({ deployment: dep, servers, files, scripts, events, watchdogIncidentId: watchdog?.incident_id ?? null }); + const fileContentsExpired = !!db().prepare("SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?").get(params.id); + return json({ fileContentsExpired, deployment: dep, servers, files, scripts, events, watchdogIncidentId: watchdog?.incident_id ?? null }); } diff --git a/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts b/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts index b2e86cf..4b6b2b5 100644 --- a/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts +++ b/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts @@ -5,6 +5,9 @@ import { db } from '$lib/server/db/index.js'; // so the DiffViewer can render a unified diff without a second query. export async function GET({ params }: { params: { id: string; fileId: string } }) { + if (db().prepare("SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?").get(params.id)) { + return json({ error: 'File contents expired under the retention policy.' }, { status: 410 }); + } const row = db() .prepare( `SELECT verb, rel_path, sha256_before, sha256_after, content_before, content_after diff --git a/src/routes/api/servers/[name]/telemetry-requests/+server.ts b/src/routes/api/servers/[name]/telemetry-requests/+server.ts index d50d708..ce277dd 100644 --- a/src/routes/api/servers/[name]/telemetry-requests/+server.ts +++ b/src/routes/api/servers/[name]/telemetry-requests/+server.ts @@ -1,11 +1,10 @@ import { json } from '@sveltejs/kit'; import { getServer } from '$lib/server/fleet/servers.js'; -import { listTelemetryRequests } from '$lib/server/fleet/telemetry-requests.js'; -import { TELEMETRY_REQUEST_RETENTION } from '$lib/telemetry-diagnostics.js'; +import { listTelemetryRequests, telemetryRetention } from '$lib/server/fleet/telemetry-requests.js'; export async function GET({ params }: { params: { name: string } }) { if (!getServer(params.name)) return json({ ok: false, error: 'no such server' }, { status: 404 }); - return json({ ok: true, requests: listTelemetryRequests(params.name), retention: TELEMETRY_REQUEST_RETENTION }, { + return json({ ok: true, requests: listTelemetryRequests(params.name), retention: telemetryRetention() }, { headers: { 'Cache-Control': 'no-store' } }); } diff --git a/src/routes/api/settings/retention/+server.ts b/src/routes/api/settings/retention/+server.ts new file mode 100644 index 0000000..6b7b4c6 --- /dev/null +++ b/src/routes/api/settings/retention/+server.ts @@ -0,0 +1,11 @@ +import { json } from '@sveltejs/kit'; +import { retentionPreview, retentionControls, sampleRetentionPreview, setRetentionControl } from '$lib/server/retention.js'; +import { gameFileRetentionPreview } from '$lib/server/game-files/index.js'; + +export function GET() { + return json({ policies: [...retentionPreview(),...sampleRetentionPreview(),...gameFileRetentionPreview()], controls: retentionControls() }, { headers: { 'Cache-Control': 'no-store' } }); +} +export async function POST({ request }: { request: Request }) { + try { setRetentionControl(await request.json()); return json({ ok: true }); } + catch (error) { return json({ error: (error as Error).message }, { status: 400 }); } +} diff --git a/src/routes/api/settings/retention/preview/+server.ts b/src/routes/api/settings/retention/preview/+server.ts new file mode 100644 index 0000000..ccc3f80 --- /dev/null +++ b/src/routes/api/settings/retention/preview/+server.ts @@ -0,0 +1,10 @@ +import { json } from '@sveltejs/kit'; +import { validateFleetSettings } from '$lib/fleet-settings.js'; +import { retentionPreview,retentionControls,sampleRetentionPreview } from '$lib/server/retention.js'; +import { gameFileRetentionPreview } from '$lib/server/game-files/index.js'; +export async function POST({ request }: { request: Request }) { + try { + const settings = validateFleetSettings(await request.json()); + return json({ policies:[...retentionPreview(settings),...sampleRetentionPreview(settings),...gameFileRetentionPreview(settings)],controls:retentionControls() },{ headers:{'Cache-Control':'no-store'} }); + } catch (error) { return json({error:(error as Error).message},{status:400}); } +} diff --git a/src/routes/campaigns/[id]/+page.svelte b/src/routes/campaigns/[id]/+page.svelte index 21b197c..da0ba30 100644 --- a/src/routes/campaigns/[id]/+page.svelte +++ b/src/routes/campaigns/[id]/+page.svelte @@ -31,6 +31,7 @@ attempts: number; error: string | null; } interface Detail { + fileContentsExpired?: boolean; deployment: { id: string; action: string; started_at: string; finished_at: string | null; triggered_by: string | null; beta_id?: number | null; @@ -96,7 +97,7 @@ diffOpen = true; diffLoading = true; diffError = ''; openFile = null; try { const response = await fetch('/api/campaigns/' + id + '/file/' + f.id); - if (!response.ok) throw new Error('Could not load this file comparison. Close and select it again to retry.'); + if (!response.ok) throw new Error(response.status === 410 ? 'File contents expired under the retention policy.' : 'Could not load this file comparison. Close and select it again to retry.'); const body = await response.json(); if (id === $page.params.id && current === diffRequest) openFile = {id:f.id, ...body}; } catch (error) { if (id === $page.params.id && current === diffRequest) diffError = (error as Error).message; } @@ -139,6 +140,7 @@ {/snippet} + {#if detail.fileContentsExpired}

File contents have expired under the retention policy. File comparisons and rollback are unavailable.

{/if}

{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{withdrawn ? ` · ${withdrawn} withdrawn` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}

@@ -209,7 +211,7 @@

server.server === srv)?.server_deleted_at} />

{#each files as f (f.id)} -