From 4fe8613678467b4272a041f7b307981574c99f4c Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sun, 27 Sep 2026 16:11:45 -0400 Subject: [PATCH] Move fleet retention and concurrency controls into Settings Persist settings in SQLite, import legacy environment values once, and apply changes to cleanup and deployment queues without restarting. Closes #51 --- .env.example | 11 +-- README.md | 30 ++++++++- docker-compose.yml | 3 - docs/deployment-blueprints.md | 2 +- komodo/stack.toml | 2 - src/lib/fleet-settings.ts | 34 ++++++++++ src/lib/server/config.ts | 9 --- src/lib/server/db/fleet-settings.ts | 20 ++++++ src/lib/server/db/index.ts | 2 + src/lib/server/fleet/collections.ts | 5 +- src/lib/server/pipeline/sweep.ts | 4 +- src/lib/server/provisioning/instances.ts | 7 +- src/lib/server/provisioning/restarts.ts | 4 +- src/lib/server/settings.ts | 14 ++++ src/routes/api/settings/+server.ts | 12 ++++ src/routes/settings/+page.svelte | 64 +++++++++++++++++- tests/unit/creation-queue.test.ts | 45 ++++++++++++- tests/unit/deployment-collections.test.ts | 10 +++ tests/unit/fleet-settings.test.ts | 81 +++++++++++++++++++++++ tests/unit/restart-rollout.test.ts | 29 ++++---- 20 files changed, 334 insertions(+), 54 deletions(-) create mode 100644 src/lib/fleet-settings.ts create mode 100644 src/lib/server/db/fleet-settings.ts create mode 100644 src/lib/server/settings.ts create mode 100644 src/routes/api/settings/+server.ts create mode 100644 tests/unit/fleet-settings.test.ts diff --git a/.env.example b/.env.example index f19f8da..b2829c3 100644 --- a/.env.example +++ b/.env.example @@ -9,16 +9,7 @@ BETA_ARCHIVE_DIR=/root/zombies/beta-archive # SQLite location DB_PATH=/data/cherry.db -# Retention (days) for deployment rows and blob diffs -DEPLOY_KEEP_DAYS=7 - -# Fleet-wide rolling restart limit, shared by deployments and betas. -# A slot stays occupied until Komodo and RCON confirm recovery. -DEPLOYMENT_RESTART_CONCURRENCY=1 - -# New-server startup limit, shared by all collections and single-server launches. -# A slot stays occupied through Komodo, RCON, and selected-map verification. -DEPLOYMENT_CREATE_CONCURRENCY=1 +# Configure campaign retention and restart/startup concurrency in Settings → Fleet settings. # Komodo KOMODO_ADDRESS=https://komodo.example.internal diff --git a/README.md b/README.md index 9590bef..59855dc 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,30 @@ submitted recovery continue. Each capability retains its own policy. Automatic updates remain disabled until explicitly enabled. See [Maintenance](docs/maintenance.md) for verification, retry, scheduling, and compatibility details. +## Fleet settings + +**Settings → Fleet settings** controls campaign history retention (default 7 days), +concurrent restarts (default 1), and concurrent server startups (default 1). +Values are positive whole numbers; retention supports up to 36500 days. Settings +are saved in SQLite for all users and apply at the next queue or cleanup check +without restarting the app. Lowering a limit leaves active operations running +and holds queued work until capacity is available. Collection previews and +progress pages display the current startup limit. + +Retention removes eligible old campaign records and file diffs. Pending, failed, +held, and health-recovery-linked campaigns remain protected; Provisioning history +is retained. Shortening retention can remove older history on the next sweep. + +On the first startup after upgrading, existing `DEPLOY_KEEP_DAYS`, +`DEPLOYMENT_RESTART_CONCURRENCY`, and `DEPLOYMENT_CREATE_CONCURRENCY` environment +values initialize the saved settings. Keep custom values for that first startup; +afterward, the database values take precedence and those variables can be removed. +The supplied Compose and Komodo examples use the built-in defaults. + +`GET /api/settings` reads the three values (`deployKeepDays`, `restartConcurrency`, +`createConcurrency`); `PUT /api/settings` saves all three together as JSON. +Invalid or incomplete requests return HTTP 400 without changing saved settings. + ## How restarts stay safe Betas support configurable gameplay, participation, stability and manual-review @@ -107,7 +131,7 @@ requirements, saved presets, coverage rules and staged waves. See File changes are staged sequentially. Their activation uses rolling restarts: by default only **one server across the fleet** can restart or recover at a time. Promotions, rollbacks, drift reconciliation, and beta restarts share this -limit. Set `DEPLOYMENT_RESTART_CONCURRENCY` to a positive integer to raise it. +limit. Adjust **Concurrent restarts** in **Settings → Fleet settings** to raise it. The Provisioning worker runs every 15s. Campaigns, betas, and maintenance submit durable restart requests and read their results. Provisioning checks recovering @@ -164,8 +188,8 @@ launches. A startup slot stays occupied until Komodo is healthy and RCON or fresh telemetry reports the selected map; an unready server holds the queue until it recovers or is removed. The queue survives application restarts. The collection preview shows the startup limit, and its progress page shows active startups and each -server's place in the shared queue. Set `DEPLOYMENT_CREATE_CONCURRENCY` to a -positive integer to increase the limit. Each worker pass starts at most the +server's place in the shared queue. Adjust **Concurrent server startups** in +**Settings → Fleet settings** to increase the limit. Each worker pass starts at most the available number of slots; immediate failures do not drain the remaining queue. Existing single-server collections can be expanded with **Edit**. Single-server launches and launching again from recorded recipes remain available. diff --git a/docker-compose.yml b/docker-compose.yml index b50ba84..13c33ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,9 +31,6 @@ services: REL_DEST: runtime/plutonium/storage/t5/raw/scripts/sp BETA_ARCHIVE_DIR: /root/zombies/beta-archive DB_PATH: /data/cherry.db - DEPLOY_KEEP_DAYS: "7" - DEPLOYMENT_RESTART_CONCURRENCY: ${DEPLOYMENT_RESTART_CONCURRENCY:-1} - DEPLOYMENT_CREATE_CONCURRENCY: ${DEPLOYMENT_CREATE_CONCURRENCY:-1} KOMODO_ADDRESS: ${KOMODO_ADDRESS} KOMODO_API_KEY: ${KOMODO_API_KEY} KOMODO_API_SECRET: ${KOMODO_API_SECRET} diff --git a/docs/deployment-blueprints.md b/docs/deployment-blueprints.md index c87a054..28d1ca6 100644 --- a/docs/deployment-blueprints.md +++ b/docs/deployment-blueprints.md @@ -95,7 +95,7 @@ the current shortfall too. All collections and individual launches share one startup queue, ordered by request time. By default, one server starts at a time. Its startup slot remains occupied until Komodo is healthy, RCON answers, and the reported map matches the selection. -Only then can the next queued server start. Set `DEPLOYMENT_CREATE_CONCURRENCY` to +Only then can the next queued server start. Set **Concurrent server startups** in **Settings → Fleet settings** to a positive integer to allow more simultaneous startups. Each worker pass admits at most the available slots, even if an attempt fails immediately. diff --git a/komodo/stack.toml b/komodo/stack.toml index b787461..4811b5c 100644 --- a/komodo/stack.toml +++ b/komodo/stack.toml @@ -27,8 +27,6 @@ PLUTAINER_T6_GAMEFILES_VOLUME = "t6_game_files" REL_DEST = "runtime/plutonium/storage/t5/raw/scripts/sp" BETA_ARCHIVE_DIR = "/root/zombies/beta-archive" DB_PATH = "/data/cherry.db" -DEPLOY_KEEP_DAYS = "7" -DEPLOYMENT_RESTART_CONCURRENCY = "1" # secrets: bind via Komodo's env-file or secret store KOMODO_ADDRESS = "https://komodo.example.internal" diff --git a/src/lib/fleet-settings.ts b/src/lib/fleet-settings.ts new file mode 100644 index 0000000..d5d3fd1 --- /dev/null +++ b/src/lib/fleet-settings.ts @@ -0,0 +1,34 @@ +export interface FleetSettings { + deployKeepDays: number; + restartConcurrency: number; + createConcurrency: number; +} + +export const defaultFleetSettings: FleetSettings = { + deployKeepDays: 7, + restartConcurrency: 1, + createConcurrency: 1 +}; + +export const fleetSettingLabels: Record = { + deployKeepDays: 'Campaign history retention', + restartConcurrency: 'Concurrent restarts', + createConcurrency: 'Concurrent server startups' +}; + +export function validateFleetSettings(input: unknown): FleetSettings { + if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide fleet settings.'); + const values = input as Record; + for (const key of Object.keys(values)) { + if (!Object.hasOwn(defaultFleetSettings, key)) throw new Error(`Unknown setting: ${key}`); + } + for (const key of Object.keys(defaultFleetSettings) as (keyof FleetSettings)[]) { + const value = values[key]; + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) { + throw new Error(`${fleetSettingLabels[key]} must be a positive whole number.`); + } + if (key === 'deployKeepDays' && value > 36500) throw new Error('Campaign history retention must be at most 36500 days.'); + } + return { deployKeepDays: values.deployKeepDays as number, restartConcurrency: values.restartConcurrency as number, + createConcurrency: values.createConcurrency as number }; +} diff --git a/src/lib/server/config.ts b/src/lib/server/config.ts index 2a81a72..346c8c3 100644 --- a/src/lib/server/config.ts +++ b/src/lib/server/config.ts @@ -15,12 +15,6 @@ function num(name: string, fallback: number): number { return n; } -function positiveInt(name: string, fallback: number): number { - const value = num(name, fallback); - if (!Number.isSafeInteger(value) || value < 1) throw new Error(`env ${name} must be a positive integer`); - return value; -} - export const config = { serversRoot: req('SERVERS_ROOT', '/root/zombies/t5'), t6ServersRoot: env.T6_SERVERS_ROOT ?? join(dirname(req('SERVERS_ROOT', '/root/zombies/t5')), 't6'), @@ -38,9 +32,6 @@ export const config = { // Cleared per-beta on promote/rollback. betaArchiveDir: req('BETA_ARCHIVE_DIR', '/root/zombies/beta-archive'), dbPath: req('DB_PATH', './data/cherry.db'), - deployKeepDays: num('DEPLOY_KEEP_DAYS', 7), - restartConcurrency: positiveInt('DEPLOYMENT_RESTART_CONCURRENCY', 1), - createConcurrency: positiveInt('DEPLOYMENT_CREATE_CONCURRENCY', 1), httpTimeoutMs: num('HTTP_TIMEOUT_MS', 10_000), komodo: { address: env.KOMODO_ADDRESS ?? '', diff --git a/src/lib/server/db/fleet-settings.ts b/src/lib/server/db/fleet-settings.ts new file mode 100644 index 0000000..3589b29 --- /dev/null +++ b/src/lib/server/db/fleet-settings.ts @@ -0,0 +1,20 @@ +import type Database from 'better-sqlite3'; +import { defaultFleetSettings, validateFleetSettings, type FleetSettings } from '../../fleet-settings.js'; + +export const FLEET_SETTINGS_KEY = 'fleet_settings'; + +/** Import legacy environment values once; persisted settings win on every later boot. */ +export function initializeFleetSettings(handle: Database.Database): void { + if (handle.prepare('SELECT 1 FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY)) return; + const values = { ...defaultFleetSettings }; + const legacy: Record = { + deployKeepDays: 'DEPLOY_KEEP_DAYS', restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY', + createConcurrency: 'DEPLOYMENT_CREATE_CONCURRENCY' + }; + for (const key of Object.keys(legacy) as (keyof FleetSettings)[]) { + const raw = process.env[legacy[key]]; + if (raw) values[key] = Number(raw); + } + handle.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)') + .run(FLEET_SETTINGS_KEY, JSON.stringify(validateFleetSettings(values))); +} diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index ac0a11b..7692bb7 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -6,6 +6,7 @@ import schemaSql from './schema.sql?raw'; import { requirement, type BetaPlan, type BetaRequirement } from '../../beta-plan.js'; import { emptyMetrics } from '../fleet/beta-evaluation.js'; import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js'; +import { initializeFleetSettings } from './fleet-settings.js'; let _db: Database.Database | null = null; @@ -345,6 +346,7 @@ export function db(): Database.Database { migrateGameTelemetry(handle); migrateTable(handle, 'leaderboard_matches', [{ name: 'start_players', ddl: 'ALTER TABLE leaderboard_matches ADD COLUMN start_players INTEGER' }]); backfillLeaderboard(handle); + initializeFleetSettings(handle); } catch (err) { handle.close(); throw err; diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index 9dad176..c7f65b1 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -1,3 +1,4 @@ +import { fleetSettings } from '../settings.js'; import { randomUUID } from 'node:crypto'; import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js'; import { db } from '../db/index.js'; @@ -106,7 +107,7 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill'): db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString()); db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)') .run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null); - return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: config.createConcurrency, servers, blockers, ready: blockers.length === 0, + return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ready: blockers.length === 0, entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) }; })(); } @@ -216,6 +217,6 @@ export function getCollection(id: string): CollectionDetail | null { const queue = creationQueue(); const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1])); const stored = db().prepare('SELECT retained_servers FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string }; - return { collection, retainedServers: JSON.parse(stored.retained_servers), queue: { concurrency: config.createConcurrency, active: queue.active, queued: queue.queued.length }, + return { collection, retainedServers: JSON.parse(stored.retained_servers), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length }, servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null })) }; } diff --git a/src/lib/server/pipeline/sweep.ts b/src/lib/server/pipeline/sweep.ts index f45705f..1c92ff6 100644 --- a/src/lib/server/pipeline/sweep.ts +++ b/src/lib/server/pipeline/sweep.ts @@ -1,4 +1,4 @@ -import { config } from '../config.js'; +import { fleetSettings } from '../settings.js'; import { db } from '../db/index.js'; import { emit } from './bus.js'; import { requestRestart } from '../provisioning/index.js'; @@ -44,7 +44,7 @@ async function runSweep(): Promise { db().prepare(`DELETE FROM deployments WHERE started_at < ? AND NOT EXISTS (SELECT 1 FROM watchdog_actions WHERE campaign_id = deployments.id) AND NOT EXISTS ( SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`) - .run(new Date(Date.now() - config.deployKeepDays * 86_400_000).toISOString()); + .run(new Date(Date.now() - fleetSettings().deployKeepDays * 86_400_000).toISOString()); emit({ type: 'sweep-tick', frozen: false }); return result; } diff --git a/src/lib/server/provisioning/instances.ts b/src/lib/server/provisioning/instances.ts index 48d33dd..2c6683a 100644 --- a/src/lib/server/provisioning/instances.ts +++ b/src/lib/server/provisioning/instances.ts @@ -1,3 +1,4 @@ +import { fleetSettings } from '../settings.js'; import { probePlayerCount, probeServerStatus } from '../fleet/status-evidence.js'; import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js'; import { randomUUID } from 'node:crypto'; @@ -258,7 +259,7 @@ export async function rebuildServer(id: string): Promise { const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined; if (!job) return; - if (creating || destroying.size || creationQueue().active.length >= config.createConcurrency) { + if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) { // The cleanup checkpoint must survive capacity waits and app restarts. if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding'); return; @@ -775,7 +776,7 @@ async function runCreations(): Promise { // Snapshot the budget once. Fast success/failure must not turn a single // worker pass into an unbounded stream of expensive container startups. const queue = creationQueue(); - const available = Math.max(0, config.createConcurrency - queue.active.length); + const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length); const eligible = queue.queued.filter(({ id }) => { const blocker = creationBlocker(id); if (blocker) deploymentStage(id, blocker); @@ -785,7 +786,7 @@ async function runCreations(): Promise { if (creating || destroying.size > 0) break; // API calls can cancel a selected job or resume an older startup while // a previous provision awaits Komodo. Recheck without expanding this pass. - if (creationQueue().active.length >= config.createConcurrency) break; + if (creationQueue().active.length >= fleetSettings().createConcurrency) break; if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue; const blocker = creationBlocker(id); if (blocker) { diff --git a/src/lib/server/provisioning/restarts.ts b/src/lib/server/provisioning/restarts.ts index da1bf5f..4610acd 100644 --- a/src/lib/server/provisioning/restarts.ts +++ b/src/lib/server/provisioning/restarts.ts @@ -1,4 +1,4 @@ -import { config } from '../config.js'; +import { fleetSettings } from '../settings.js'; import { db } from '../db/index.js'; import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js'; import { probeServerStatus } from '../fleet/status-evidence.js'; @@ -49,7 +49,7 @@ export function rolloutWaitReason(server: string, operationId = ''): string | nu return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`; } const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number }; - return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null; + return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null; } export type RolloutRestartResult = diff --git a/src/lib/server/settings.ts b/src/lib/server/settings.ts new file mode 100644 index 0000000..9b1dd24 --- /dev/null +++ b/src/lib/server/settings.ts @@ -0,0 +1,14 @@ +import { validateFleetSettings, type FleetSettings } from '../fleet-settings.js'; +import { db } from './db/index.js'; +import { FLEET_SETTINGS_KEY } from './db/fleet-settings.js'; + +export function fleetSettings(): FleetSettings { + const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string }; + return JSON.parse(row.value); +} + +export function saveFleetSettings(input: unknown): FleetSettings { + const settings = validateFleetSettings(input); + db().prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(settings), FLEET_SETTINGS_KEY); + return settings; +} diff --git a/src/routes/api/settings/+server.ts b/src/routes/api/settings/+server.ts new file mode 100644 index 0000000..de228b2 --- /dev/null +++ b/src/routes/api/settings/+server.ts @@ -0,0 +1,12 @@ +import { json } from '@sveltejs/kit'; +import { validateFleetSettings } from '$lib/fleet-settings.js'; +import { fleetSettings, saveFleetSettings } from '$lib/server/settings.js'; + +export function GET() { return json(fleetSettings()); } + +export async function PUT({ request }: { request: Request }) { + let settings; + try { settings = validateFleetSettings(await request.json()); } + catch (error) { return json({ error: (error as Error).message }, { status: 400 }); } + return json(saveFleetSettings(settings)); +} diff --git a/src/routes/settings/+page.svelte b/src/routes/settings/+page.svelte index 31196e9..52e5ed2 100644 --- a/src/routes/settings/+page.svelte +++ b/src/routes/settings/+page.svelte @@ -3,14 +3,71 @@ import { availableTimeZones, DEFAULT_TIME_ZONE } from '$lib/time-zone.js'; import { onMount } from 'svelte'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; + import Button from '$ui/components/primitives/Button.svelte'; + import type { FleetSettings } from '$lib/fleet-settings.js'; let exampleDate = $state(null); + let settings = $state(null); + let loading = $state(true); + let saving = $state(false); + let error = $state(''); + let notice = $state(''); const zones = $derived([...new Set([...availableTimeZones(), $timeZone])].sort()); - onMount(() => { exampleDate = new Date(); }); + onMount(() => { exampleDate = new Date(); void loadSettings(); }); + + async function loadSettings() { + loading = true; error = ''; + try { + const response = await fetch('/api/settings'); + if (!response.ok) throw new Error('Could not load fleet settings.'); + settings = await response.json(); + } catch (cause) { error = (cause as Error).message; } + finally { loading = false; } + } + + async function saveSettings(event: SubmitEvent) { + event.preventDefault(); + saving = true; error = ''; notice = ''; + try { + const response = await fetch('/api/settings', { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(settings) }); + const result = await response.json(); + if (!response.ok) throw new Error(result.error ?? 'Could not save fleet settings.'); + settings = result; + notice = 'Fleet settings saved. Changes apply to the next queue or cleanup check.'; + } catch (cause) { error = (cause as Error).message; } + finally { saving = false; } + } Settings · Electric Cherry
- + +
+

Fleet settings

+

Saved for everyone using this Electric Cherry instance. Changes apply without restarting the app.

+ {#if error}{/if} + {#if notice}

{notice}

{/if} + {#if loading} +

Loading fleet settings…

+ {:else if settings} +
+
+ + +

Default: 7 days. Removes eligible old campaign records and file diffs on the next cleanup check. Pending, failed, held, and health-recovery-linked campaigns are kept. Provisioning history is retained.

+ + +

Default: 1 across the fleet. Campaigns, betas, maintenance, and lifecycle actions share recovery slots until container and game health are verified.

+ + +

Default: 1 across collections and individual launches. Each startup holds its slot through container, game health, and selected-map verification.

+

Lowering a limit lets active operations finish; queued work waits for capacity. Player safety checks still apply.

+ +
+
+ {:else} + + {/if} +

Display preferences