diff --git a/docs/provisioning.md b/docs/provisioning.md index a2ffd19..fe302ea 100644 --- a/docs/provisioning.md +++ b/docs/provisioning.md @@ -196,3 +196,19 @@ detail endpoints. No history or IDs are renamed or discarded. Start and Stop use durable `instance.start` and `instance.stop` host-agent operations. Rebuild uses `instance.rebuild` and replaces the instance files. See [native host management](native-hosts.md) for command delivery and recovery. + +### Additional concurrency limits + +Administration → Settings separates Concurrency and Retention, with independent +save actions. Existing fleet restart, startup and destruction limits continue to +apply. Additional per-host restart, startup and destruction limits, a combined +lifecycle limit, a rebuild limit, and a limit on hosts preparing game files default +to 0 (no additional cap), preserving existing throughput on upgrade. + +Lifecycle work counts once toward the combined limit, including launch preparation +and unresolved delivered commands. Rebuilds include drain migrations and retain +capacity between source removal and destination creation. Per-host recovery slots +remain occupied until health verification. Lowering limits does not cancel work. +Game-file preparation is limited when commands are delivered; downloads and imports +share the cap, including manual requests. Replayed or interrupted commands retain +the same host slot. Runtime image pulls and per-host file transfer counts are unchanged. diff --git a/src/lib/fleet-settings.ts b/src/lib/fleet-settings.ts index f2de19b..8e64790 100644 --- a/src/lib/fleet-settings.ts +++ b/src/lib/fleet-settings.ts @@ -4,6 +4,13 @@ export interface FleetSettings { restartConcurrency: number; createConcurrency: number; destroyConcurrency: number; + restartConcurrencyPerHost: number; + createConcurrencyPerHost: number; + destroyConcurrencyPerHost: number; + lifecycleConcurrency: number; + rebuildConcurrency: number; + preparationConcurrency: number; + } export const defaultFleetSettings: FleetSettings = { @@ -11,7 +18,13 @@ export const defaultFleetSettings: FleetSettings = { observationKeepDays: 1, restartConcurrency: 1, createConcurrency: 1, - destroyConcurrency: 1 + destroyConcurrency: 1, + restartConcurrencyPerHost: 0, + createConcurrencyPerHost: 0, + destroyConcurrencyPerHost: 0, + lifecycleConcurrency: 0, + rebuildConcurrency: 0, + preparationConcurrency: 0, }; export const fleetSettingLabels: Record = { @@ -19,7 +32,14 @@ export const fleetSettingLabels: Record = { observationKeepDays: 'Routine operation history retention', restartConcurrency: 'Concurrent restarts', createConcurrency: 'Concurrent server startups', - destroyConcurrency: 'Concurrent server destructions' + destroyConcurrency: 'Concurrent server destructions', + restartConcurrencyPerHost: 'Concurrent restarts per host', + createConcurrencyPerHost: 'Concurrent startups per host', + destroyConcurrencyPerHost: 'Concurrent destructions per host', + lifecycleConcurrency: 'Combined lifecycle limit', + rebuildConcurrency: 'Concurrent rebuilds', + preparationConcurrency: 'Hosts preparing game files simultaneously', + }; export function validateFleetSettings(input: unknown): FleetSettings { @@ -30,7 +50,7 @@ export function validateFleetSettings(input: unknown): FleetSettings { } for (const key of Object.keys(defaultFleetSettings) as (keyof FleetSettings)[]) { const value = values[key]; - const minimum = key === 'observationKeepDays' ? 0 : 1; + const minimum = key === 'observationKeepDays' || additionalConcurrencyKeys.includes(key) ? 0 : 1; if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) { throw new Error(`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`); } @@ -38,6 +58,22 @@ export function validateFleetSettings(input: unknown): FleetSettings { throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`); } } - return { deployKeepDays: values.deployKeepDays as number, observationKeepDays: values.observationKeepDays as number, restartConcurrency: values.restartConcurrency as number, - createConcurrency: values.createConcurrency as number, destroyConcurrency: values.destroyConcurrency as number }; + return Object.fromEntries(Object.keys(defaultFleetSettings).map(key => [key, values[key]])) as unknown as FleetSettings; } + +export const additionalConcurrencyKeys: (keyof FleetSettings)[] = ["restartConcurrencyPerHost", "createConcurrencyPerHost", "destroyConcurrencyPerHost", "lifecycleConcurrency", "rebuildConcurrency", "preparationConcurrency"]; +export const concurrencyKeys: (keyof FleetSettings)[] = ['restartConcurrency', 'createConcurrency', 'destroyConcurrency', ...additionalConcurrencyKeys]; +export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays']; +export const settingHelp: Partial> = { + restartConcurrency: 'Fleet recovery slots stay occupied until container and game health are verified.', + createConcurrency: 'Fleet startup slots cover creation and rebuilds through health and selected-map verification.', + destroyConcurrency: 'Fleet removals hold capacity until teardown and cleanup finish.', + restartConcurrencyPerHost: 'Additional recovery limit on each host, including starts and rebuilds.', + createConcurrencyPerHost: 'Additional startup limit on each host, including rebuilds.', + destroyConcurrencyPerHost: 'Additional removal limit on each host, including migration source removal.', + lifecycleConcurrency: 'Combined fleet limit for creates, starts, stops, restarts, rebuilds and destructions. Each workflow counts once.', + rebuildConcurrency: 'Additional fleet limit for rebuilds, including host drain migrations.', + preparationConcurrency: 'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.', + deployKeepDays: 'Eligible campaign records and file diffs. Pending, failed, held and recovery-linked campaigns are protected.', + observationKeepDays: 'Successful monitoring and file-read records. Failures and actions are retained. 0 keeps records indefinitely.' +}; diff --git a/src/lib/server/db/fleet-settings.ts b/src/lib/server/db/fleet-settings.ts index b9c71f6..291c0a6 100644 --- a/src/lib/server/db/fleet-settings.ts +++ b/src/lib/server/db/fleet-settings.ts @@ -8,11 +8,8 @@ export function initializeFleetSettings(handle: Database.Database): void { const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined; if (saved) { const settings = JSON.parse(saved.value); - if (!Object.hasOwn(settings, 'destroyConcurrency') || !Object.hasOwn(settings, 'observationKeepDays')) { - const upgraded = validateFleetSettings({ destroyConcurrency: defaultFleetSettings.destroyConcurrency, - observationKeepDays: defaultFleetSettings.observationKeepDays, ...settings }); - handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY); - } + const upgraded = validateFleetSettings({ ...defaultFleetSettings, ...settings }); + handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY); return; } const values = { ...defaultFleetSettings }; diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index 74d64ae..50a75db 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -533,9 +533,18 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = ' // sibling's work, but still recovers a lost receipt or agent restart. .all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false), ...(lane === 'health' ? [new Date(Date.now() - 30_000).toISOString()] : [])) as OperationDeliveryRow[]; + const preparationLimit = fleetSettings().preparationConcurrency; + const preparing = new Set((database().prepare(`SELECT DISTINCT host_id FROM ec_host_operations + WHERE kind IN ('library.download','library.import') AND last_delivered_at IS NOT NULL + AND state IN ('pending','accepted','running','unknown')`).all() as { host_id: string }[]).map(row => row.host_id)); + const admitted = rows.filter(operation => { + if (!['library.download','library.import'].includes(operation.kind) || preparing.has(hostId) || !preparationLimit) return true; + if (preparing.size >= preparationLimit) return false; + preparing.add(hostId); return true; + }); const deliveredAt = now(); - for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id); - return rows; + for (const operation of admitted) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id); + return admitted; })(); } /** Payloads never change after enqueueing; fetch one delivery's UTF-8 bytes without decoding. */ diff --git a/src/lib/server/hosts/preparation.ts b/src/lib/server/hosts/preparation.ts index e96c8bb..a501222 100644 --- a/src/lib/server/hosts/preparation.ts +++ b/src/lib/server/hosts/preparation.ts @@ -8,7 +8,7 @@ import { scheduleAvailabilityRefresh } from './availability.js'; const activeStates = ['pending', 'accepted', 'running', 'unknown']; const supersededDownload = 'Superseded by newer game files before delivery to the host.'; -type PreparationOperation = { id: string; sequence: number; state: HostOperationState; updated_at: string; error: string | null }; +type PreparationOperation = { id: string; sequence: number; last_delivered_at: string | null; state: HostOperationState; updated_at: string; error: string | null }; type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation }; type QueuedSelection = { libraryId: string; image: string | null }; type LibraryTarget = Pick[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean }; @@ -50,7 +50,7 @@ function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] { } function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined { - return db().prepare(`SELECT id,rowid AS sequence,state,updated_at,error FROM ec_host_operations + return db().prepare(`SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`) .get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined; } @@ -117,7 +117,9 @@ function targets(host: Host): Target[] { if (blocker) { Object.assign(target.item, blocker); continue; } if (operation && activeStates.includes(operation.state)) { target.item.state = 'preparing'; - target.item.detail = operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.' + target.item.detail = !target.image && operation.state === 'pending' && !operation.last_delivered_at + ? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.' + : operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.' : target.image ? 'Downloading the runtime automatically.' : target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.'; } else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) { diff --git a/src/lib/server/provisioning/concurrency.ts b/src/lib/server/provisioning/concurrency.ts new file mode 100644 index 0000000..32838e9 --- /dev/null +++ b/src/lib/server/provisioning/concurrency.ts @@ -0,0 +1,47 @@ +import { ensureHostSchema } from '../hosts/schema.js'; +import { db } from '../db/index.js'; +import { fleetSettings } from '../settings.js'; + +// A preparation owns capacity across asynchronous launch-file generation. +export const preparingJobs = new Set(); +interface Holder { id: string; action: string; host_id: string | null; target_host_id: string | null; removing: number } + +export function lifecycleHolders(): Holder[] { + ensureHostSchema(db()); + return db().prepare(`SELECT d.id,d.action,i.host_id,m.target_host_id, + CASE WHEN m.remove_operation_id IS NOT NULL AND m.removed_at IS NULL THEN 1 ELSE 0 END AS removing + FROM server_deployments d + LEFT JOIN native_provisioning n ON n.deployment_id=d.id + LEFT JOIN native_instances i ON i.id=n.instance_id + LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id + WHERE d.state IN ('running','awaiting-health') + OR d.id IN (SELECT value FROM json_each(?)) + OR EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=d.id AND r.healthy_at IS NULL) + OR (m.remove_operation_id IS NOT NULL AND d.state NOT IN ('done','warning','cancelled')) + OR EXISTS (SELECT 1 FROM ec_host_operations o WHERE o.id IN (n.operation_id,m.remove_operation_id) + AND o.state IN ('pending','accepted','running','unknown'))`) + .all(JSON.stringify([...preparingJobs])) as Holder[]; +} + +/** Recheck inside the command transaction after every asynchronous preflight. */ +export function concurrencyBlocker(id: string, action: string, hostId?: string, removing = false): string | null { + const settings = fleetSettings(); + const holders = lifecycleHolders(); + const others = holders.filter(row => row.id !== id); + const own = holders.find(row => row.id === id); + const continuing = own && !preparingJobs.has(id); + const full = (limit: number, used: number) => limit > 0 && used >= limit; + if (!continuing && full(settings.lifecycleConcurrency, others.length)) return 'Waiting for combined lifecycle capacity.'; + if (!continuing && action === 'rebuild' && full(settings.rebuildConcurrency, others.filter(row => row.action === 'rebuild').length)) return 'Waiting for rebuild capacity.'; + if (!hostId || continuing) return null; + if (['create','rebuild'].includes(action) && full(settings.createConcurrencyPerHost, + others.filter(row => ['create','rebuild'].includes(row.action) && (row.host_id === hostId || row.target_host_id === hostId)).length)) return 'Waiting for startup capacity on this host.'; + if ((action === 'destroy' || removing) && full(settings.destroyConcurrencyPerHost, + others.filter(row => (row.action === 'destroy' || row.removing) && row.host_id === hostId).length)) return 'Waiting for destruction capacity on this host.'; + if (['start','restart','rebuild'].includes(action)) { + const slots = db().prepare(`SELECT r.operation_id FROM restart_rollout r JOIN native_instances i + ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND i.host_id=? AND (r.operation_id IS NULL OR r.operation_id!=?)`).all(hostId, id); + if (full(settings.restartConcurrencyPerHost, slots.length)) return 'Waiting for restart recovery capacity on this host.'; + } + return null; +} diff --git a/src/lib/server/provisioning/host-drain.ts b/src/lib/server/provisioning/host-drain.ts index 5c1ae79..cae16ec 100644 --- a/src/lib/server/provisioning/host-drain.ts +++ b/src/lib/server/provisioning/host-drain.ts @@ -1,3 +1,4 @@ +import { concurrencyBlocker } from './concurrency.js'; import { db } from '../db/index.js'; import { provisioningCancelled } from './cancellation.js'; import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js'; @@ -143,6 +144,8 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number }; if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; } const { host, port } = choosePlacement(options); + const capacityBlocker = concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? concurrencyBlocker(job.id, 'rebuild', host.id); + if (capacityBlocker) { hold(capacityBlocker); return; } reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' }); reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' }); const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 2d3e269..1c84319 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -1,3 +1,4 @@ +import { concurrencyBlocker, preparingJobs } from './concurrency.js'; import { isIP } from 'node:net'; import { setImmediate as yieldToController } from 'node:timers/promises'; import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js'; @@ -356,7 +357,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s const activeJobs = new Map>(); // Preparation owns startup capacity until its command is committed or discarded. -const preparingJobs = new Set(); + function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); } let launchCommit: Promise = Promise.resolve(); @@ -391,6 +392,8 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null): P const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }).n; if (active + otherPreparations(job.id) >= fleetSettings().createConcurrency) { hold(job, 'Waiting for startup capacity.'); return null; } + const capacityBlocker = concurrencyBlocker(job.id, job.action, decision.hostId); + if (capacityBlocker) { hold(job, capacityBlocker); return null; } if (getHost(decision.hostId)?.lastHeartbeat !== decision.lastHeartbeat) { hold(job, 'Host measurements changed; waiting for another placement check.'); return null; } // A short current-state check protects against reservations made while the // reader ranked hosts. It never scans resource history or contacts agents. @@ -435,6 +438,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }; if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; } + const capacityBlocker = concurrencyBlocker(job.id, job.action); + if (capacityBlocker) { hold(job, capacityBlocker); return; } if (eligibility(job)) return; instance = await placeQueuedCreation(job, instance); if (!instance) return; @@ -582,6 +587,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return; } } + const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id); + if (capacityBlocker) { hold(job, capacityBlocker); return; } const settings = fleetSettings(); const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n; @@ -623,6 +630,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; } + const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id); + if (capacityBlocker) { hold(job, capacityBlocker); return; } preparingJobs.add(job.id); preparedKey = getServerKey(job.server) ?? null; if (!preparedKey) throw new Error('The saved server key was removed.'); @@ -653,6 +662,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) { finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return; } + const capacityBlocker = concurrencyBlocker(job.id, job.action, instance!.host_id); + if (capacityBlocker) { hold(job, capacityBlocker); return; } // Async occupancy checks can overlap another job claiming the final slot. if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) { deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return; diff --git a/src/routes/api/settings/+server.ts b/src/routes/api/settings/+server.ts index de228b2..c982dfd 100644 --- a/src/routes/api/settings/+server.ts +++ b/src/routes/api/settings/+server.ts @@ -1,5 +1,5 @@ import { json } from '@sveltejs/kit'; -import { validateFleetSettings } from '$lib/fleet-settings.js'; +import { validateFleetSettings, concurrencyKeys, retentionKeys } from '$lib/fleet-settings.js'; import { fleetSettings, saveFleetSettings } from '$lib/server/settings.js'; export function GET() { return json(fleetSettings()); } @@ -10,3 +10,14 @@ export async function PUT({ request }: { request: Request }) { catch (error) { return json({ error: (error as Error).message }, { status: 400 }); } return json(saveFleetSettings(settings)); } + +export async function PATCH({ request }: { request: Request }) { + try { + const { section, values } = await request.json(); + const keys = section === 'Concurrency' ? concurrencyKeys : section === 'Retention' ? retentionKeys : null; + if (!keys || !values || typeof values !== 'object' || Array.isArray(values) + || Object.keys(values).length !== keys.length || keys.some(key => !Object.hasOwn(values, key)) + || Object.keys(values).some(key => !keys.includes(key as typeof keys[number]))) throw new Error('Provide all settings for one section.'); + return json(saveFleetSettings({ ...fleetSettings(), ...values })); + } catch (error) { return json({ error: (error as Error).message }, { status: 400 }); } +} diff --git a/src/routes/settings/+page.svelte b/src/routes/settings/+page.svelte index fb6ebf2..fdd6c99 100644 --- a/src/routes/settings/+page.svelte +++ b/src/routes/settings/+page.svelte @@ -2,7 +2,7 @@ import { onMount } from 'svelte'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Button from '$ui/components/primitives/Button.svelte'; - import type { FleetSettings } from '$lib/fleet-settings.js'; + import { defaultFleetSettings, fleetSettingLabels, additionalConcurrencyKeys, concurrencyKeys, retentionKeys, settingHelp, type FleetSettings } from '$lib/fleet-settings.js'; let settings = $state(null); let loading = $state(true); let saving = $state(false); @@ -20,14 +20,14 @@ finally { loading = false; } } - async function saveSettings(event: SubmitEvent) { + async function saveSettings(event: SubmitEvent, section: string, keys: (keyof FleetSettings)[]) { event.preventDefault(); saving = true; error = ''; notice = ''; try { - const response = await fetch('/api/settings', { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(settings) }); + const response = await fetch('/api/settings', { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ section, values: Object.fromEntries(keys.map(key => [key, settings![key]])) }) }); const result = await response.json(); if (!response.ok) throw new Error(result.error ?? 'Could not save fleet settings.'); - settings = result; + for (const key of keys) settings![key] = result[key]; notice = 'Fleet settings saved. Changes apply to the next queue or cleanup check.'; } catch (cause) { error = (cause as Error).message; } finally { saving = false; } @@ -45,27 +45,23 @@ {#if loading}

Loading fleet settings…

{:else if settings} -
-
- - -

Default: 7 days. Removes eligible old campaign records and file diffs on the next cleanup check. Pending, failed, held, and health-recovery-linked campaigns are kept. Provisioning history is retained.

- - -

Default: 1 day. Removes successful monitoring and file-read operation records on the next cleanup check. Failures and actions are retained. Set to 0 to keep these records indefinitely.

- - -

Default: 1 across the fleet. Campaigns, betas, maintenance, and lifecycle actions share recovery slots until container and game health are verified.

- - -

Default: 1 across collections and individual launches. Each startup holds its slot through container, game health, and selected-map verification.

- - -

Default: 1 across individual and fleet removals. Each destruction holds its slot until teardown and local cleanup finish.

-

Lowering a limit lets active operations finish; queued work waits for capacity. Player safety checks still apply.

- -
-
+ {#each [{ title: 'Concurrency', keys: concurrencyKeys }, { title: 'Retention', keys: retentionKeys }] as section} +
+

{section.title}

+
saveSettings(event, section.title, section.keys)}> +
+ {#each section.keys as key} + + +

Default: {defaultFleetSettings[key]}. {settingHelp[key] ?? ''} {additionalConcurrencyKeys.includes(key) ? '0 adds no extra limit.' : ''}

+ {/each} + {#if section.title === 'Concurrency'}

Lowering a limit lets active operations finish; queued work waits for capacity. Player safety checks still apply.

{/if} + +
+
+
+ {/each} {:else} {/if} @@ -76,6 +72,7 @@