Add fleet and per-host concurrency controls

docs/provisioning.md

@@ -196,3 +196,19 @@ detail endpoints. No history or IDs are renamed or discarded.
 Start and Stop use durable `instance.start` and `instance.stop` host-agent
 operations. Rebuild uses `instance.rebuild` and replaces the instance files.
 See [native host management](native-hosts.md) for command delivery and recovery.
+
+### Additional concurrency limits
+
+Administration → Settings separates Concurrency and Retention, with independent
+save actions. Existing fleet restart, startup and destruction limits continue to
+apply. Additional per-host restart, startup and destruction limits, a combined
+lifecycle limit, a rebuild limit, and a limit on hosts preparing game files default
+to 0 (no additional cap), preserving existing throughput on upgrade.
+
+Lifecycle work counts once toward the combined limit, including launch preparation
+and unresolved delivered commands. Rebuilds include drain migrations and retain
+capacity between source removal and destination creation. Per-host recovery slots
+remain occupied until health verification. Lowering limits does not cancel work.
+Game-file preparation is limited when commands are delivered; downloads and imports
+share the cap, including manual requests. Replayed or interrupted commands retain
+the same host slot. Runtime image pulls and per-host file transfer counts are unchanged.

src/lib/fleet-settings.ts

@@ -4,6 +4,13 @@ export interface FleetSettings {
     restartConcurrency: number;
     createConcurrency: number;
     destroyConcurrency: number;
+    restartConcurrencyPerHost: number;
+    createConcurrencyPerHost: number;
+    destroyConcurrencyPerHost: number;
+    lifecycleConcurrency: number;
+    rebuildConcurrency: number;
+    preparationConcurrency: number;
+
 }
 
 export const defaultFleetSettings: FleetSettings = {
@@ -11,7 +18,13 @@ export const defaultFleetSettings: FleetSettings = {
     observationKeepDays: 1,
     restartConcurrency: 1,
     createConcurrency: 1,
-    destroyConcurrency: 1
+    destroyConcurrency: 1,
+    restartConcurrencyPerHost: 0,
+    createConcurrencyPerHost: 0,
+    destroyConcurrencyPerHost: 0,
+    lifecycleConcurrency: 0,
+    rebuildConcurrency: 0,
+    preparationConcurrency: 0,
 };
 
 export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
@@ -19,7 +32,14 @@ export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
     observationKeepDays: 'Routine operation history retention',
     restartConcurrency: 'Concurrent restarts',
     createConcurrency: 'Concurrent server startups',
-    destroyConcurrency: 'Concurrent server destructions'
+    destroyConcurrency: 'Concurrent server destructions',
+    restartConcurrencyPerHost: 'Concurrent restarts per host',
+    createConcurrencyPerHost: 'Concurrent startups per host',
+    destroyConcurrencyPerHost: 'Concurrent destructions per host',
+    lifecycleConcurrency: 'Combined lifecycle limit',
+    rebuildConcurrency: 'Concurrent rebuilds',
+    preparationConcurrency: 'Hosts preparing game files simultaneously',
+
 };
 
 export function validateFleetSettings(input: unknown): FleetSettings {
@@ -30,7 +50,7 @@ export function validateFleetSettings(input: unknown): FleetSettings {
     }
     for (const key of Object.keys(defaultFleetSettings) as (keyof FleetSettings)[]) {
         const value = values[key];
-        const minimum = key === 'observationKeepDays' ? 0 : 1;
+        const minimum = key === 'observationKeepDays' || additionalConcurrencyKeys.includes(key) ? 0 : 1;
         if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) {
             throw new Error(`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`);
         }
@@ -38,6 +58,22 @@ export function validateFleetSettings(input: unknown): FleetSettings {
             throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`);
         }
     }
-    return { deployKeepDays: values.deployKeepDays as number, observationKeepDays: values.observationKeepDays as number, restartConcurrency: values.restartConcurrency as number,
-        createConcurrency: values.createConcurrency as number, destroyConcurrency: values.destroyConcurrency as number };
+    return Object.fromEntries(Object.keys(defaultFleetSettings).map(key => [key, values[key]])) as unknown as FleetSettings;
 }
+
+export const additionalConcurrencyKeys: (keyof FleetSettings)[] = ["restartConcurrencyPerHost", "createConcurrencyPerHost", "destroyConcurrencyPerHost", "lifecycleConcurrency", "rebuildConcurrency", "preparationConcurrency"];
+export const concurrencyKeys: (keyof FleetSettings)[] = ['restartConcurrency', 'createConcurrency', 'destroyConcurrency', ...additionalConcurrencyKeys];
+export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays'];
+export const settingHelp: Partial<Record<keyof FleetSettings, string>> = {
+    restartConcurrency: 'Fleet recovery slots stay occupied until container and game health are verified.',
+    createConcurrency: 'Fleet startup slots cover creation and rebuilds through health and selected-map verification.',
+    destroyConcurrency: 'Fleet removals hold capacity until teardown and cleanup finish.',
+    restartConcurrencyPerHost: 'Additional recovery limit on each host, including starts and rebuilds.',
+    createConcurrencyPerHost: 'Additional startup limit on each host, including rebuilds.',
+    destroyConcurrencyPerHost: 'Additional removal limit on each host, including migration source removal.',
+    lifecycleConcurrency: 'Combined fleet limit for creates, starts, stops, restarts, rebuilds and destructions. Each workflow counts once.',
+    rebuildConcurrency: 'Additional fleet limit for rebuilds, including host drain migrations.',
+    preparationConcurrency: 'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.',
+    deployKeepDays: 'Eligible campaign records and file diffs. Pending, failed, held and recovery-linked campaigns are protected.',
+    observationKeepDays: 'Successful monitoring and file-read records. Failures and actions are retained. 0 keeps records indefinitely.'
+};

src/lib/server/db/fleet-settings.ts

@@ -8,11 +8,8 @@ export function initializeFleetSettings(handle: Database.Database): void {
     const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
     if (saved) {
         const settings = JSON.parse(saved.value);
-        if (!Object.hasOwn(settings, 'destroyConcurrency') || !Object.hasOwn(settings, 'observationKeepDays')) {
-            const upgraded = validateFleetSettings({ destroyConcurrency: defaultFleetSettings.destroyConcurrency,
-                observationKeepDays: defaultFleetSettings.observationKeepDays, ...settings });
-            handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
-        }
+        const upgraded = validateFleetSettings({ ...defaultFleetSettings, ...settings });
+        handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
         return;
     }
     const values = { ...defaultFleetSettings };

src/lib/server/hosts/index.ts

@@ -533,9 +533,18 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
             // sibling's work, but still recovers a lost receipt or agent restart.
             .all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false),
                 ...(lane === 'health' ? [new Date(Date.now() - 30_000).toISOString()] : [])) as OperationDeliveryRow[];
+        const preparationLimit = fleetSettings().preparationConcurrency;
+        const preparing = new Set((database().prepare(`SELECT DISTINCT host_id FROM ec_host_operations
+            WHERE kind IN ('library.download','library.import') AND last_delivered_at IS NOT NULL
+            AND state IN ('pending','accepted','running','unknown')`).all() as { host_id: string }[]).map(row => row.host_id));
+        const admitted = rows.filter(operation => {
+            if (!['library.download','library.import'].includes(operation.kind) || preparing.has(hostId) || !preparationLimit) return true;
+            if (preparing.size >= preparationLimit) return false;
+            preparing.add(hostId); return true;
+        });
         const deliveredAt = now();
-        for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
-        return rows;
+        for (const operation of admitted) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
+        return admitted;
     })();
 }
 /** Payloads never change after enqueueing; fetch one delivery's UTF-8 bytes without decoding. */

src/lib/server/hosts/preparation.ts

@@ -8,7 +8,7 @@ import { scheduleAvailabilityRefresh } from './availability.js';
 
 const activeStates = ['pending', 'accepted', 'running', 'unknown'];
 const supersededDownload = 'Superseded by newer game files before delivery to the host.';
-type PreparationOperation = { id: string; sequence: number; state: HostOperationState; updated_at: string; error: string | null };
+type PreparationOperation = { id: string; sequence: number; last_delivered_at: string | null; state: HostOperationState; updated_at: string; error: string | null };
 type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
 type QueuedSelection = { libraryId: string; image: string | null };
 type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
@@ -50,7 +50,7 @@ function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
 }
 
 function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
-    return db().prepare(`SELECT id,rowid AS sequence,state,updated_at,error FROM ec_host_operations
+    return db().prepare(`SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations
         WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
         .get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
 }
@@ -117,7 +117,9 @@ function targets(host: Host): Target[] {
         if (blocker) { Object.assign(target.item, blocker); continue; }
         if (operation && activeStates.includes(operation.state)) {
             target.item.state = 'preparing';
-            target.item.detail = operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
+            target.item.detail = !target.image && operation.state === 'pending' && !operation.last_delivered_at
+                ? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.'
+                : operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
                 : target.image ? 'Downloading the runtime automatically.'
                 : target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
         } else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {

src/lib/server/provisioning/concurrency.ts

@@ -0,0 +1,47 @@
+import { ensureHostSchema } from '../hosts/schema.js';
+import { db } from '../db/index.js';
+import { fleetSettings } from '../settings.js';
+
+// A preparation owns capacity across asynchronous launch-file generation.
+export const preparingJobs = new Set<string>();
+interface Holder { id: string; action: string; host_id: string | null; target_host_id: string | null; removing: number }
+
+export function lifecycleHolders(): Holder[] {
+    ensureHostSchema(db());
+    return db().prepare(`SELECT d.id,d.action,i.host_id,m.target_host_id,
+        CASE WHEN m.remove_operation_id IS NOT NULL AND m.removed_at IS NULL THEN 1 ELSE 0 END AS removing
+        FROM server_deployments d
+        LEFT JOIN native_provisioning n ON n.deployment_id=d.id
+        LEFT JOIN native_instances i ON i.id=n.instance_id
+        LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
+        WHERE d.state IN ('running','awaiting-health')
+        OR d.id IN (SELECT value FROM json_each(?))
+        OR EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=d.id AND r.healthy_at IS NULL)
+        OR (m.remove_operation_id IS NOT NULL AND d.state NOT IN ('done','warning','cancelled'))
+        OR EXISTS (SELECT 1 FROM ec_host_operations o WHERE o.id IN (n.operation_id,m.remove_operation_id)
+            AND o.state IN ('pending','accepted','running','unknown'))`)
+        .all(JSON.stringify([...preparingJobs])) as Holder[];
+}
+
+/** Recheck inside the command transaction after every asynchronous preflight. */
+export function concurrencyBlocker(id: string, action: string, hostId?: string, removing = false): string | null {
+    const settings = fleetSettings();
+    const holders = lifecycleHolders();
+    const others = holders.filter(row => row.id !== id);
+    const own = holders.find(row => row.id === id);
+    const continuing = own && !preparingJobs.has(id);
+    const full = (limit: number, used: number) => limit > 0 && used >= limit;
+    if (!continuing && full(settings.lifecycleConcurrency, others.length)) return 'Waiting for combined lifecycle capacity.';
+    if (!continuing && action === 'rebuild' && full(settings.rebuildConcurrency, others.filter(row => row.action === 'rebuild').length)) return 'Waiting for rebuild capacity.';
+    if (!hostId || continuing) return null;
+    if (['create','rebuild'].includes(action) && full(settings.createConcurrencyPerHost,
+        others.filter(row => ['create','rebuild'].includes(row.action) && (row.host_id === hostId || row.target_host_id === hostId)).length)) return 'Waiting for startup capacity on this host.';
+    if ((action === 'destroy' || removing) && full(settings.destroyConcurrencyPerHost,
+        others.filter(row => (row.action === 'destroy' || row.removing) && row.host_id === hostId).length)) return 'Waiting for destruction capacity on this host.';
+    if (['start','restart','rebuild'].includes(action)) {
+        const slots = db().prepare(`SELECT r.operation_id FROM restart_rollout r JOIN native_instances i
+            ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND i.host_id=? AND (r.operation_id IS NULL OR r.operation_id!=?)`).all(hostId, id);
+        if (full(settings.restartConcurrencyPerHost, slots.length)) return 'Waiting for restart recovery capacity on this host.';
+    }
+    return null;
+}

src/lib/server/provisioning/host-drain.ts

@@ -1,3 +1,4 @@
+import { concurrencyBlocker } from './concurrency.js';
 import { db } from '../db/index.js';
 import { provisioningCancelled } from './cancellation.js';
 import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
@@ -143,6 +144,8 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance
         const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
         if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
         const { host, port } = choosePlacement(options);
+        const capacityBlocker = concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? concurrencyBlocker(job.id, 'rebuild', host.id);
+        if (capacityBlocker) { hold(capacityBlocker); return; }
         reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
         reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
         const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations

src/lib/server/provisioning/native.ts

@@ -1,3 +1,4 @@
+import { concurrencyBlocker, preparingJobs } from './concurrency.js';
 import { isIP } from 'node:net';
 import { setImmediate as yieldToController } from 'node:timers/promises';
 import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
@@ -356,7 +357,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
 
 const activeJobs = new Map<string, Promise<void>>();
 // Preparation owns startup capacity until its command is committed or discarded.
-const preparingJobs = new Set<string>();
+
 function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
 
 let launchCommit: Promise<void> = Promise.resolve();
@@ -391,6 +392,8 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null): P
         const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
             AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }).n;
         if (active + otherPreparations(job.id) >= fleetSettings().createConcurrency) { hold(job, 'Waiting for startup capacity.'); return null; }
+        const capacityBlocker = concurrencyBlocker(job.id, job.action, decision.hostId);
+        if (capacityBlocker) { hold(job, capacityBlocker); return null; }
         if (getHost(decision.hostId)?.lastHeartbeat !== decision.lastHeartbeat) { hold(job, 'Host measurements changed; waiting for another placement check.'); return null; }
         // A short current-state check protects against reservations made while the
         // reader ranked hosts. It never scans resource history or contacts agents.
@@ -435,6 +438,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
             AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
         if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
+        const capacityBlocker = concurrencyBlocker(job.id, job.action);
+        if (capacityBlocker) { hold(job, capacityBlocker); return; }
         if (eligibility(job)) return;
         instance = await placeQueuedCreation(job, instance);
         if (!instance) return;
@@ -582,6 +587,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return;
         }
     }
+    const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id);
+    if (capacityBlocker) { hold(job, capacityBlocker); return; }
     const settings = fleetSettings();
     const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
         AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
@@ -623,6 +630,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
                 hold(job, 'Waiting for startup capacity.'); return;
             }
+            const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id);
+            if (capacityBlocker) { hold(job, capacityBlocker); return; }
             preparingJobs.add(job.id);
             preparedKey = getServerKey(job.server) ?? null;
             if (!preparedKey) throw new Error('The saved server key was removed.');
@@ -653,6 +662,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
                 finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
             }
+            const capacityBlocker = concurrencyBlocker(job.id, job.action, instance!.host_id);
+            if (capacityBlocker) { hold(job, capacityBlocker); return; }
             // Async occupancy checks can overlap another job claiming the final slot.
             if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
                 deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;

src/routes/api/settings/+server.ts

@@ -1,5 +1,5 @@
 import { json } from '@sveltejs/kit';
-import { validateFleetSettings } from '$lib/fleet-settings.js';
+import { validateFleetSettings, concurrencyKeys, retentionKeys } from '$lib/fleet-settings.js';
 import { fleetSettings, saveFleetSettings } from '$lib/server/settings.js';
 
 export function GET() { return json(fleetSettings()); }
@@ -10,3 +10,14 @@ export async function PUT({ request }: { request: Request }) {
     catch (error) { return json({ error: (error as Error).message }, { status: 400 }); }
     return json(saveFleetSettings(settings));
 }
+
+export async function PATCH({ request }: { request: Request }) {
+    try {
+        const { section, values } = await request.json();
+        const keys = section === 'Concurrency' ? concurrencyKeys : section === 'Retention' ? retentionKeys : null;
+        if (!keys || !values || typeof values !== 'object' || Array.isArray(values)
+            || Object.keys(values).length !== keys.length || keys.some(key => !Object.hasOwn(values, key))
+            || Object.keys(values).some(key => !keys.includes(key as typeof keys[number]))) throw new Error('Provide all settings for one section.');
+        return json(saveFleetSettings({ ...fleetSettings(), ...values }));
+    } catch (error) { return json({ error: (error as Error).message }, { status: 400 }); }
+}

src/routes/settings/+page.svelte

@@ -2,7 +2,7 @@
     import { onMount } from 'svelte';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
-    import type { FleetSettings } from '$lib/fleet-settings.js';
+    import { defaultFleetSettings, fleetSettingLabels, additionalConcurrencyKeys, concurrencyKeys, retentionKeys, settingHelp, type FleetSettings } from '$lib/fleet-settings.js';
     let settings = $state<FleetSettings | null>(null);
     let loading = $state(true);
     let saving = $state(false);
@@ -20,14 +20,14 @@
         finally { loading = false; }
     }
 
-    async function saveSettings(event: SubmitEvent) {
+    async function saveSettings(event: SubmitEvent, section: string, keys: (keyof FleetSettings)[]) {
         event.preventDefault();
         saving = true; error = ''; notice = '';
         try {
-            const response = await fetch('/api/settings', { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(settings) });
+            const response = await fetch('/api/settings', { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ section, values: Object.fromEntries(keys.map(key => [key, settings![key]])) }) });
             const result = await response.json();
             if (!response.ok) throw new Error(result.error ?? 'Could not save fleet settings.');
-            settings = result;
+            for (const key of keys) settings![key] = result[key];
             notice = 'Fleet settings saved. Changes apply to the next queue or cleanup check.';
         } catch (cause) { error = (cause as Error).message; }
         finally { saving = false; }
@@ -45,27 +45,23 @@
         {#if loading}
             <p role="status">Loading fleet settings…</p>
         {:else if settings}
-            <form onsubmit={saveSettings}>
-                <fieldset disabled={saving}>
-                    <label for="retention">Campaign history retention (days)</label>
-                    <input id="retention" type="number" min="1" max="36500" step="1" required bind:value={settings.deployKeepDays} aria-describedby="retention-help" />
-                    <p id="retention-help">Default: 7 days. Removes eligible old campaign records and file diffs on the next cleanup check. Pending, failed, held, and health-recovery-linked campaigns are kept. Provisioning history is retained.</p>
-                    <label for="observation-retention">Routine operation history retention (days)</label>
-                    <input id="observation-retention" type="number" min="0" max="36500" step="1" required bind:value={settings.observationKeepDays} aria-describedby="observation-retention-help" />
-                    <p id="observation-retention-help">Default: 1 day. Removes successful monitoring and file-read operation records on the next cleanup check. Failures and actions are retained. Set to 0 to keep these records indefinitely.</p>
-                    <label for="restarts">Concurrent restarts</label>
-                    <input id="restarts" type="number" min="1" step="1" required bind:value={settings.restartConcurrency} aria-describedby="restarts-help" />
-                    <p id="restarts-help">Default: 1 across the fleet. Campaigns, betas, maintenance, and lifecycle actions share recovery slots until container and game health are verified.</p>
-                    <label for="startups">Concurrent server startups</label>
-                    <input id="startups" type="number" min="1" step="1" required bind:value={settings.createConcurrency} aria-describedby="startups-help" />
-                    <p id="startups-help">Default: 1 across collections and individual launches. Each startup holds its slot through container, game health, and selected-map verification.</p>
-                    <label for="destructions">Concurrent server destructions</label>
-                    <input id="destructions" type="number" min="1" step="1" required bind:value={settings.destroyConcurrency} aria-describedby="destructions-help" />
-                    <p id="destructions-help">Default: 1 across individual and fleet removals. Each destruction holds its slot until teardown and local cleanup finish.</p>
-                    <p>Lowering a limit lets active operations finish; queued work waits for capacity. Player safety checks still apply.</p>
-                    <Button type="submit" disabled={saving}>{saving ? 'Saving…' : 'Save fleet settings'}</Button>
-                </fieldset>
-            </form>
+            {#each [{ title: 'Concurrency', keys: concurrencyKeys }, { title: 'Retention', keys: retentionKeys }] as section}
+                <section class="settings-section" aria-labelledby={section.title}>
+                    <h2 id={section.title}>{section.title}</h2>
+                    <form onsubmit={(event) => saveSettings(event, section.title, section.keys)}>
+                        <fieldset disabled={saving}>
+                            {#each section.keys as key}
+                                <label for={key}>{fleetSettingLabels[key]}{key.endsWith('Days') ? ' (days)' : ''}</label>
+                                <input id={key} type="number" min={key === 'observationKeepDays' || additionalConcurrencyKeys.includes(key) ? 0 : 1}
+                                    max={key.endsWith('Days') ? 36500 : undefined} step="1" required bind:value={settings[key]} aria-describedby={`${key}-help`} />
+                                <p id={`${key}-help`}>Default: {defaultFleetSettings[key]}. {settingHelp[key] ?? ''} {additionalConcurrencyKeys.includes(key) ? '0 adds no extra limit.' : ''}</p>
+                            {/each}
+                            {#if section.title === 'Concurrency'}<p>Lowering a limit lets active operations finish; queued work waits for capacity. Player safety checks still apply.</p>{/if}
+                            <Button type="submit" disabled={saving}>{saving ? 'Saving…' : `Save ${section.title.toLowerCase()}`}</Button>
+                        </fieldset>
+                    </form>
+                </section>
+            {/each}
         {:else}
             <Button onclick={loadSettings}>Retry loading</Button>
         {/if}
@@ -76,6 +72,7 @@
 <style>
     .page { display: grid; gap: 20px; }
     .guide { padding: 20px; border: 1px solid var(--border-strong); border-radius: var(--radius-md); background: var(--surface); }
+    .settings-section { margin-top: 24px; padding-top: 24px; border-top: 1px solid var(--border-strong); }
     h2 { font-size: var(--text-md); margin: 0; }
     p { font-size: var(--text-sm); color: var(--text-secondary); line-height: 1.65; }
     fieldset { border: 0; padding: 0; margin: 20px 0 0; min-width: 0; }

tests/unit/concurrency.test.ts

@@ -0,0 +1,45 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture(['t5_test']); });
+afterEach(async () => { await fixture.cleanup(); });
+
+it('counts rebuilds once across pools and keeps their capacity through migration handoff and lower limits', async () => {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+    const { concurrencyBlocker, lifecycleHolders, preparingJobs } = await import('../../src/lib/server/provisioning/concurrency.js');
+    const h = db();
+    h.prepare("INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('i','t5_test','source',1,'lib','img',1,'2026-01-01')").run();
+    h.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','rebuild','waiting','migrating','2026-01-01','test')").run();
+    h.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES('a','i',1)").run();
+    h.prepare("INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id,remove_operation_id,removed_at) VALUES('a','source','i','target','remove','2026-01-02')").run();
+    saveFleetSettings({ ...fleetSettings(), lifecycleConcurrency: 1, rebuildConcurrency: 1 });
+    expect(lifecycleHolders()).toHaveLength(1);
+    expect(concurrencyBlocker('b','stop','other')).toContain('lifecycle');
+    expect(concurrencyBlocker('a','rebuild','target')).toBeNull();
+    saveFleetSettings({ ...fleetSettings(), lifecycleConcurrency: 0 });
+    expect(concurrencyBlocker('b','rebuild','other')).toContain('rebuild');
+    saveFleetSettings({ ...fleetSettings(), rebuildConcurrency: 0, createConcurrencyPerHost: 1 });
+    expect(concurrencyBlocker('b','create','target')).toContain('startup');
+    expect(concurrencyBlocker('b','create','other')).toBeNull();
+    preparingJobs.add('a');
+    expect(lifecycleHolders()).toHaveLength(1);
+    preparingJobs.clear();
+});
+
+it('enforces per-host removal and recovery limits independently of other hosts', async () => {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+    const { concurrencyBlocker } = await import('../../src/lib/server/provisioning/concurrency.js');
+    const h = db();
+    h.prepare("INSERT INTO servers(name) VALUES('t5_test')").run();
+    h.prepare("INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('i','t5_test','host',1,'lib','img',1,'2026-01-01')").run();
+    h.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','destroy','running','removing','2026-01-01','test')").run();
+    h.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES('a','i',1)").run();
+    saveFleetSettings({ ...fleetSettings(), destroyConcurrencyPerHost: 1, restartConcurrencyPerHost: 1 });
+    expect(concurrencyBlocker('b','destroy','host')).toContain('destruction');
+    expect(concurrencyBlocker('b','destroy','other')).toBeNull();
+    h.prepare("INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES('t5_test','2026-01-01','a')").run();
+    expect(concurrencyBlocker('b','restart','host')).toContain('recovery');
+    expect(concurrencyBlocker('b','restart','other')).toBeNull();
+});

tests/unit/fleet-settings.test.ts

@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 
 let fixture: Fixture;
-const defaults = { deployKeepDays: 7, observationKeepDays: 1, restartConcurrency: 1, createConcurrency: 1, destroyConcurrency: 1 };
+import { defaultFleetSettings as defaults } from '../../src/lib/fleet-settings.js';
 beforeEach(async () => {
     vi.resetModules();
     fixture = await newFixture([]);
@@ -14,7 +14,7 @@ describe('fleet settings API and persistence', () => {
     it('returns defaults and saves all settings across a module reload and database reopen', async () => {
         const { GET, PUT } = await import('../../src/routes/api/settings/+server.js');
         expect(await GET().json()).toEqual(defaults);
-        const saved = { deployKeepDays: 30, observationKeepDays: 5, restartConcurrency: 3, createConcurrency: 2, destroyConcurrency: 4 };
+        const saved = { ...defaults, deployKeepDays: 30, observationKeepDays: 5, restartConcurrency: 3, createConcurrency: 2, destroyConcurrency: 4 };
         const response = await PUT({ request: new Request('http://localhost/api/settings', { method: 'PUT', body: JSON.stringify(saved) }) });
         expect(response.status).toBe(200);
         expect(await response.json()).toEqual(saved);
@@ -37,7 +37,7 @@ describe('fleet settings API and persistence', () => {
         vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '4');
         vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
         const { fleetSettings } = await import('../../src/lib/server/settings.js');
-        expect(fleetSettings()).toEqual({ deployKeepDays: 14, observationKeepDays: 1, restartConcurrency: 4, createConcurrency: 2, destroyConcurrency: 1 });
+        expect(fleetSettings()).toEqual({ ...defaults, deployKeepDays: 14, observationKeepDays: 1, restartConcurrency: 4, createConcurrency: 2, destroyConcurrency: 1 });
         closeDb();
         vi.stubEnv('DEPLOY_KEEP_DAYS', '7');
         expect(fleetSettings().deployKeepDays).toBe(14);
@@ -53,7 +53,7 @@ describe('fleet settings API and persistence', () => {
         vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '9');
         vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '8');
         const { GET } = await import('../../src/routes/api/settings/+server.js');
-        const upgraded = { ...previous, destroyConcurrency: 1, observationKeepDays: 1 };
+        const upgraded = { ...defaults, ...previous, destroyConcurrency: 1, observationKeepDays: 1 };
         expect(await GET().json()).toEqual(upgraded);
         expect(JSON.parse((db().prepare("SELECT value FROM settings WHERE key = 'fleet_settings'").get() as { value: string }).value)).toEqual(upgraded);
         closeDb();
@@ -68,7 +68,7 @@ describe('fleet settings API and persistence', () => {
         closeDb();
         vi.stubEnv('DEPLOY_KEEP_DAYS', 'bad');
         const { GET } = await import('../../src/routes/api/settings/+server.js');
-        const upgraded = { ...previous, observationKeepDays: 1 };
+        const upgraded = { ...defaults, ...previous, observationKeepDays: 1 };
         expect(await GET().json()).toEqual(upgraded);
         expect(JSON.parse((db().prepare("SELECT value FROM settings WHERE key = 'fleet_settings'").get() as { value: string }).value)).toEqual(upgraded);
         expect(db().prepare("SELECT value FROM settings WHERE key='unrelated'").get()).toEqual({ value: 'keep' });
@@ -125,3 +125,15 @@ it('applies saved retention to cleanup while protecting unfinished campaigns and
     expect(db().prepare("SELECT server FROM deployment_servers WHERE deployment_id = 'completed'").all()).toEqual([]);
     expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([{ id: 'history' }]);
 });
+
+it('saves a section without overwriting changes in the other section', async () => {
+    const { PATCH, GET } = await import('../../src/routes/api/settings/+server.js');
+    const { concurrencyKeys } = await import('../../src/lib/fleet-settings.js');
+    const { saveFleetSettings } = await import('../../src/lib/server/settings.js');
+    saveFleetSettings({ ...defaults, deployKeepDays: 42 });
+    const response = await PATCH({ request: new Request('http://localhost/api/settings', { method: 'PATCH', body: JSON.stringify({
+        section: 'Concurrency', values: Object.fromEntries(concurrencyKeys.map(key => [key, key === 'rebuildConcurrency' ? 2 : defaults[key]]))
+    }) }) });
+    expect(response.status).toBe(200);
+    expect(await GET().json()).toMatchObject({ deployKeepDays: 42, rebuildConcurrency: 2 });
+});

tests/unit/native-hosts.test.ts

@@ -353,3 +353,20 @@ describe('durable host operations', () => {
         expect(api.getHostOperation(operation.id)?.state).toBe('pending');
     });
 });
+
+it('limits preparing hosts at delivery, preserves replays, and releases capacity after completion', async () => {
+    const api = await enrolled();
+    const second = api.createHost({ name: 'Second' });
+    api.enrollHost(api.createEnrollment(second.id).token, inventory);
+    const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+    saveFleetSettings({ ...fleetSettings(), preparationConcurrency: 1 });
+    const first = api.queueHostOperation(api.host.id, 'library.download', { libraryVersionId: 'one' });
+    const next = api.queueHostOperation(second.id, 'library.download', { libraryVersionId: 'two' });
+    const inspect = api.queueHostOperation(second.id, 'host.inspect', {});
+    expect(api.pendingHostOperations(api.host.id).map(row => row.id)).toContain(first.id);
+    expect(api.pendingHostOperations(second.id).map(row => row.id)).not.toContain(next.id);
+    expect(api.pendingHostOperations(second.id).map(row => row.id)).toContain(inspect.id);
+    expect(api.pendingHostOperations(api.host.id).map(row => row.id)).toContain(first.id);
+    api.reportHostOperation(api.host.id, first.id, { state: 'completed', result: {} });
+    expect(api.pendingHostOperations(second.id).map(row => row.id)).toContain(next.id);
+});

tests/unit/native-provisioning.test.ts

@@ -1372,3 +1372,21 @@ describe('cancelled native mutation bookkeeping', () => {
         expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toHaveLength(delivered ? 0 : 2);
     });
 });
+
+describe('additional concurrency admission', () => {
+    it.each(['createConcurrencyPerHost', 'lifecycleConcurrency'] as const)('holds simultaneous startups with %s until recovery completes', async key => {
+        const first = await queue();
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 5, [key]: 1 });
+        const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
+        const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: 'second-key' });
+        if (!second.ok) throw new Error(second.message);
+        await advance();
+        const operations = [await operationFor(first.deploymentId), await operationFor(second.deploymentId)];
+        expect(operations.filter(Boolean)).toHaveLength(1);
+        expect((await detail(second.deploymentId)).stage).toContain('capacity');
+        saveFleetSettings({ ...fleetSettings(), [key]: 0 });
+        await advance();
+        expect(await operationFor(second.deploymentId)).not.toBeNull();
+    });
+});