Add fleet and per-host concurrency controls

src/lib/fleet-settings.ts

@@ -4,6 +4,13 @@ export interface FleetSettings {
     restartConcurrency: number;
     createConcurrency: number;
     destroyConcurrency: number;
+    restartConcurrencyPerHost: number;
+    createConcurrencyPerHost: number;
+    destroyConcurrencyPerHost: number;
+    lifecycleConcurrency: number;
+    rebuildConcurrency: number;
+    preparationConcurrency: number;
+
 }
 
 export const defaultFleetSettings: FleetSettings = {
@@ -11,7 +18,13 @@ export const defaultFleetSettings: FleetSettings = {
     observationKeepDays: 1,
     restartConcurrency: 1,
     createConcurrency: 1,
-    destroyConcurrency: 1
+    destroyConcurrency: 1,
+    restartConcurrencyPerHost: 0,
+    createConcurrencyPerHost: 0,
+    destroyConcurrencyPerHost: 0,
+    lifecycleConcurrency: 0,
+    rebuildConcurrency: 0,
+    preparationConcurrency: 0,
 };
 
 export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
@@ -19,7 +32,14 @@ export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
     observationKeepDays: 'Routine operation history retention',
     restartConcurrency: 'Concurrent restarts',
     createConcurrency: 'Concurrent server startups',
-    destroyConcurrency: 'Concurrent server destructions'
+    destroyConcurrency: 'Concurrent server destructions',
+    restartConcurrencyPerHost: 'Concurrent restarts per host',
+    createConcurrencyPerHost: 'Concurrent startups per host',
+    destroyConcurrencyPerHost: 'Concurrent destructions per host',
+    lifecycleConcurrency: 'Combined lifecycle limit',
+    rebuildConcurrency: 'Concurrent rebuilds',
+    preparationConcurrency: 'Hosts preparing game files simultaneously',
+
 };
 
 export function validateFleetSettings(input: unknown): FleetSettings {
@@ -30,7 +50,7 @@ export function validateFleetSettings(input: unknown): FleetSettings {
     }
     for (const key of Object.keys(defaultFleetSettings) as (keyof FleetSettings)[]) {
         const value = values[key];
-        const minimum = key === 'observationKeepDays' ? 0 : 1;
+        const minimum = key === 'observationKeepDays' || additionalConcurrencyKeys.includes(key) ? 0 : 1;
         if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) {
             throw new Error(`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`);
         }
@@ -38,6 +58,22 @@ export function validateFleetSettings(input: unknown): FleetSettings {
             throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`);
         }
     }
-    return { deployKeepDays: values.deployKeepDays as number, observationKeepDays: values.observationKeepDays as number, restartConcurrency: values.restartConcurrency as number,
-        createConcurrency: values.createConcurrency as number, destroyConcurrency: values.destroyConcurrency as number };
+    return Object.fromEntries(Object.keys(defaultFleetSettings).map(key => [key, values[key]])) as unknown as FleetSettings;
 }
+
+export const additionalConcurrencyKeys: (keyof FleetSettings)[] = ["restartConcurrencyPerHost", "createConcurrencyPerHost", "destroyConcurrencyPerHost", "lifecycleConcurrency", "rebuildConcurrency", "preparationConcurrency"];
+export const concurrencyKeys: (keyof FleetSettings)[] = ['restartConcurrency', 'createConcurrency', 'destroyConcurrency', ...additionalConcurrencyKeys];
+export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays'];
+export const settingHelp: Partial<Record<keyof FleetSettings, string>> = {
+    restartConcurrency: 'Fleet recovery slots stay occupied until container and game health are verified.',
+    createConcurrency: 'Fleet startup slots cover creation and rebuilds through health and selected-map verification.',
+    destroyConcurrency: 'Fleet removals hold capacity until teardown and cleanup finish.',
+    restartConcurrencyPerHost: 'Additional recovery limit on each host, including starts and rebuilds.',
+    createConcurrencyPerHost: 'Additional startup limit on each host, including rebuilds.',
+    destroyConcurrencyPerHost: 'Additional removal limit on each host, including migration source removal.',
+    lifecycleConcurrency: 'Combined fleet limit for creates, starts, stops, restarts, rebuilds and destructions. Each workflow counts once.',
+    rebuildConcurrency: 'Additional fleet limit for rebuilds, including host drain migrations.',
+    preparationConcurrency: 'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.',
+    deployKeepDays: 'Eligible campaign records and file diffs. Pending, failed, held and recovery-linked campaigns are protected.',
+    observationKeepDays: 'Successful monitoring and file-read records. Failures and actions are retained. 0 keeps records indefinitely.'
+};

src/lib/server/db/fleet-settings.ts

@@ -8,11 +8,8 @@ export function initializeFleetSettings(handle: Database.Database): void {
     const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
     if (saved) {
         const settings = JSON.parse(saved.value);
-        if (!Object.hasOwn(settings, 'destroyConcurrency') || !Object.hasOwn(settings, 'observationKeepDays')) {
-            const upgraded = validateFleetSettings({ destroyConcurrency: defaultFleetSettings.destroyConcurrency,
-                observationKeepDays: defaultFleetSettings.observationKeepDays, ...settings });
-            handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
-        }
+        const upgraded = validateFleetSettings({ ...defaultFleetSettings, ...settings });
+        handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
         return;
     }
     const values = { ...defaultFleetSettings };

src/lib/server/hosts/index.ts

@@ -533,9 +533,18 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
             // sibling's work, but still recovers a lost receipt or agent restart.
             .all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false),
                 ...(lane === 'health' ? [new Date(Date.now() - 30_000).toISOString()] : [])) as OperationDeliveryRow[];
+        const preparationLimit = fleetSettings().preparationConcurrency;
+        const preparing = new Set((database().prepare(`SELECT DISTINCT host_id FROM ec_host_operations
+            WHERE kind IN ('library.download','library.import') AND last_delivered_at IS NOT NULL
+            AND state IN ('pending','accepted','running','unknown')`).all() as { host_id: string }[]).map(row => row.host_id));
+        const admitted = rows.filter(operation => {
+            if (!['library.download','library.import'].includes(operation.kind) || preparing.has(hostId) || !preparationLimit) return true;
+            if (preparing.size >= preparationLimit) return false;
+            preparing.add(hostId); return true;
+        });
         const deliveredAt = now();
-        for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
-        return rows;
+        for (const operation of admitted) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
+        return admitted;
     })();
 }
 /** Payloads never change after enqueueing; fetch one delivery's UTF-8 bytes without decoding. */

src/lib/server/hosts/preparation.ts

@@ -8,7 +8,7 @@ import { scheduleAvailabilityRefresh } from './availability.js';
 
 const activeStates = ['pending', 'accepted', 'running', 'unknown'];
 const supersededDownload = 'Superseded by newer game files before delivery to the host.';
-type PreparationOperation = { id: string; sequence: number; state: HostOperationState; updated_at: string; error: string | null };
+type PreparationOperation = { id: string; sequence: number; last_delivered_at: string | null; state: HostOperationState; updated_at: string; error: string | null };
 type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
 type QueuedSelection = { libraryId: string; image: string | null };
 type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
@@ -50,7 +50,7 @@ function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
 }
 
 function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
-    return db().prepare(`SELECT id,rowid AS sequence,state,updated_at,error FROM ec_host_operations
+    return db().prepare(`SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations
         WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
         .get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
 }
@@ -117,7 +117,9 @@ function targets(host: Host): Target[] {
         if (blocker) { Object.assign(target.item, blocker); continue; }
         if (operation && activeStates.includes(operation.state)) {
             target.item.state = 'preparing';
-            target.item.detail = operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
+            target.item.detail = !target.image && operation.state === 'pending' && !operation.last_delivered_at
+                ? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.'
+                : operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
                 : target.image ? 'Downloading the runtime automatically.'
                 : target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
         } else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {

src/lib/server/provisioning/concurrency.ts

@@ -0,0 +1,47 @@
+import { ensureHostSchema } from '../hosts/schema.js';
+import { db } from '../db/index.js';
+import { fleetSettings } from '../settings.js';
+
+// A preparation owns capacity across asynchronous launch-file generation.
+export const preparingJobs = new Set<string>();
+interface Holder { id: string; action: string; host_id: string | null; target_host_id: string | null; removing: number }
+
+export function lifecycleHolders(): Holder[] {
+    ensureHostSchema(db());
+    return db().prepare(`SELECT d.id,d.action,i.host_id,m.target_host_id,
+        CASE WHEN m.remove_operation_id IS NOT NULL AND m.removed_at IS NULL THEN 1 ELSE 0 END AS removing
+        FROM server_deployments d
+        LEFT JOIN native_provisioning n ON n.deployment_id=d.id
+        LEFT JOIN native_instances i ON i.id=n.instance_id
+        LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
+        WHERE d.state IN ('running','awaiting-health')
+        OR d.id IN (SELECT value FROM json_each(?))
+        OR EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=d.id AND r.healthy_at IS NULL)
+        OR (m.remove_operation_id IS NOT NULL AND d.state NOT IN ('done','warning','cancelled'))
+        OR EXISTS (SELECT 1 FROM ec_host_operations o WHERE o.id IN (n.operation_id,m.remove_operation_id)
+            AND o.state IN ('pending','accepted','running','unknown'))`)
+        .all(JSON.stringify([...preparingJobs])) as Holder[];
+}
+
+/** Recheck inside the command transaction after every asynchronous preflight. */
+export function concurrencyBlocker(id: string, action: string, hostId?: string, removing = false): string | null {
+    const settings = fleetSettings();
+    const holders = lifecycleHolders();
+    const others = holders.filter(row => row.id !== id);
+    const own = holders.find(row => row.id === id);
+    const continuing = own && !preparingJobs.has(id);
+    const full = (limit: number, used: number) => limit > 0 && used >= limit;
+    if (!continuing && full(settings.lifecycleConcurrency, others.length)) return 'Waiting for combined lifecycle capacity.';
+    if (!continuing && action === 'rebuild' && full(settings.rebuildConcurrency, others.filter(row => row.action === 'rebuild').length)) return 'Waiting for rebuild capacity.';
+    if (!hostId || continuing) return null;
+    if (['create','rebuild'].includes(action) && full(settings.createConcurrencyPerHost,
+        others.filter(row => ['create','rebuild'].includes(row.action) && (row.host_id === hostId || row.target_host_id === hostId)).length)) return 'Waiting for startup capacity on this host.';
+    if ((action === 'destroy' || removing) && full(settings.destroyConcurrencyPerHost,
+        others.filter(row => (row.action === 'destroy' || row.removing) && row.host_id === hostId).length)) return 'Waiting for destruction capacity on this host.';
+    if (['start','restart','rebuild'].includes(action)) {
+        const slots = db().prepare(`SELECT r.operation_id FROM restart_rollout r JOIN native_instances i
+            ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND i.host_id=? AND (r.operation_id IS NULL OR r.operation_id!=?)`).all(hostId, id);
+        if (full(settings.restartConcurrencyPerHost, slots.length)) return 'Waiting for restart recovery capacity on this host.';
+    }
+    return null;
+}

src/lib/server/provisioning/host-drain.ts

@@ -1,3 +1,4 @@
+import { concurrencyBlocker } from './concurrency.js';
 import { db } from '../db/index.js';
 import { provisioningCancelled } from './cancellation.js';
 import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
@@ -143,6 +144,8 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance
         const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
         if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
         const { host, port } = choosePlacement(options);
+        const capacityBlocker = concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? concurrencyBlocker(job.id, 'rebuild', host.id);
+        if (capacityBlocker) { hold(capacityBlocker); return; }
         reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
         reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
         const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations

src/lib/server/provisioning/native.ts

@@ -1,3 +1,4 @@
+import { concurrencyBlocker, preparingJobs } from './concurrency.js';
 import { isIP } from 'node:net';
 import { setImmediate as yieldToController } from 'node:timers/promises';
 import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
@@ -356,7 +357,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
 
 const activeJobs = new Map<string, Promise<void>>();
 // Preparation owns startup capacity until its command is committed or discarded.
-const preparingJobs = new Set<string>();
+
 function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
 
 let launchCommit: Promise<void> = Promise.resolve();
@@ -391,6 +392,8 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null): P
         const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
             AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }).n;
         if (active + otherPreparations(job.id) >= fleetSettings().createConcurrency) { hold(job, 'Waiting for startup capacity.'); return null; }
+        const capacityBlocker = concurrencyBlocker(job.id, job.action, decision.hostId);
+        if (capacityBlocker) { hold(job, capacityBlocker); return null; }
         if (getHost(decision.hostId)?.lastHeartbeat !== decision.lastHeartbeat) { hold(job, 'Host measurements changed; waiting for another placement check.'); return null; }
         // A short current-state check protects against reservations made while the
         // reader ranked hosts. It never scans resource history or contacts agents.
@@ -435,6 +438,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
             AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
         if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
+        const capacityBlocker = concurrencyBlocker(job.id, job.action);
+        if (capacityBlocker) { hold(job, capacityBlocker); return; }
         if (eligibility(job)) return;
         instance = await placeQueuedCreation(job, instance);
         if (!instance) return;
@@ -582,6 +587,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return;
         }
     }
+    const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id);
+    if (capacityBlocker) { hold(job, capacityBlocker); return; }
     const settings = fleetSettings();
     const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
         AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
@@ -623,6 +630,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
                 hold(job, 'Waiting for startup capacity.'); return;
             }
+            const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id);
+            if (capacityBlocker) { hold(job, capacityBlocker); return; }
             preparingJobs.add(job.id);
             preparedKey = getServerKey(job.server) ?? null;
             if (!preparedKey) throw new Error('The saved server key was removed.');
@@ -653,6 +662,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
                 finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
             }
+            const capacityBlocker = concurrencyBlocker(job.id, job.action, instance!.host_id);
+            if (capacityBlocker) { hold(job, capacityBlocker); return; }
             // Async occupancy checks can overlap another job claiming the final slot.
             if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
                 deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;