Add fleet and per-host concurrency controls

src/lib/server/provisioning/concurrency.ts

@@ -0,0 +1,47 @@
+import { ensureHostSchema } from '../hosts/schema.js';
+import { db } from '../db/index.js';
+import { fleetSettings } from '../settings.js';
+
+// A preparation owns capacity across asynchronous launch-file generation.
+export const preparingJobs = new Set<string>();
+interface Holder { id: string; action: string; host_id: string | null; target_host_id: string | null; removing: number }
+
+export function lifecycleHolders(): Holder[] {
+    ensureHostSchema(db());
+    return db().prepare(`SELECT d.id,d.action,i.host_id,m.target_host_id,
+        CASE WHEN m.remove_operation_id IS NOT NULL AND m.removed_at IS NULL THEN 1 ELSE 0 END AS removing
+        FROM server_deployments d
+        LEFT JOIN native_provisioning n ON n.deployment_id=d.id
+        LEFT JOIN native_instances i ON i.id=n.instance_id
+        LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
+        WHERE d.state IN ('running','awaiting-health')
+        OR d.id IN (SELECT value FROM json_each(?))
+        OR EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=d.id AND r.healthy_at IS NULL)
+        OR (m.remove_operation_id IS NOT NULL AND d.state NOT IN ('done','warning','cancelled'))
+        OR EXISTS (SELECT 1 FROM ec_host_operations o WHERE o.id IN (n.operation_id,m.remove_operation_id)
+            AND o.state IN ('pending','accepted','running','unknown'))`)
+        .all(JSON.stringify([...preparingJobs])) as Holder[];
+}
+
+/** Recheck inside the command transaction after every asynchronous preflight. */
+export function concurrencyBlocker(id: string, action: string, hostId?: string, removing = false): string | null {
+    const settings = fleetSettings();
+    const holders = lifecycleHolders();
+    const others = holders.filter(row => row.id !== id);
+    const own = holders.find(row => row.id === id);
+    const continuing = own && !preparingJobs.has(id);
+    const full = (limit: number, used: number) => limit > 0 && used >= limit;
+    if (!continuing && full(settings.lifecycleConcurrency, others.length)) return 'Waiting for combined lifecycle capacity.';
+    if (!continuing && action === 'rebuild' && full(settings.rebuildConcurrency, others.filter(row => row.action === 'rebuild').length)) return 'Waiting for rebuild capacity.';
+    if (!hostId || continuing) return null;
+    if (['create','rebuild'].includes(action) && full(settings.createConcurrencyPerHost,
+        others.filter(row => ['create','rebuild'].includes(row.action) && (row.host_id === hostId || row.target_host_id === hostId)).length)) return 'Waiting for startup capacity on this host.';
+    if ((action === 'destroy' || removing) && full(settings.destroyConcurrencyPerHost,
+        others.filter(row => (row.action === 'destroy' || row.removing) && row.host_id === hostId).length)) return 'Waiting for destruction capacity on this host.';
+    if (['start','restart','rebuild'].includes(action)) {
+        const slots = db().prepare(`SELECT r.operation_id FROM restart_rollout r JOIN native_instances i
+            ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND i.host_id=? AND (r.operation_id IS NULL OR r.operation_id!=?)`).all(hostId, id);
+        if (full(settings.restartConcurrencyPerHost, slots.length)) return 'Waiting for restart recovery capacity on this host.';
+    }
+    return null;
+}