Add fleet and per-host concurrency controls

src/lib/server/provisioning/native.ts

@@ -1,3 +1,4 @@
+import { concurrencyBlocker, preparingJobs } from './concurrency.js';
 import { isIP } from 'node:net';
 import { setImmediate as yieldToController } from 'node:timers/promises';
 import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
@@ -356,7 +357,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
 
 const activeJobs = new Map<string, Promise<void>>();
 // Preparation owns startup capacity until its command is committed or discarded.
-const preparingJobs = new Set<string>();
+
 function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
 
 let launchCommit: Promise<void> = Promise.resolve();
@@ -391,6 +392,8 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null): P
         const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
             AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }).n;
         if (active + otherPreparations(job.id) >= fleetSettings().createConcurrency) { hold(job, 'Waiting for startup capacity.'); return null; }
+        const capacityBlocker = concurrencyBlocker(job.id, job.action, decision.hostId);
+        if (capacityBlocker) { hold(job, capacityBlocker); return null; }
         if (getHost(decision.hostId)?.lastHeartbeat !== decision.lastHeartbeat) { hold(job, 'Host measurements changed; waiting for another placement check.'); return null; }
         // A short current-state check protects against reservations made while the
         // reader ranked hosts. It never scans resource history or contacts agents.
@@ -435,6 +438,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
             AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
         if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
+        const capacityBlocker = concurrencyBlocker(job.id, job.action);
+        if (capacityBlocker) { hold(job, capacityBlocker); return; }
         if (eligibility(job)) return;
         instance = await placeQueuedCreation(job, instance);
         if (!instance) return;
@@ -582,6 +587,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return;
         }
     }
+    const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id);
+    if (capacityBlocker) { hold(job, capacityBlocker); return; }
     const settings = fleetSettings();
     const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
         AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
@@ -623,6 +630,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
                 hold(job, 'Waiting for startup capacity.'); return;
             }
+            const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id);
+            if (capacityBlocker) { hold(job, capacityBlocker); return; }
             preparingJobs.add(job.id);
             preparedKey = getServerKey(job.server) ?? null;
             if (!preparedKey) throw new Error('The saved server key was removed.');
@@ -653,6 +662,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
                 finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
             }
+            const capacityBlocker = concurrencyBlocker(job.id, job.action, instance!.host_id);
+            if (capacityBlocker) { hold(job, capacityBlocker); return; }
             // Async occupancy checks can overlap another job claiming the final slot.
             if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
                 deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;