josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add fleet and per-host concurrency controls

Browse Source
This commit is contained in:
josh
2026-10-05 22:27:27 -04:00
parent c3b990c5fd
commit 7c98df4fb0
14 changed files with 268 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/concurrency.ts
+47
View File
@@ -0,0 +1,47 @@
import { ensureHostSchema } from '../hosts/schema.js';
import { db } from '../db/index.js';
import { fleetSettings } from '../settings.js';
// A preparation owns capacity across asynchronous launch-file generation.
export const preparingJobs = new Set<string>();
interface Holder { id: string; action: string; host_id: string | null; target_host_id: string | null; removing: number }
export function lifecycleHolders(): Holder[] {
ensureHostSchema(db());
return db().prepare(`SELECT d.id,d.action,i.host_id,m.target_host_id,
CASE WHEN m.remove_operation_id IS NOT NULL AND m.removed_at IS NULL THEN 1 ELSE 0 END AS removing
FROM server_deployments d
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
LEFT JOIN native_instances i ON i.id=n.instance_id
LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
WHERE d.state IN ('running','awaiting-health')
OR d.id IN (SELECT value FROM json_each(?))
OR EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=d.id AND r.healthy_at IS NULL)
OR (m.remove_operation_id IS NOT NULL AND d.state NOT IN ('done','warning','cancelled'))
OR EXISTS (SELECT 1 FROM ec_host_operations o WHERE o.id IN (n.operation_id,m.remove_operation_id)
AND o.state IN ('pending','accepted','running','unknown'))`)
.all(JSON.stringify([...preparingJobs])) as Holder[];
}
/** Recheck inside the command transaction after every asynchronous preflight. */
export function concurrencyBlocker(id: string, action: string, hostId?: string, removing = false): string | null {
const settings = fleetSettings();
const holders = lifecycleHolders();
const others = holders.filter(row => row.id !== id);
const own = holders.find(row => row.id === id);
const continuing = own && !preparingJobs.has(id);
const full = (limit: number, used: number) => limit > 0 && used >= limit;
if (!continuing && full(settings.lifecycleConcurrency, others.length)) return 'Waiting for combined lifecycle capacity.';
if (!continuing && action === 'rebuild' && full(settings.rebuildConcurrency, others.filter(row => row.action === 'rebuild').length)) return 'Waiting for rebuild capacity.';
if (!hostId || continuing) return null;
if (['create','rebuild'].includes(action) && full(settings.createConcurrencyPerHost,
others.filter(row => ['create','rebuild'].includes(row.action) && (row.host_id === hostId || row.target_host_id === hostId)).length)) return 'Waiting for startup capacity on this host.';
if ((action === 'destroy' || removing) && full(settings.destroyConcurrencyPerHost,
others.filter(row => (row.action === 'destroy' || row.removing) && row.host_id === hostId).length)) return 'Waiting for destruction capacity on this host.';
if (['start','restart','rebuild'].includes(action)) {
const slots = db().prepare(`SELECT r.operation_id FROM restart_rollout r JOIN native_instances i
ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND i.host_id=? AND (r.operation_id IS NULL OR r.operation_id!=?)`).all(hostId, id);
if (full(settings.restartConcurrencyPerHost, slots.length)) return 'Waiting for restart recovery capacity on this host.';
}
return null;
}
src/lib/server/provisioning/host-drain.ts
+3
View File
@@ -1,3 +1,4 @@
import { concurrencyBlocker } from './concurrency.js';
import { db } from '../db/index.js';
import { provisioningCancelled } from './cancellation.js';
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
@@ -143,6 +144,8 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance
const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
const { host, port } = choosePlacement(options);
const capacityBlocker = concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? concurrencyBlocker(job.id, 'rebuild', host.id);
if (capacityBlocker) { hold(capacityBlocker); return; }
reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
src/lib/server/provisioning/native.ts
+12 -1
View File
@@ -1,3 +1,4 @@
import { concurrencyBlocker, preparingJobs } from './concurrency.js';
import { isIP } from 'node:net';
import { setImmediate as yieldToController } from 'node:timers/promises';
import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
@@ -356,7 +357,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
const activeJobs = new Map<string, Promise<void>>();
// Preparation owns startup capacity until its command is committed or discarded.
const preparingJobs = new Set<string>();
function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
let launchCommit: Promise<void> = Promise.resolve();
@@ -391,6 +392,8 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null): P
const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }).n;
if (active + otherPreparations(job.id) >= fleetSettings().createConcurrency) { hold(job, 'Waiting for startup capacity.'); return null; }
const capacityBlocker = concurrencyBlocker(job.id, job.action, decision.hostId);
if (capacityBlocker) { hold(job, capacityBlocker); return null; }
if (getHost(decision.hostId)?.lastHeartbeat !== decision.lastHeartbeat) { hold(job, 'Host measurements changed; waiting for another placement check.'); return null; }
// A short current-state check protects against reservations made while the
// reader ranked hosts. It never scans resource history or contacts agents.
@@ -435,6 +438,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
const capacityBlocker = concurrencyBlocker(job.id, job.action);
if (capacityBlocker) { hold(job, capacityBlocker); return; }
if (eligibility(job)) return;
instance = await placeQueuedCreation(job, instance);
if (!instance) return;
@@ -582,6 +587,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return;
}
}
const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id);
if (capacityBlocker) { hold(job, capacityBlocker); return; }
const settings = fleetSettings();
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
@@ -623,6 +630,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
hold(job, 'Waiting for startup capacity.'); return;
}
const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id);
if (capacityBlocker) { hold(job, capacityBlocker); return; }
preparingJobs.add(job.id);
preparedKey = getServerKey(job.server) ?? null;
if (!preparedKey) throw new Error('The saved server key was removed.');
@@ -653,6 +662,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
}
const capacityBlocker = concurrencyBlocker(job.id, job.action, instance!.host_id);
if (capacityBlocker) { hold(job, capacityBlocker); return; }
// Async occupancy checks can overlap another job claiming the final slot.
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;