josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix capacity accounting and queued host placement

Browse Source
This commit is contained in:
josh
2026-10-04 21:28:15 -04:00
parent e6cdf9eb7f
commit 8a31de92fa
4 changed files with 160 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/capacity.ts
+54 -22
View File
@@ -11,6 +11,9 @@ export class CapacityError extends Error {}
export interface PlacementOptions {
hostId?: string; libraryId: string; image: string; bindAddress?: string;
thorough?: boolean; excludeInstanceId?: string;
// Execution only: not-yet-submitted launches wait their turn instead of
// requiring their entire backlog to fit before any one can start.
dispatch?: boolean;
planned?: { hostId: string; port: number }[];
}
@@ -31,34 +34,63 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) {
throw new Error('Waiting for valid CPU and memory measurements.');
}
let cpu = cpuUsagePercent;
let available = memoryAvailable;
let historicalCpu = cpuUsagePercent;
let historicalAvailable = memoryAvailable;
if (options.thorough) {
const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`)
.get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null };
cpu = Math.max(cpu, history.cpu ?? cpu);
available = Math.min(available, history.memory ?? available);
historicalCpu = Math.max(cpuUsagePercent, history.cpu ?? cpuUsagePercent);
historicalAvailable = Math.min(memoryAvailable, history.memory ?? memoryAvailable);
}
// Unfinished/uncertain launches remain charged until recovery and a later
// heartbeat. Observed usage accounts for established servers; the count
// floor also reserves their minimum footprint while they are idle/stopped.
const instances = db().prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(NOT EXISTS (
SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?
)),0) AS pending FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
.get(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as { total: number; pending: number };
const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
const count = instances.total + planned + 1;
const pending = instances.pending + planned + 1;
const instances = db().prepare(`SELECT i.id, i.generation,
NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE n.instance_id=i.id AND d.action='create' AND d.state='queued' AND d.submitted_at IS NULL
AND n.operation_id IS NULL AND n.stage_operation_id IS NULL) AS unsubmitted
FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
.all(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as
{ id: string; generation: number; pending: number; unsubmitted: number }[];
const { running, startup } = capacityCalibration(host.id);
const cores = Math.max(cpuCount * cpu / 100 + pending * startup.cpuCores,
(count - pending) * running.cpuCores + pending * startup.cpuCores);
const memory = Math.max(memoryTotal - available + pending * startup.memoryBytes,
(count - pending) * running.memoryBytes + pending * startup.memoryBytes);
if (cores > cpuCount * 0.9) throw new Error('Insufficient CPU capacity.');
if (memory > memoryTotal - MEMORY_HEADROOM) throw new Error('Insufficient memory capacity.');
return Math.max(cores / cpuCount, memory / memoryTotal);
const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
let extraCpu = (planned + 1) * startup.cpuCores;
let extraMemory = (planned + 1) * startup.memoryBytes;
let reservedCpu = extraCpu;
let reservedMemory = extraMemory;
let reservations = planned + 1;
const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
for (const instance of instances) {
if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue;
reservations++;
const allowance = instance.pending ? startup : running;
reservedCpu += allowance.cpuCores;
reservedMemory += allowance.memoryBytes;
const sample = measurements.get(instance.id);
if (sample?.generation === instance.generation) {
// Host usage already includes this container. Reserve only the
// unused part of its allowance, including while awaiting health.
extraCpu += Math.max(0, allowance.cpuCores - sample.cpuCores);
extraMemory += Math.max(0, allowance.memoryBytes - sample.memoryBytes);
} else if (instance.pending) {
// Missing observations cannot release an uncertain remote launch.
extraCpu += allowance.cpuCores;
extraMemory += allowance.memoryBytes;
}
}
const cores = Math.max(cpuCount * cpuUsagePercent / 100 + extraCpu, reservedCpu);
const usedMemory = memoryTotal - memoryAvailable;
const memory = Math.max(usedMemory + extraMemory, reservedMemory);
const usableMemory = memoryTotal - MEMORY_HEADROOM;
const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
if (cores > cpuCount * 0.9) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * 0.9).toFixed(2)} usable; ${reservations} reservations including this launch.`);
if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. `
+ `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(startup.memoryBytes)} startup allowance).`);
// Recovered resources must not remain unavailable until the historical
// average catches up. History ranks hosts after current capacity passes.
const historicalMemory = Math.max(memoryTotal - historicalAvailable + extraMemory, reservedMemory);
const historicalCores = Math.max(cpuCount * historicalCpu / 100 + extraCpu, reservedCpu);
return Math.max(historicalCores / cpuCount, historicalMemory / memoryTotal);
}
/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */
src/lib/server/provisioning/native.ts
+2 -2
View File
@@ -276,7 +276,7 @@ function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance
const network = prior.automatic ? prior.network ?? {} : prior;
const { host, port } = choosePlacement({ hostId: prior.automatic ? undefined : instance.host_id,
libraryId: instance.library_id, image: instance.image, bindAddress: network.bindAddress,
thorough: true, excludeInstanceId: instance.id });
thorough: true, dispatch: true, excludeInstanceId: instance.id });
const bind = network.bindAddress ?? host.bindAddress;
releaseHostPorts(instance.id);
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'udp' });
@@ -433,7 +433,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true });
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true });
if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
if (job.action !== 'create' && job.action !== 'start') {
// Fast controller check keeps occupied lobbies out of the operation slots.