josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Use live host usage and running estimates for collection admission

Browse Source
This commit is contained in:
josh
2026-10-04 23:41:48 -04:00
parent b0f592b41c
commit caa7672bec
3 changed files with 70 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/capacity.ts
+23 -23
View File
@@ -55,42 +55,42 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
{ id: string; generation: number; pending: number; unsubmitted: number }[];
const { running, startup } = capacityCalibration(host.id);
const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
let extraCpu = (planned + 1) * startup.cpuCores;
let extraMemory = (planned + 1) * startup.memoryBytes;
let reservedCpu = extraCpu;
let reservedMemory = extraMemory;
// Admission budgets the eventual running collection. Dispatch budgets only
// the next startup; the rest of the queue waits for live capacity to recover.
const launch = options.dispatch ? startup : running;
let extraCpu = planned * running.cpuCores + launch.cpuCores;
let extraMemory = planned * running.memoryBytes + launch.memoryBytes;
let reservations = planned + 1;
const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
for (const instance of instances) {
if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue;
reservations++;
const allowance = instance.pending ? startup : running;
reservedCpu += allowance.cpuCores;
reservedMemory += allowance.memoryBytes;
const sample = measurements.get(instance.id);
if (sample?.generation === instance.generation) {
// Host usage already includes this container. Reserve only the
// unused part of its allowance, including while awaiting health.
extraCpu += Math.max(0, allowance.cpuCores - sample.cpuCores);
extraMemory += Math.max(0, allowance.memoryBytes - sample.memoryBytes);
} else if (instance.pending) {
// Missing observations cannot release an uncertain remote launch.
extraCpu += allowance.cpuCores;
extraMemory += allowance.memoryBytes;
if (instance.unsubmitted) {
extraCpu += running.cpuCores;
extraMemory += running.memoryBytes;
reservations++;
continue;
}
const sample = measurements.get(instance.id);
// Current host usage already includes observed containers, including
// those still awaiting health verification. Do not top up to old peaks.
if (sample?.generation === instance.generation || !instance.pending) continue;
// A submitted launch absent from current telemetry may still start.
extraCpu += startup.cpuCores;
extraMemory += startup.memoryBytes;
reservations++;
}
const cores = Math.max(cpuCount * cpuUsagePercent / 100 + extraCpu, reservedCpu);
const cores = cpuCount * cpuUsagePercent / 100 + extraCpu;
const usedMemory = memoryTotal - memoryAvailable;
const memory = Math.max(usedMemory + extraMemory, reservedMemory);
const memory = usedMemory + extraMemory;
const usableMemory = memoryTotal - MEMORY_HEADROOM;
const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`);
if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. `
+ `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(startup.memoryBytes)} startup allowance).`);
+ `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).`);
// Recovered resources must not remain unavailable until the historical
// average catches up. History ranks hosts after current capacity passes.
const historicalMemory = Math.max(memoryTotal - historicalAvailable + extraMemory, reservedMemory);
const historicalCores = Math.max(cpuCount * historicalCpu / 100 + extraCpu, reservedCpu);
const historicalMemory = memoryTotal - historicalAvailable + extraMemory;
const historicalCores = cpuCount * historicalCpu / 100 + extraCpu;
return Math.max(historicalCores / cpuCount, historicalMemory / memoryTotal);
}