Reduce host capacity headroom and remove learned usage padding
This commit is contained in:
@@ -4,7 +4,6 @@ import type { CapacityCalibration, CapacitySample, CapacitySamplePage, HostInven
|
||||
|
||||
const BOOTSTRAP_CPU = 0.5;
|
||||
const BOOTSTRAP_MEMORY = 1024 ** 3;
|
||||
const MARGIN = 1.25;
|
||||
const WINDOW_MS = 24 * 60 * 60_000;
|
||||
const MAX_SAMPLES = 10_000;
|
||||
type Phase = 'startup' | 'running';
|
||||
@@ -83,8 +82,8 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory
|
||||
.all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
|
||||
{ cpu_cores: number; memory_bytes: number; bucket: string }[];
|
||||
const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
|
||||
const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu) * MARGIN;
|
||||
const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory) * MARGIN);
|
||||
const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu);
|
||||
const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory));
|
||||
handle.prepare(`INSERT INTO ec_resource_allowances
|
||||
(scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET
|
||||
cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples,
|
||||
@@ -96,7 +95,7 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory
|
||||
// Raise the allowance immediately on a new peak; a lower sample
|
||||
// cannot reduce it until the next scheduled percentile calculation.
|
||||
handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`)
|
||||
.run(peak.cpu * MARGIN, Math.ceil(peak.memory * MARGIN), timestamp, scope, phase);
|
||||
.run(peak.cpu, Math.ceil(peak.memory), timestamp, scope, phase);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,8 @@ import { capacityCalibration } from './calibration.js';
|
||||
|
||||
// Storage and OS headroom are separate from learned instance CPU/memory usage.
|
||||
const INSTANCE_STORAGE_BYTES = 1024 ** 3;
|
||||
const MEMORY_HEADROOM = 512 * 1024 ** 2;
|
||||
const MEMORY_HEADROOM = 500_000_000;
|
||||
const CPU_USABLE_FRACTION = 0.95;
|
||||
export class CapacityError extends Error {}
|
||||
export interface PlacementOptions {
|
||||
hostId?: string; libraryId: string; image: string; bindAddress?: string;
|
||||
@@ -83,7 +84,7 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
|
||||
const memory = Math.max(usedMemory + extraMemory, reservedMemory);
|
||||
const usableMemory = memoryTotal - MEMORY_HEADROOM;
|
||||
const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
|
||||
if (cores > cpuCount * 0.9) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * 0.9).toFixed(2)} usable; ${reservations} reservations including this launch.`);
|
||||
if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`);
|
||||
if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. `
|
||||
+ `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(startup.memoryBytes)} startup allowance).`);
|
||||
// Recovered resources must not remain unavailable until the historical
|
||||
|
||||
@@ -57,6 +57,7 @@ export function ensureHostSchema(handle: Database.Database): void {
|
||||
CREATE TABLE IF NOT EXISTS ec_resource_allowances (
|
||||
scope TEXT NOT NULL, phase TEXT NOT NULL, cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
|
||||
samples INTEGER NOT NULL, window_start TEXT NOT NULL, observed_at TEXT NOT NULL, computed_at TEXT NOT NULL,
|
||||
usage_margin REAL NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY(scope,phase)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_decommissions (
|
||||
@@ -71,5 +72,14 @@ export function ensureHostSchema(handle: Database.Database): void {
|
||||
if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
|
||||
const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
|
||||
if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
|
||||
const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[];
|
||||
if (!allowanceColumns.some(column => column.name === 'usage_margin')) {
|
||||
// Existing caches include 25% padding. Preserve learned evidence while
|
||||
// removing that padding exactly once, including dormant startup models.
|
||||
handle.transaction(() => {
|
||||
handle.exec('ALTER TABLE ec_resource_allowances ADD COLUMN usage_margin REAL NOT NULL DEFAULT 1');
|
||||
handle.exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)');
|
||||
})();
|
||||
}
|
||||
initialized.add(handle);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user