Use the 95th percentile for startup capacity

This commit is contained in:
2026-10-08 11:36:57 -04:00
parent 8863e8a882
commit a33b897cd0
3 changed files with 26 additions and 9 deletions
+6 -5
View File
@@ -255,8 +255,8 @@ startup samples require observing the first five minutes of a container's run.
The controller defaults to retaining per-instance minute peaks for 24 hours and caches learned
CPU/memory allowances. The first five minutes after a container starts count as
startup; later samples train the running allowance. Startup uses the observed
maximum and running uses the 95th percentile of minute peaks, with **no per-server
startup; later samples train the running allowance. Both phases use the
95th percentile of minute peaks, with **no per-server
headroom**. Each calculation considers up to the latest 10,000 minute peaks.
Host-local evidence is preferred to account for different hardware; new hosts
can borrow the fleet's learned allowance. Startup allowances are never lower
@@ -276,7 +276,8 @@ if observations exceed them. They are replaced by measured allowances once the
evidence threshold is met; a tiny 0.01-core floor prevents zero-CPU admission.
Models expire after 24 hours without measurements. Recalculation happens during
heartbeat ingestion, at most once per minute per host/fleet phase; new peaks
raise the cache immediately. Launch admission reads only the cached model.
raise the cache immediately. Existing startup estimates refresh when new startup
measurements arrive. Launch admission reads only the cached model.
### Admission and placement
@@ -285,7 +286,7 @@ These allowances are scheduling estimates, not Docker limits. Collection preview
estimate how many new servers fit at their running allowance. Jobs may still be
queued when that estimate is insufficient. Current host CPU and
available memory account for existing containers and other processes. Observed
containers are not topped up to historical startup or running peaks, even while
containers are not topped up to historical startup or running allowances, even while
awaiting health verification. Established instances are covered by host usage
when individual container measurements are unavailable.
@@ -294,7 +295,7 @@ Unplaced queued launches wait their turn without holding physical resources.
Any previously assigned reservations remain protected. Submitted or uncertain launches missing a matching-generation
measurement still consume a full startup allowance until telemetry accounts for
them. This lets collections start as capacity
becomes available, without stacking every queued server's startup peak.
becomes available, without stacking every queued server's startup allowance.
Capacity errors report projected usage, usable capacity and reservations.
When startup capacity is available, provisioning checks each eligible host’s
+1 -1
View File
@@ -82,7 +82,7 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory
WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
.all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
{ cpu_cores: number; memory_bytes: number; bucket: string }[];
const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
const percentile = (values: number[]) => values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1];
const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu);
const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory));
handle.prepare(`INSERT INTO ec_resource_allowances
+19 -3
View File
@@ -196,13 +196,29 @@ describe('measured instance capacity', () => {
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1);
expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
});
it('uses a running percentile and startup peaks without padding, keeping admission reads cached', async () => {
for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 });
it.each(['startup', 'running'] as const)('uses the %s 95th percentile for CPU and memory, keeping admission reads cached', async phase => {
await learned();
// Repeated starts or running observations include isolated CPU and
// memory outliers in different minutes; neither should set the allowance.
const uptimeSeconds = phase === 'startup' ? 60 : 600;
for (let minute = 25; minute < 125; minute++) await report(minute, {
uptimeSeconds, cpuCores: minute === 25 ? 2 : 0.4, memoryBytes: minute === 26 ? 3 * GiB : GiB / 2
});
const { db } = await import('../../src/lib/server/db/index.js');
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
const prepare = vi.spyOn(db(), 'prepare');
expect(capacityCalibration(hostId).running.cpuCores).toBe(0.1);
expect(capacityCalibration(hostId)[phase]).toMatchObject({ source: 'host', cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(capacityCalibration('new-host')[phase]).toMatchObject({ source: 'fleet', cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false);
prepare.mockRestore();
// Both phases still raise immediately for fresh spikes, hold that value
// within the minute, then return to P95 on the next recalculation.
await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB });
expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB });
await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB });
await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 0.4, memoryBytes: GiB / 2 });
});
});