Use the 95th percentile for startup capacity
This commit is contained in:
@@ -255,8 +255,8 @@ startup samples require observing the first five minutes of a container's run.
|
||||
|
||||
The controller defaults to retaining per-instance minute peaks for 24 hours and caches learned
|
||||
CPU/memory allowances. The first five minutes after a container starts count as
|
||||
startup; later samples train the running allowance. Startup uses the observed
|
||||
maximum and running uses the 95th percentile of minute peaks, with **no per-server
|
||||
startup; later samples train the running allowance. Both phases use the
|
||||
95th percentile of minute peaks, with **no per-server
|
||||
headroom**. Each calculation considers up to the latest 10,000 minute peaks.
|
||||
Host-local evidence is preferred to account for different hardware; new hosts
|
||||
can borrow the fleet's learned allowance. Startup allowances are never lower
|
||||
@@ -276,7 +276,8 @@ if observations exceed them. They are replaced by measured allowances once the
|
||||
evidence threshold is met; a tiny 0.01-core floor prevents zero-CPU admission.
|
||||
Models expire after 24 hours without measurements. Recalculation happens during
|
||||
heartbeat ingestion, at most once per minute per host/fleet phase; new peaks
|
||||
raise the cache immediately. Launch admission reads only the cached model.
|
||||
raise the cache immediately. Existing startup estimates refresh when new startup
|
||||
measurements arrive. Launch admission reads only the cached model.
|
||||
|
||||
### Admission and placement
|
||||
|
||||
@@ -285,7 +286,7 @@ These allowances are scheduling estimates, not Docker limits. Collection preview
|
||||
estimate how many new servers fit at their running allowance. Jobs may still be
|
||||
queued when that estimate is insufficient. Current host CPU and
|
||||
available memory account for existing containers and other processes. Observed
|
||||
containers are not topped up to historical startup or running peaks, even while
|
||||
containers are not topped up to historical startup or running allowances, even while
|
||||
awaiting health verification. Established instances are covered by host usage
|
||||
when individual container measurements are unavailable.
|
||||
|
||||
@@ -294,7 +295,7 @@ Unplaced queued launches wait their turn without holding physical resources.
|
||||
Any previously assigned reservations remain protected. Submitted or uncertain launches missing a matching-generation
|
||||
measurement still consume a full startup allowance until telemetry accounts for
|
||||
them. This lets collections start as capacity
|
||||
becomes available, without stacking every queued server's startup peak.
|
||||
becomes available, without stacking every queued server's startup allowance.
|
||||
Capacity errors report projected usage, usable capacity and reservations.
|
||||
|
||||
When startup capacity is available, provisioning checks each eligible host’s
|
||||
|
||||
@@ -82,7 +82,7 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory
|
||||
WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
|
||||
.all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
|
||||
{ cpu_cores: number; memory_bytes: number; bucket: string }[];
|
||||
const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
|
||||
const percentile = (values: number[]) => values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1];
|
||||
const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu);
|
||||
const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory));
|
||||
handle.prepare(`INSERT INTO ec_resource_allowances
|
||||
|
||||
@@ -196,13 +196,29 @@ describe('measured instance capacity', () => {
|
||||
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1);
|
||||
expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
|
||||
});
|
||||
it('uses a running percentile and startup peaks without padding, keeping admission reads cached', async () => {
|
||||
for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 });
|
||||
it.each(['startup', 'running'] as const)('uses the %s 95th percentile for CPU and memory, keeping admission reads cached', async phase => {
|
||||
await learned();
|
||||
// Repeated starts or running observations include isolated CPU and
|
||||
// memory outliers in different minutes; neither should set the allowance.
|
||||
const uptimeSeconds = phase === 'startup' ? 60 : 600;
|
||||
for (let minute = 25; minute < 125; minute++) await report(minute, {
|
||||
uptimeSeconds, cpuCores: minute === 25 ? 2 : 0.4, memoryBytes: minute === 26 ? 3 * GiB : GiB / 2
|
||||
});
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
||||
const prepare = vi.spyOn(db(), 'prepare');
|
||||
expect(capacityCalibration(hostId).running.cpuCores).toBe(0.1);
|
||||
expect(capacityCalibration(hostId)[phase]).toMatchObject({ source: 'host', cpuCores: 0.4, memoryBytes: GiB / 2 });
|
||||
expect(capacityCalibration('new-host')[phase]).toMatchObject({ source: 'fleet', cpuCores: 0.4, memoryBytes: GiB / 2 });
|
||||
expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false);
|
||||
prepare.mockRestore();
|
||||
|
||||
// Both phases still raise immediately for fresh spikes, hold that value
|
||||
// within the minute, then return to P95 on the next recalculation.
|
||||
await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB });
|
||||
expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB });
|
||||
await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
|
||||
expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB });
|
||||
await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
|
||||
expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 0.4, memoryBytes: GiB / 2 });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user