From a33b897cd0e6f1277d6f9cc05c176aa4a51befe7 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Thu, 8 Oct 2026 11:36:57 -0400 Subject: [PATCH] Use the 95th percentile for startup capacity --- docs/guides/native-hosts.md | 11 +++++----- src/lib/server/hosts/calibration.ts | 2 +- tests/unit/host-capacity-calibration.test.ts | 22 +++++++++++++++++--- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/docs/guides/native-hosts.md b/docs/guides/native-hosts.md index d941fa5..eb01362 100644 --- a/docs/guides/native-hosts.md +++ b/docs/guides/native-hosts.md @@ -255,8 +255,8 @@ startup samples require observing the first five minutes of a container's run. The controller defaults to retaining per-instance minute peaks for 24 hours and caches learned CPU/memory allowances. The first five minutes after a container starts count as -startup; later samples train the running allowance. Startup uses the observed -maximum and running uses the 95th percentile of minute peaks, with **no per-server +startup; later samples train the running allowance. Both phases use the +95th percentile of minute peaks, with **no per-server headroom**. Each calculation considers up to the latest 10,000 minute peaks. Host-local evidence is preferred to account for different hardware; new hosts can borrow the fleet's learned allowance. Startup allowances are never lower @@ -276,7 +276,8 @@ if observations exceed them. They are replaced by measured allowances once the evidence threshold is met; a tiny 0.01-core floor prevents zero-CPU admission. Models expire after 24 hours without measurements. Recalculation happens during heartbeat ingestion, at most once per minute per host/fleet phase; new peaks -raise the cache immediately. Launch admission reads only the cached model. +raise the cache immediately. Existing startup estimates refresh when new startup +measurements arrive. Launch admission reads only the cached model. ### Admission and placement @@ -285,7 +286,7 @@ These allowances are scheduling estimates, not Docker limits. Collection preview estimate how many new servers fit at their running allowance. Jobs may still be queued when that estimate is insufficient. Current host CPU and available memory account for existing containers and other processes. Observed -containers are not topped up to historical startup or running peaks, even while +containers are not topped up to historical startup or running allowances, even while awaiting health verification. Established instances are covered by host usage when individual container measurements are unavailable. @@ -294,7 +295,7 @@ Unplaced queued launches wait their turn without holding physical resources. Any previously assigned reservations remain protected. Submitted or uncertain launches missing a matching-generation measurement still consume a full startup allowance until telemetry accounts for them. This lets collections start as capacity -becomes available, without stacking every queued server's startup peak. +becomes available, without stacking every queued server's startup allowance. Capacity errors report projected usage, usable capacity and reservations. When startup capacity is available, provisioning checks each eligible host’s diff --git a/src/lib/server/hosts/calibration.ts b/src/lib/server/hosts/calibration.ts index 0a86147..92a99da 100644 --- a/src/lib/server/hosts/calibration.ts +++ b/src/lib/server/hosts/calibration.ts @@ -82,7 +82,7 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`) .all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as { cpu_cores: number; memory_bytes: number; bucket: string }[]; - const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1]; + const percentile = (values: number[]) => values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1]; const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu); const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory)); handle.prepare(`INSERT INTO ec_resource_allowances diff --git a/tests/unit/host-capacity-calibration.test.ts b/tests/unit/host-capacity-calibration.test.ts index 826deb9..07e7d75 100644 --- a/tests/unit/host-capacity-calibration.test.ts +++ b/tests/unit/host-capacity-calibration.test.ts @@ -196,13 +196,29 @@ describe('measured instance capacity', () => { expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1); expect(capacityCalibration(hostId).running.source).toBe('bootstrap'); }); - it('uses a running percentile and startup peaks without padding, keeping admission reads cached', async () => { - for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 }); + it.each(['startup', 'running'] as const)('uses the %s 95th percentile for CPU and memory, keeping admission reads cached', async phase => { + await learned(); + // Repeated starts or running observations include isolated CPU and + // memory outliers in different minutes; neither should set the allowance. + const uptimeSeconds = phase === 'startup' ? 60 : 600; + for (let minute = 25; minute < 125; minute++) await report(minute, { + uptimeSeconds, cpuCores: minute === 25 ? 2 : 0.4, memoryBytes: minute === 26 ? 3 * GiB : GiB / 2 + }); const { db } = await import('../../src/lib/server/db/index.js'); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); const prepare = vi.spyOn(db(), 'prepare'); - expect(capacityCalibration(hostId).running.cpuCores).toBe(0.1); + expect(capacityCalibration(hostId)[phase]).toMatchObject({ source: 'host', cpuCores: 0.4, memoryBytes: GiB / 2 }); + expect(capacityCalibration('new-host')[phase]).toMatchObject({ source: 'fleet', cpuCores: 0.4, memoryBytes: GiB / 2 }); expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false); prepare.mockRestore(); + + // Both phases still raise immediately for fresh spikes, hold that value + // within the minute, then return to P95 on the next recalculation. + await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB }); + expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB }); + await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 }); + expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB }); + await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 }); + expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 0.4, memoryBytes: GiB / 2 }); }); });