From b0f592b41cf02980b209e75bfd5a7c9e0dd697e8 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sun, 4 Oct 2026 23:33:21 -0400 Subject: [PATCH] Reduce host capacity headroom and remove learned usage padding --- docs/native-hosts.md | 6 ++-- src/lib/server/hosts/calibration.ts | 7 ++--- src/lib/server/hosts/capacity.ts | 5 +-- src/lib/server/hosts/schema.ts | 10 ++++++ src/lib/ui/components/CapacitySamples.svelte | 2 +- src/routes/hosts/[id]/+page.svelte | 2 +- tests/unit/host-capacity-calibration.test.ts | 33 ++++++++++++++------ tests/unit/native-provisioning.test.ts | 20 ++++++++++-- 8 files changed, 62 insertions(+), 23 deletions(-) diff --git a/docs/native-hosts.md b/docs/native-hosts.md index 99b07a5..9d1234b 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -157,7 +157,7 @@ startup samples require observing the first five minutes of a container's run. The controller retains per-instance minute peaks for 24 hours and caches learned CPU/memory allowances. The first five minutes after a container starts count as startup; later samples train the running allowance. Startup uses the observed -maximum and running uses the 95th percentile of minute peaks, both with **25% +maximum and running uses the 95th percentile of minute peaks, with **no per-server headroom**. Each calculation considers up to the latest 10,000 minute peaks. Host-local evidence is preferred to account for different hardware; new hosts can borrow the fleet's learned allowance. Startup allowances are never lower @@ -165,7 +165,7 @@ than running allowances. The host page shows both allowances and their source. Expand **View samples** in Learned server capacity to inspect retained minute peaks for the last 24 hours. Filter by startup/running phase or switch from this host to the entire fleet. Rows show time, server and generation, CPU cores, and -memory in MiB before headroom; CPU and memory peaks may come from different +memory in MiB; CPU and memory peaks may come from different heartbeats. Results are paginated, and **Refresh samples** loads the latest stored measurements without contacting agents. The allowance count may lag the table until the next calculation or reflect a different source. @@ -179,7 +179,7 @@ Models expire after 24 hours without measurements. Recalculation happens during heartbeat ingestion, at most once per minute per host/fleet phase; new peaks raise the cache immediately. Launch admission reads only the cached model. -Scheduling still leaves **10% host CPU and 512 MiB RAM** for the OS and bursts. +Scheduling still leaves **5% host CPU and 500 MB RAM (500,000,000 bytes)** for the OS and bursts. These allowances are scheduling reservations, not Docker limits. Queued and unresolved creations consume the startup allowance; established instances keep the running allowance even while idle or stopped. Current container measurements diff --git a/src/lib/server/hosts/calibration.ts b/src/lib/server/hosts/calibration.ts index 89db95c..6f2a1ce 100644 --- a/src/lib/server/hosts/calibration.ts +++ b/src/lib/server/hosts/calibration.ts @@ -4,7 +4,6 @@ import type { CapacityCalibration, CapacitySample, CapacitySamplePage, HostInven const BOOTSTRAP_CPU = 0.5; const BOOTSTRAP_MEMORY = 1024 ** 3; -const MARGIN = 1.25; const WINDOW_MS = 24 * 60 * 60_000; const MAX_SAMPLES = 10_000; type Phase = 'startup' | 'running'; @@ -83,8 +82,8 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory .all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as { cpu_cores: number; memory_bytes: number; bucket: string }[]; const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1]; - const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu) * MARGIN; - const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory) * MARGIN); + const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu); + const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory)); handle.prepare(`INSERT INTO ec_resource_allowances (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples, @@ -96,7 +95,7 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory // Raise the allowance immediately on a new peak; a lower sample // cannot reduce it until the next scheduled percentile calculation. handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`) - .run(peak.cpu * MARGIN, Math.ceil(peak.memory * MARGIN), timestamp, scope, phase); + .run(peak.cpu, Math.ceil(peak.memory), timestamp, scope, phase); } } } diff --git a/src/lib/server/hosts/capacity.ts b/src/lib/server/hosts/capacity.ts index bc30638..04d6a3a 100644 --- a/src/lib/server/hosts/capacity.ts +++ b/src/lib/server/hosts/capacity.ts @@ -6,7 +6,8 @@ import { capacityCalibration } from './calibration.js'; // Storage and OS headroom are separate from learned instance CPU/memory usage. const INSTANCE_STORAGE_BYTES = 1024 ** 3; -const MEMORY_HEADROOM = 512 * 1024 ** 2; +const MEMORY_HEADROOM = 500_000_000; +const CPU_USABLE_FRACTION = 0.95; export class CapacityError extends Error {} export interface PlacementOptions { hostId?: string; libraryId: string; image: string; bindAddress?: string; @@ -83,7 +84,7 @@ function projectedLoad(host: Host, options: PlacementOptions): number { const memory = Math.max(usedMemory + extraMemory, reservedMemory); const usableMemory = memoryTotal - MEMORY_HEADROOM; const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`; - if (cores > cpuCount * 0.9) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * 0.9).toFixed(2)} usable; ${reservations} reservations including this launch.`); + if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`); if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. ` + `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(startup.memoryBytes)} startup allowance).`); // Recovered resources must not remain unavailable until the historical diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts index 702cf0d..0075062 100644 --- a/src/lib/server/hosts/schema.ts +++ b/src/lib/server/hosts/schema.ts @@ -57,6 +57,7 @@ export function ensureHostSchema(handle: Database.Database): void { CREATE TABLE IF NOT EXISTS ec_resource_allowances ( scope TEXT NOT NULL, phase TEXT NOT NULL, cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL, samples INTEGER NOT NULL, window_start TEXT NOT NULL, observed_at TEXT NOT NULL, computed_at TEXT NOT NULL, + usage_margin REAL NOT NULL DEFAULT 1, PRIMARY KEY(scope,phase) ); CREATE TABLE IF NOT EXISTS ec_host_decommissions ( @@ -71,5 +72,14 @@ export function ensureHostSchema(handle: Database.Database): void { if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT'); const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[]; if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT'); + const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[]; + if (!allowanceColumns.some(column => column.name === 'usage_margin')) { + // Existing caches include 25% padding. Preserve learned evidence while + // removing that padding exactly once, including dormant startup models. + handle.transaction(() => { + handle.exec('ALTER TABLE ec_resource_allowances ADD COLUMN usage_margin REAL NOT NULL DEFAULT 1'); + handle.exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)'); + })(); + } initialized.add(handle); } diff --git a/src/lib/ui/components/CapacitySamples.svelte b/src/lib/ui/components/CapacitySamples.svelte index dc5acd2..7932d6a 100644 --- a/src/lib/ui/components/CapacitySamples.svelte +++ b/src/lib/ui/components/CapacitySamples.svelte @@ -37,7 +37,7 @@
View samples {#if open} -

Retained minute peaks from the last 24 hours, before the 25% allowance headroom. CPU and memory peaks may come from different heartbeats. Choose the fleet to inspect measurements from other hosts used by fleet allowances.

+

Retained minute peaks from the last 24 hours, used without extra allowance padding. CPU and memory peaks may come from different heartbeats. Choose the fleet to inspect measurements from other hosts used by fleet allowances.

diff --git a/src/routes/hosts/[id]/+page.svelte b/src/routes/hosts/[id]/+page.svelte index 3da231e..cac5ae0 100644 --- a/src/routes/hosts/[id]/+page.svelte +++ b/src/routes/hosts/[id]/+page.svelte @@ -58,7 +58,7 @@ {#if !cleaning}

Automatic host setup

{/if} {#if calibration}

Learned server capacity

-

Allowances update from observed container usage with 25% headroom. Startup and running usage are tracked separately.

+

Allowances update from observed container usage. Hosts reserve 5% CPU and 500 MB of memory. Startup and running usage are tracked separately.

{#each ['startup', 'running'] as phase}{@const allowance = calibration[phase as keyof CapacityCalibration]}
{phase === 'startup' ? 'Starting a server' : 'Running a server'}
{allowance.cpuCores.toFixed(2)} CPU cores · {bytes(allowance.memoryBytes)} · {allowance.source === 'host' ? 'Learned on this host' : allowance.source === 'fleet' ? 'Learned from the fleet' : 'Calibrating; temporary allowance'} ({allowance.samples} samples)
diff --git a/tests/unit/host-capacity-calibration.test.ts b/tests/unit/host-capacity-calibration.test.ts index 74ed571..826deb9 100644 --- a/tests/unit/host-capacity-calibration.test.ts +++ b/tests/unit/host-capacity-calibration.test.ts @@ -108,17 +108,17 @@ describe('measured instance capacity', () => { const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB }); const model = await learned(); - expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.125, memoryBytes: GiB * 0.3125, samples: 20 }); - expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.25, memoryBytes: GiB * 0.46875, samples: 4 }); + expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.1, memoryBytes: GiB / 4, samples: 20 }); + expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.2, memoryBytes: 3 * GiB / 8, samples: 4 }); }); it('raises allowances immediately on a measured spike, including during cold start', async () => { await report(0, { cpuCores: 2, memoryBytes: 2 * GiB }); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); - expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2.5, memoryBytes: 2.5 * GiB }); + expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2, memoryBytes: 2 * GiB }); await learned(); await report(25, { cpuCores: 3, memoryBytes: 3 * GiB }); - expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3.75, memoryBytes: 3.75 * GiB }); - expect(capacityCalibration(hostId).startup.cpuCores).toBe(3.75); + expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3, memoryBytes: 3 * GiB }); + expect(capacityCalibration(hostId).startup.cpuCores).toBe(3); }); it('uses recent fleet evidence on new hosts and survives a database restart', async () => { const model = await learned(); @@ -129,6 +129,19 @@ describe('measured instance capacity', () => { expect(capacityCalibration(other.id).running).toMatchObject({ ...model.running, source: 'fleet' }); expect(capacityCalibration(hostId)).toEqual(model); }); + it('removes legacy cached padding once while preserving host and fleet evidence', async () => { + const model = await learned(); + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + // Recreate the previous schema and its padded allowance values. + db().exec('ALTER TABLE ec_resource_allowances DROP COLUMN usage_margin'); + db().exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores*1.25, memory_bytes=memory_bytes*1.25'); + closeDb(); + const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); + expect(capacityCalibration(hostId)).toEqual(model); + expect(capacityCalibration('new-host').startup).toEqual({ ...model.startup, source: 'fleet' }); + closeDb(); + expect(capacityCalibration(hostId)).toEqual(model); + }); it('prefers mature local measurements when hosts have different CPU footprints', async () => { await learned(); const hosts = await import('../../src/lib/server/hosts/index.js'); @@ -137,9 +150,9 @@ describe('measured instance capacity', () => { await instance(other.id, 'instance-2'); for (let minute = 25; minute < 45; minute++) await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); - expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.125 }); - expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2.5 }); - expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2.5 }); + expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.1 }); + expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2 }); + expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2 }); }); it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => { const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); @@ -183,12 +196,12 @@ describe('measured instance capacity', () => { expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1); expect(capacityCalibration(hostId).running.source).toBe('bootstrap'); }); - it('uses a running percentile and startup peaks with margin, keeping admission reads cached', async () => { + it('uses a running percentile and startup peaks without padding, keeping admission reads cached', async () => { for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 }); const { db } = await import('../../src/lib/server/db/index.js'); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); const prepare = vi.spyOn(db(), 'prepare'); - expect(capacityCalibration(hostId).running.cpuCores).toBe(0.125); + expect(capacityCalibration(hostId).running.cpuCores).toBe(0.1); expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false); prepare.mockRestore(); }); diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index a8a8bd9..4b37326 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -696,6 +696,22 @@ describe('automatic capacity admission and placement', () => { expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]); }); + it.each([ + { resource: 'CPU', measurements: { cpuCount: 4, cpuUsagePercent: 82.5 }, over: { cpuUsagePercent: 82.51 } }, + { resource: 'memory', measurements: { memoryAvailable: 1024 ** 3 + 500_000_000 }, over: { memoryAvailable: 1024 ** 3 + 499_999_999 } } + ])('reserves only the requested host $resource headroom', async ({ measurements, over }) => { + await prepareLibrary(); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js'); + const options = { hostId, libraryId, image }; + heartbeat(hostId, { ...inventory(), ...measurements }); + // Includes the new server's 0.5-core / 1-GiB bootstrap reservation. + expect(choosePlacement(options).host.id).toBe(hostId); + expect(choosePlacement({ ...options, thorough: true }).host.id).toBe(hostId); + heartbeat(hostId, { ...inventory(), ...measurements, ...over }); + expect(() => choosePlacement(options)).toThrow('capacity'); + expect(() => choosePlacement({ ...options, thorough: true })).toThrow('capacity'); + }); it('uses calibrated footprints to admit more instances than the cold-start allowance', async () => { const first = await queue(); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); @@ -786,7 +802,7 @@ describe('automatic capacity admission and placement', () => { await advance(); const instance = nativeInstance(first.name)!; // 0.2 GiB OS + 0.8 GiB container. The remaining 0.2 GiB of its 1 GiB - // reservation plus a new 1 GiB server fit below 2.5 GiB usable memory. + // reservation plus a new 1 GiB server fit below 2.53 GiB usable memory. heartbeat(hostId, { ...host, memoryAvailable: 2 * GiB, instanceResources: [{ instanceId: instance.id, generation: instance.generation, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] }); const second = queueNativeCreation(input); @@ -809,7 +825,7 @@ describe('automatic capacity admission and placement', () => { instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] }); expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict', - message: expect.stringContaining('3.00 GiB projected, 2.50 GiB usable') }); + message: expect.stringContaining('3.00 GiB projected, 2.53 GiB usable') }); }); it('waits when capacity disappears after admission without sending a create command', async () => { await prepareLibrary();