Reduce host capacity headroom and remove learned usage padding

docs/native-hosts.md

@@ -157,7 +157,7 @@ startup samples require observing the first five minutes of a container's run.
 The controller retains per-instance minute peaks for 24 hours and caches learned
 CPU/memory allowances. The first five minutes after a container starts count as
 startup; later samples train the running allowance. Startup uses the observed
-maximum and running uses the 95th percentile of minute peaks, both with **25%
+maximum and running uses the 95th percentile of minute peaks, with **no per-server
 headroom**. Each calculation considers up to the latest 10,000 minute peaks.
 Host-local evidence is preferred to account for different hardware; new hosts
 can borrow the fleet's learned allowance. Startup allowances are never lower
@@ -165,7 +165,7 @@ than running allowances. The host page shows both allowances and their source.
 Expand **View samples** in Learned server capacity to inspect retained minute
 peaks for the last 24 hours. Filter by startup/running phase or switch from this
 host to the entire fleet. Rows show time, server and generation, CPU cores, and
-memory in MiB before headroom; CPU and memory peaks may come from different
+memory in MiB; CPU and memory peaks may come from different
 heartbeats. Results are paginated, and **Refresh samples** loads the latest stored
 measurements without contacting agents. The allowance count may lag the table
 until the next calculation or reflect a different source.
@@ -179,7 +179,7 @@ Models expire after 24 hours without measurements. Recalculation happens during
 heartbeat ingestion, at most once per minute per host/fleet phase; new peaks
 raise the cache immediately. Launch admission reads only the cached model.
 
-Scheduling still leaves **10% host CPU and 512 MiB RAM** for the OS and bursts.
+Scheduling still leaves **5% host CPU and 500 MB RAM (500,000,000 bytes)** for the OS and bursts.
 These allowances are scheduling reservations, not Docker limits. Queued and
 unresolved creations consume the startup allowance; established instances keep
 the running allowance even while idle or stopped. Current container measurements

src/lib/server/hosts/calibration.ts

@@ -4,7 +4,6 @@ import type { CapacityCalibration, CapacitySample, CapacitySamplePage, HostInven
 
 const BOOTSTRAP_CPU = 0.5;
 const BOOTSTRAP_MEMORY = 1024 ** 3;
-const MARGIN = 1.25;
 const WINDOW_MS = 24 * 60 * 60_000;
 const MAX_SAMPLES = 10_000;
 type Phase = 'startup' | 'running';
@@ -83,8 +82,8 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory
                 .all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
                 { cpu_cores: number; memory_bytes: number; bucket: string }[];
             const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
-            const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu) * MARGIN;
+            const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu);
-            const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory) * MARGIN);
+            const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory));
             handle.prepare(`INSERT INTO ec_resource_allowances
                 (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET
                 cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples,
@@ -96,7 +95,7 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory
             // Raise the allowance immediately on a new peak; a lower sample
             // cannot reduce it until the next scheduled percentile calculation.
             handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`)
-                .run(peak.cpu * MARGIN, Math.ceil(peak.memory * MARGIN), timestamp, scope, phase);
+                .run(peak.cpu, Math.ceil(peak.memory), timestamp, scope, phase);
         }
     }
 }

src/lib/server/hosts/capacity.ts

@@ -6,7 +6,8 @@ import { capacityCalibration } from './calibration.js';
 
 // Storage and OS headroom are separate from learned instance CPU/memory usage.
 const INSTANCE_STORAGE_BYTES = 1024 ** 3;
-const MEMORY_HEADROOM = 512 * 1024 ** 2;
+const MEMORY_HEADROOM = 500_000_000;
+const CPU_USABLE_FRACTION = 0.95;
 export class CapacityError extends Error {}
 export interface PlacementOptions {
     hostId?: string; libraryId: string; image: string; bindAddress?: string;
@@ -83,7 +84,7 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
     const memory = Math.max(usedMemory + extraMemory, reservedMemory);
     const usableMemory = memoryTotal - MEMORY_HEADROOM;
     const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
-    if (cores > cpuCount * 0.9) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * 0.9).toFixed(2)} usable; ${reservations} reservations including this launch.`);
+    if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`);
     if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. `
         + `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(startup.memoryBytes)} startup allowance).`);
     // Recovered resources must not remain unavailable until the historical

src/lib/server/hosts/schema.ts

@@ -57,6 +57,7 @@ export function ensureHostSchema(handle: Database.Database): void {
         CREATE TABLE IF NOT EXISTS ec_resource_allowances (
             scope TEXT NOT NULL, phase TEXT NOT NULL, cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
             samples INTEGER NOT NULL, window_start TEXT NOT NULL, observed_at TEXT NOT NULL, computed_at TEXT NOT NULL,
+            usage_margin REAL NOT NULL DEFAULT 1,
             PRIMARY KEY(scope,phase)
         );
         CREATE TABLE IF NOT EXISTS ec_host_decommissions (
@@ -71,5 +72,14 @@ export function ensureHostSchema(handle: Database.Database): void {
     if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
     const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
     if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
+    const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[];
+    if (!allowanceColumns.some(column => column.name === 'usage_margin')) {
+        // Existing caches include 25% padding. Preserve learned evidence while
+        // removing that padding exactly once, including dormant startup models.
+        handle.transaction(() => {
+            handle.exec('ALTER TABLE ec_resource_allowances ADD COLUMN usage_margin REAL NOT NULL DEFAULT 1');
+            handle.exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)');
+        })();
+    }
     initialized.add(handle);
 }

src/lib/ui/components/CapacitySamples.svelte

@@ -37,7 +37,7 @@
 <details bind:open>
     <summary>View samples</summary>
     {#if open}
-        <p>Retained minute peaks from the last 24 hours, before the 25% allowance headroom. CPU and memory peaks may come from different heartbeats. Choose the fleet to inspect measurements from other hosts used by fleet allowances.</p>
+        <p>Retained minute peaks from the last 24 hours, used without extra allowance padding. CPU and memory peaks may come from different heartbeats. Choose the fleet to inspect measurements from other hosts used by fleet allowances.</p>
         <div class="controls">
             <label>Source<select bind:value={scope} onchange={() => page = 1}><option value="host">This host</option><option value="fleet">Entire fleet</option></select></label>
             <label>Phase<select bind:value={phase} onchange={() => page = 1}><option value="all">All phases</option><option value="startup">Starting</option><option value="running">Running</option></select></label>

src/routes/hosts/[id]/+page.svelte

@@ -58,7 +58,7 @@
         </section>
         {#if !cleaning}<section class="panel"><h2>Automatic host setup</h2><HostPreparation {host} /></section>{/if}
         {#if calibration}<section class="panel"><h2>Learned server capacity</h2>
-            <p>Allowances update from observed container usage with 25% headroom. Startup and running usage are tracked separately.</p>
+            <p>Allowances update from observed container usage. Hosts reserve 5% CPU and 500 MB of memory. Startup and running usage are tracked separately.</p>
             <dl>{#each ['startup', 'running'] as phase}{@const allowance = calibration[phase as keyof CapacityCalibration]}
                 <dt>{phase === 'startup' ? 'Starting a server' : 'Running a server'}</dt>
                 <dd>{allowance.cpuCores.toFixed(2)} CPU cores · {bytes(allowance.memoryBytes)} · {allowance.source === 'host' ? 'Learned on this host' : allowance.source === 'fleet' ? 'Learned from the fleet' : 'Calibrating; temporary allowance'} ({allowance.samples} samples)</dd>

tests/unit/host-capacity-calibration.test.ts

@@ -108,17 +108,17 @@ describe('measured instance capacity', () => {
         const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
         expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB });
         const model = await learned();
-        expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.125, memoryBytes: GiB * 0.3125, samples: 20 });
+        expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.1, memoryBytes: GiB / 4, samples: 20 });
-        expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.25, memoryBytes: GiB * 0.46875, samples: 4 });
+        expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.2, memoryBytes: 3 * GiB / 8, samples: 4 });
     });
     it('raises allowances immediately on a measured spike, including during cold start', async () => {
         await report(0, { cpuCores: 2, memoryBytes: 2 * GiB });
         const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
-        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2.5, memoryBytes: 2.5 * GiB });
+        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2, memoryBytes: 2 * GiB });
         await learned();
         await report(25, { cpuCores: 3, memoryBytes: 3 * GiB });
-        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3.75, memoryBytes: 3.75 * GiB });
+        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3, memoryBytes: 3 * GiB });
-        expect(capacityCalibration(hostId).startup.cpuCores).toBe(3.75);
+        expect(capacityCalibration(hostId).startup.cpuCores).toBe(3);
     });
     it('uses recent fleet evidence on new hosts and survives a database restart', async () => {
         const model = await learned();
@@ -129,6 +129,19 @@ describe('measured instance capacity', () => {
         expect(capacityCalibration(other.id).running).toMatchObject({ ...model.running, source: 'fleet' });
         expect(capacityCalibration(hostId)).toEqual(model);
     });
+    it('removes legacy cached padding once while preserving host and fleet evidence', async () => {
+        const model = await learned();
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        // Recreate the previous schema and its padded allowance values.
+        db().exec('ALTER TABLE ec_resource_allowances DROP COLUMN usage_margin');
+        db().exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores*1.25, memory_bytes=memory_bytes*1.25');
+        closeDb();
+        const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
+        expect(capacityCalibration(hostId)).toEqual(model);
+        expect(capacityCalibration('new-host').startup).toEqual({ ...model.startup, source: 'fleet' });
+        closeDb();
+        expect(capacityCalibration(hostId)).toEqual(model);
+    });
     it('prefers mature local measurements when hosts have different CPU footprints', async () => {
         await learned();
         const hosts = await import('../../src/lib/server/hosts/index.js');
@@ -137,9 +150,9 @@ describe('measured instance capacity', () => {
         await instance(other.id, 'instance-2');
         for (let minute = 25; minute < 45; minute++) await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id);
         const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
-        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.125 });
+        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.1 });
-        expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2.5 });
+        expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2 });
-        expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2.5 });
+        expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2 });
     });
     it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => {
         const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
@@ -183,12 +196,12 @@ describe('measured instance capacity', () => {
         expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1);
         expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
     });
-    it('uses a running percentile and startup peaks with margin, keeping admission reads cached', async () => {
+    it('uses a running percentile and startup peaks without padding, keeping admission reads cached', async () => {
         for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 });
         const { db } = await import('../../src/lib/server/db/index.js');
         const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
         const prepare = vi.spyOn(db(), 'prepare');
-        expect(capacityCalibration(hostId).running.cpuCores).toBe(0.125);
+        expect(capacityCalibration(hostId).running.cpuCores).toBe(0.1);
         expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false);
         prepare.mockRestore();
     });

tests/unit/native-provisioning.test.ts

@@ -696,6 +696,22 @@ describe('automatic capacity admission and placement', () => {
         expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
         expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
     });
+    it.each([
+        { resource: 'CPU', measurements: { cpuCount: 4, cpuUsagePercent: 82.5 }, over: { cpuUsagePercent: 82.51 } },
+        { resource: 'memory', measurements: { memoryAvailable: 1024 ** 3 + 500_000_000 }, over: { memoryAvailable: 1024 ** 3 + 499_999_999 } }
+    ])('reserves only the requested host $resource headroom', async ({ measurements, over }) => {
+        await prepareLibrary();
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
+        const options = { hostId, libraryId, image };
+        heartbeat(hostId, { ...inventory(), ...measurements });
+        // Includes the new server's 0.5-core / 1-GiB bootstrap reservation.
+        expect(choosePlacement(options).host.id).toBe(hostId);
+        expect(choosePlacement({ ...options, thorough: true }).host.id).toBe(hostId);
+        heartbeat(hostId, { ...inventory(), ...measurements, ...over });
+        expect(() => choosePlacement(options)).toThrow('capacity');
+        expect(() => choosePlacement({ ...options, thorough: true })).toThrow('capacity');
+    });
     it('uses calibrated footprints to admit more instances than the cold-start allowance', async () => {
         const first = await queue();
         const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
@@ -786,7 +802,7 @@ describe('automatic capacity admission and placement', () => {
         await advance();
         const instance = nativeInstance(first.name)!;
         // 0.2 GiB OS + 0.8 GiB container. The remaining 0.2 GiB of its 1 GiB
-        // reservation plus a new 1 GiB server fit below 2.5 GiB usable memory.
+        // reservation plus a new 1 GiB server fit below 2.53 GiB usable memory.
         heartbeat(hostId, { ...host, memoryAvailable: 2 * GiB, instanceResources: [{ instanceId: instance.id,
             generation: instance.generation, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
         const second = queueNativeCreation(input);
@@ -809,7 +825,7 @@ describe('automatic capacity admission and placement', () => {
             instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1,
                 uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
         expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict',
-            message: expect.stringContaining('3.00 GiB projected, 2.50 GiB usable') });
+            message: expect.stringContaining('3.00 GiB projected, 2.53 GiB usable') });
     });
     it('waits when capacity disappears after admission without sending a create command', async () => {
         await prepareLibrary();