Automatically place servers using learned resource capacity

2026-10-04 21:04:09 -04:00
parent a1b957392f
commit e6cdf9eb7f
25 changed files with 927 additions and 85 deletions
@@ -0,0 +1,129 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+let fixture: Fixture;
+let hostId: string;
+const epoch = Date.parse('2026-10-04T12:00:00Z');
+const GiB = 1024 ** 3;
+const inventory = { cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB,
+    dockerRunning: true, protocolVersion: 1, directoriesWritable: true, storageAvailable: 100 * GiB,
+    runtimeImages: ['image'], libraries: [{ id: 'library', state: 'ready' }] };
+const sample = { instanceId: 'instance-1', generation: 1, uptimeSeconds: 600, cpuCores: 0.1, memoryBytes: GiB / 4 };
+
+beforeEach(async () => {
+    vi.resetModules();
+    fixture = await newFixture([]);
+    vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch);
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    hostId = hosts.createHost({ name: 'First' }).id;
+    hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory);
+    await instance(hostId, sample.instanceId);
+});
+afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); });
+async function instance(host: string, id: string) {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
+        VALUES(?,?,?,'library','image',28960,?)`).run(id, 't5_zombie_theater_' + id, host, new Date().toISOString());
+}
+async function report(minute: number, overrides = {}, host = hostId) {
+    vi.setSystemTime(epoch + minute * 60_000);
+    const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+    heartbeat(host, { ...inventory, instanceResources: [{ ...sample, ...overrides }] });
+}
+async function learned() {
+    // One real startup followed by twenty running minute peaks.
+    for (let minute = 0; minute < 4; minute++) await report(minute, { uptimeSeconds: minute * 60, cpuCores: 0.2, memoryBytes: 3 * GiB / 8 });
+    for (let minute = 5; minute < 25; minute++) await report(minute);
+    return (await import('../../src/lib/server/hosts/calibration.js')).capacityCalibration(hostId);
+}
+
+describe('measured instance capacity', () => {
+    it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => {
+        const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
+        expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB });
+        const model = await learned();
+        expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.125, memoryBytes: GiB * 0.3125, samples: 20 });
+        expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.25, memoryBytes: GiB * 0.46875, samples: 4 });
+    });
+    it('raises allowances immediately on a measured spike, including during cold start', async () => {
+        await report(0, { cpuCores: 2, memoryBytes: 2 * GiB });
+        const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
+        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2.5, memoryBytes: 2.5 * GiB });
+        await learned();
+        await report(25, { cpuCores: 3, memoryBytes: 3 * GiB });
+        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3.75, memoryBytes: 3.75 * GiB });
+        expect(capacityCalibration(hostId).startup.cpuCores).toBe(3.75);
+    });
+    it('uses recent fleet evidence on new hosts and survives a database restart', async () => {
+        const model = await learned();
+        const hosts = await import('../../src/lib/server/hosts/index.js');
+        const other = hosts.createHost({ name: 'Second' });
+        const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
+        const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
+        expect(capacityCalibration(other.id).running).toMatchObject({ ...model.running, source: 'fleet' });
+        expect(capacityCalibration(hostId)).toEqual(model);
+    });
+    it('prefers mature local measurements when hosts have different CPU footprints', async () => {
+        await learned();
+        const hosts = await import('../../src/lib/server/hosts/index.js');
+        const other = hosts.createHost({ name: 'Slower CPU' });
+        hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
+        await instance(other.id, 'instance-2');
+        for (let minute = 25; minute < 45; minute++) await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id);
+        const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
+        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.125 });
+        expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2.5 });
+        expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2.5 });
+    });
+    it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => {
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        for (let n = 0; n < 40; n++) {
+            vi.setSystemTime(epoch + n * 1000);
+            heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }] });
+        }
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([{ cpu_cores: 0.4 }]);
+        const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
+        expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', samples: 1 });
+    });
+    it('does not train from missing, foreign, retired or old-generation samples', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { heartbeat, createHost } = await import('../../src/lib/server/hosts/index.js');
+        heartbeat(hostId, inventory);
+        const other = createHost({ name: 'Second' });
+        await instance(other.id, 'foreign');
+        await report(0, { instanceId: 'foreign' });
+        await report(1, { generation: 2 });
+        db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId);
+        await report(2);
+        expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toEqual([]);
+    });
+    it('rejects impossible and malformed measurements before changing heartbeat state', async () => {
+        const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js');
+        for (const overrides of [{ cpuCores: -1 }, { cpuCores: NaN }, { cpuCores: 17 }, { memoryBytes: 0 },
+            { memoryBytes: 65 * GiB }, { generation: 0 }, { uptimeSeconds: -1 }]) {
+            expect(() => heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, ...overrides }] })).toThrow('Invalid instance resource');
+        }
+        expect(() => heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] })).toThrow('Invalid instance resource');
+        expect(getHost(hostId)?.inventory.instanceResources).toBeUndefined();
+    });
+    it('expires stale models and prunes old measurements as new samples arrive', async () => {
+        await learned();
+        vi.setSystemTime(epoch + 26 * 60 * 60_000);
+        const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
+        expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
+        await report(26 * 60);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1);
+        expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
+    });
+    it('uses a running percentile and startup peaks with margin, keeping admission reads cached', async () => {
+        for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 });
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
+        const prepare = vi.spyOn(db(), 'prepare');
+        expect(capacityCalibration(hostId).running.cpuCores).toBe(0.125);
+        expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false);
+        prepare.mockRestore();
+    });
+});