import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; let hostId: string; const epoch = Date.parse('2026-10-04T12:00:00Z'); const GiB = 1024 ** 3; const inventory = { cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB, dockerRunning: true, protocolVersion: 1, directoriesWritable: true, storageAvailable: 100 * GiB, runtimeImages: ['image'], libraries: [{ id: 'library', state: 'ready' }] }; const sample = { instanceId: 'instance-1', generation: 1, uptimeSeconds: 600, cpuCores: 0.1, memoryBytes: GiB / 4 }; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch); const hosts = await import('../../src/lib/server/hosts/index.js'); hostId = hosts.createHost({ name: 'First' }).id; hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory); await instance(hostId, sample.instanceId); }); afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); async function instance(host: string, id: string) { const { db } = await import('../../src/lib/server/db/index.js'); db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'library','image',28960,?)`).run(id, 't5_zombie_theater_' + id, host, new Date().toISOString()); } async function report(minute: number, overrides = {}, host = hostId) { vi.setSystemTime(epoch + minute * 60_000); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(host, { ...inventory, instanceResources: [{ ...sample, ...overrides }] }); } async function learned() { // One real startup followed by twenty running minute peaks. for (let minute = 0; minute < 4; minute++) await report(minute, { uptimeSeconds: minute * 60, cpuCores: 0.2, memoryBytes: 3 * GiB / 8 }); for (let minute = 5; minute < 25; minute++) await report(minute); return (await import('../../src/lib/server/hosts/calibration.js')).capacityCalibration(hostId); } describe('measured instance capacity', () => { it('reports sampling failure without inventing measurements and clears it after recovery', async () => { const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js'); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); const error = 'Container resource sampling failed.'; heartbeat(hostId, { ...inventory, instanceResourcesError: error }); expect(getHost(hostId)?.inventory.instanceResourcesError).toBe(error); expect(capacityCalibration(hostId).running.samples).toBe(0); for (const invalid of [42, {}, 'x'.repeat(501)]) { expect(() => heartbeat(hostId, { ...inventory, instanceResourcesError: invalid })).toThrow('Invalid instance resource sampling error'); } await report(0); expect(getHost(hostId)?.inventory.instanceResourcesError).toBeUndefined(); expect(capacityCalibration(hostId).running.samples).toBe(1); }); it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => { const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB }); const model = await learned(); expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.125, memoryBytes: GiB * 0.3125, samples: 20 }); expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.25, memoryBytes: GiB * 0.46875, samples: 4 }); }); it('raises allowances immediately on a measured spike, including during cold start', async () => { await report(0, { cpuCores: 2, memoryBytes: 2 * GiB }); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2.5, memoryBytes: 2.5 * GiB }); await learned(); await report(25, { cpuCores: 3, memoryBytes: 3 * GiB }); expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3.75, memoryBytes: 3.75 * GiB }); expect(capacityCalibration(hostId).startup.cpuCores).toBe(3.75); }); it('uses recent fleet evidence on new hosts and survives a database restart', async () => { const model = await learned(); const hosts = await import('../../src/lib/server/hosts/index.js'); const other = hosts.createHost({ name: 'Second' }); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(other.id).running).toMatchObject({ ...model.running, source: 'fleet' }); expect(capacityCalibration(hostId)).toEqual(model); }); it('prefers mature local measurements when hosts have different CPU footprints', async () => { await learned(); const hosts = await import('../../src/lib/server/hosts/index.js'); const other = hosts.createHost({ name: 'Slower CPU' }); hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory); await instance(other.id, 'instance-2'); for (let minute = 25; minute < 45; minute++) await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.125 }); expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2.5 }); expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2.5 }); }); it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => { const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); for (let n = 0; n < 40; n++) { vi.setSystemTime(epoch + n * 1000); heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }] }); } const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([{ cpu_cores: 0.4 }]); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', samples: 1 }); }); it('does not train from missing, foreign, retired or old-generation samples', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { heartbeat, createHost } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory); const other = createHost({ name: 'Second' }); await instance(other.id, 'foreign'); await report(0, { instanceId: 'foreign' }); await report(1, { generation: 2 }); db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId); await report(2); expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toEqual([]); }); it('rejects impossible and malformed measurements before changing heartbeat state', async () => { const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js'); for (const overrides of [{ cpuCores: -1 }, { cpuCores: NaN }, { cpuCores: 17 }, { memoryBytes: 0 }, { memoryBytes: 65 * GiB }, { generation: 0 }, { uptimeSeconds: -1 }]) { expect(() => heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, ...overrides }] })).toThrow('Invalid instance resource'); } expect(() => heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] })).toThrow('Invalid instance resource'); expect(getHost(hostId)?.inventory.instanceResources).toBeUndefined(); }); it('expires stale models and prunes old measurements as new samples arrive', async () => { await learned(); vi.setSystemTime(epoch + 26 * 60 * 60_000); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).running.source).toBe('bootstrap'); await report(26 * 60); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1); expect(capacityCalibration(hostId).running.source).toBe('bootstrap'); }); it('uses a running percentile and startup peaks with margin, keeping admission reads cached', async () => { for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 }); const { db } = await import('../../src/lib/server/db/index.js'); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); const prepare = vi.spyOn(db(), 'prepare'); expect(capacityCalibration(hostId).running.cpuCores).toBe(0.125); expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false); prepare.mockRestore(); }); });