import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; let hostId: string; const epoch = Date.parse('2026-10-04T12:00:00Z'); const GiB = 1024 ** 3; const inventory = { cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB, dockerRunning: true, protocolVersion: 1, directoriesWritable: true, storageAvailable: 100 * GiB, preparedRuntimeImages: ['image'], libraries: [{ id: 'library', state: 'ready' }] }; const sample = { instanceId: 'instance-1', generation: 1, uptimeSeconds: 600, cpuCores: 0.1, memoryBytes: GiB / 4 }; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch); const hosts = await import('../../src/lib/server/hosts/index.js'); hostId = hosts.createHost({ name: 'First' }).id; hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory); await instance(hostId, sample.instanceId); }); afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); async function instance(host: string, id: string) { const { db } = await import('../../src/lib/server/db/index.js'); db() .prepare( `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'library','image',28960,?)` ) .run(id, 't5_zombie_theater_' + id, host, new Date().toISOString()); } async function report(minute: number, overrides = {}, host = hostId) { vi.setSystemTime(epoch + minute * 60_000); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(host, { ...inventory, instanceResources: [{ ...sample, ...overrides }] }); } async function learned() { // One real startup followed by twenty running minute peaks. for (let minute = 0; minute < 4; minute++) await report(minute, { uptimeSeconds: minute * 60, cpuCores: 0.2, memoryBytes: (3 * GiB) / 8 }); for (let minute = 5; minute < 25; minute++) await report(minute); return (await import('../../src/lib/server/hosts/calibration.js')).capacityCalibration(hostId); } describe('measured instance capacity', () => { it('lists retained samples with phase filters, host scope, stable ordering, and bounded pages', async () => { for (let minute = 0; minute < 101; minute++) await report(minute, { uptimeSeconds: minute < 5 ? minute * 60 : 600 }); const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js'); const first = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }); const second = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 2 }); expect(first).toMatchObject({ total: 101, page: 1, pageSize: 50 }); expect(first.samples).toHaveLength(50); expect(first.samples[0]).toMatchObject({ hostId, hostName: 'First', instanceId: sample.instanceId, server: 't5_zombie_theater_instance-1', generation: 1, phase: 'running', cpuCores: 0.1, memoryBytes: GiB / 4, timestamp: new Date(epoch + 100 * 60_000).toISOString() }); expect( new Set([...first.samples, ...second.samples].map((row) => row.timestamp)).size ).toBe(100); expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 999 })).toMatchObject({ page: 3, samples: [expect.any(Object)] }); const startup = capacitySamples(hostId, { scope: 'host', phase: 'startup', page: 1 }); expect(startup.total).toBe(5); expect(startup.samples.every((row) => row.phase === 'startup')).toBe(true); expect(capacitySamples(hostId, { scope: 'host', phase: 'running', page: 1 }).total).toBe( 96 ); vi.setSystemTime(epoch + 26 * 60 * 60_000); expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 })).toMatchObject({ total: 0, samples: [] }); }); it('supports fleet samples and retains instance identity after retirement', async () => { await report(0); const hosts = await import('../../src/lib/server/hosts/index.js'); const other = hosts.createHost({ name: 'Second' }); hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory); await instance(other.id, 'instance-2'); await report(1, { instanceId: 'instance-2' }, other.id); const { db } = await import('../../src/lib/server/db/index.js'); db() .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') .run(new Date().toISOString(), sample.instanceId); const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js'); expect( capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }).samples.map( (row) => row.instanceId ) ).toEqual([sample.instanceId]); const fleet = capacitySamples(hostId, { scope: 'fleet', phase: 'all', page: 1 }); expect(fleet.total).toBe(2); expect(fleet.samples.map((row) => row.hostName)).toEqual(['Second', 'First']); }); it('validates the samples API and rejects agent credentials and cross-origin reads', async () => { await report(0); const { GET } = await import('../../src/routes/api/hosts/[id]/samples/+server.js'); const request = (query = '', headers = {}, id = hostId) => GET({ request: new Request(`https://ec.test/api/hosts/${id}/samples${query}`, { headers }), params: { id } }); const response = request(); expect(response.headers.get('cache-control')).toBe('no-store'); expect(await response.json()).toMatchObject({ total: 1, samples: [expect.objectContaining({ cpuCores: 0.1 })] }); for (const query of [ '?scope=wrong', '?phase=wrong', '?page=-1', '?page=1.5', '?page=NaN', '?page=Infinity' ]) { expect(() => request(query)).toThrow(expect.objectContaining({ status: 400 })); } expect(() => request('', {}, 'missing')).toThrow(expect.objectContaining({ status: 404 })); expect(() => request('', { origin: 'https://other.test' })).toThrow( expect.objectContaining({ status: 403 }) ); expect(() => request('', { authorization: 'Bearer ec_host_test' })).toThrow( expect.objectContaining({ status: 403 }) ); }); it('reports sampling failure without inventing measurements and clears it after recovery', async () => { const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js'); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); const error = 'Container resource sampling failed.'; heartbeat(hostId, { ...inventory, instanceResourcesError: error }); expect(getHost(hostId)?.inventory.instanceResourcesError).toBe(error); expect(capacityCalibration(hostId).running.samples).toBe(0); for (const invalid of [42, {}, 'x'.repeat(501)]) { expect(() => heartbeat(hostId, { ...inventory, instanceResourcesError: invalid }) ).toThrow('Invalid instance resource sampling error'); } await report(0); expect(getHost(hostId)?.inventory.instanceResourcesError).toBeUndefined(); expect(capacityCalibration(hostId).running.samples).toBe(1); }); it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => { const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB }); const model = await learned(); expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.1, memoryBytes: GiB / 4, samples: 20 }); expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.2, memoryBytes: (3 * GiB) / 8, samples: 4 }); }); it('raises allowances immediately on a measured spike, including during cold start', async () => { await report(0, { cpuCores: 2, memoryBytes: 2 * GiB }); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2, memoryBytes: 2 * GiB }); await learned(); await report(25, { cpuCores: 3, memoryBytes: 3 * GiB }); expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3, memoryBytes: 3 * GiB }); expect(capacityCalibration(hostId).startup.cpuCores).toBe(3); }); it('uses recent fleet evidence on new hosts and survives a database restart', async () => { const model = await learned(); const hosts = await import('../../src/lib/server/hosts/index.js'); const other = hosts.createHost({ name: 'Second' }); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(other.id).running).toMatchObject({ ...model.running, source: 'fleet' }); expect(capacityCalibration(hostId)).toEqual(model); }); it('removes legacy cached padding once while preserving host and fleet evidence', async () => { const model = await learned(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); // Recreate the previous schema and its padded allowance values. db().exec('ALTER TABLE ec_resource_allowances DROP COLUMN usage_margin'); db().exec( 'UPDATE ec_resource_allowances SET cpu_cores=cpu_cores*1.25, memory_bytes=memory_bytes*1.25' ); db().pragma('user_version = 0'); closeDb(); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId)).toEqual(model); expect(capacityCalibration('new-host').startup).toEqual({ ...model.startup, source: 'fleet' }); closeDb(); expect(capacityCalibration(hostId)).toEqual(model); }); it('prefers mature local measurements when hosts have different CPU footprints', async () => { await learned(); const hosts = await import('../../src/lib/server/hosts/index.js'); const other = hosts.createHost({ name: 'Slower CPU' }); hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory); await instance(other.id, 'instance-2'); for (let minute = 25; minute < 45; minute++) await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.1 }); expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2 }); expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2 }); }); it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => { const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); for (let n = 0; n < 40; n++) { vi.setSystemTime(epoch + n * 1000); heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }] }); } const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([ { cpu_cores: 0.4 } ]); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', samples: 1 }); }); it('does not train from missing, foreign, retired or old-generation samples', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { heartbeat, createHost } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory); const other = createHost({ name: 'Second' }); await instance(other.id, 'foreign'); await report(0, { instanceId: 'foreign' }); await report(1, { generation: 2 }); db() .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') .run(new Date().toISOString(), sample.instanceId); await report(2); expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toEqual([]); }); it('rejects impossible and malformed measurements before changing heartbeat state', async () => { const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js'); for (const overrides of [ { cpuCores: -1 }, { cpuCores: NaN }, { cpuCores: 17 }, { memoryBytes: 0 }, { memoryBytes: 65 * GiB }, { generation: 0 }, { uptimeSeconds: -1 } ]) { expect(() => heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, ...overrides }] }) ).toThrow('Invalid instance resource'); } expect(() => heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] }) ).toThrow('Invalid instance resource'); expect(getHost(hostId)?.inventory.instanceResources).toBeUndefined(); }); it('expires stale models and prunes old measurements as new samples arrive', async () => { await learned(); vi.setSystemTime(epoch + 26 * 60 * 60_000); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).running.source).toBe('bootstrap'); await report(26 * 60); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1); expect(capacityCalibration(hostId).running.source).toBe('bootstrap'); }); it.each(['startup', 'running'] as const)( 'uses the %s 95th percentile for CPU and memory, keeping admission reads cached', async (phase) => { await learned(); // Repeated starts or running observations include isolated CPU and // memory outliers in different minutes; neither should set the allowance. const uptimeSeconds = phase === 'startup' ? 60 : 600; for (let minute = 25; minute < 125; minute++) await report(minute, { uptimeSeconds, cpuCores: minute === 25 ? 2 : 0.4, memoryBytes: minute === 26 ? 3 * GiB : GiB / 2 }); const { db } = await import('../../src/lib/server/db/index.js'); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); const prepare = vi.spyOn(db(), 'prepare'); expect(capacityCalibration(hostId)[phase]).toMatchObject({ source: 'host', cpuCores: 0.4, memoryBytes: GiB / 2 }); expect(capacityCalibration('new-host')[phase]).toMatchObject({ source: 'fleet', cpuCores: 0.4, memoryBytes: GiB / 2 }); expect( prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples')) ).toBe(false); prepare.mockRestore(); // Both phases still raise immediately for fresh spikes, hold that value // within the minute, then return to P95 on the next recalculation. await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB }); expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB }); await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 }); expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB }); await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 }); expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 0.4, memoryBytes: GiB / 2 }); } ); });