196 lines
13 KiB
TypeScript
196 lines
13 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
let hostId: string;
|
|
const epoch = Date.parse('2026-10-04T12:00:00Z');
|
|
const GiB = 1024 ** 3;
|
|
const inventory = { cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB,
|
|
dockerRunning: true, protocolVersion: 1, directoriesWritable: true, storageAvailable: 100 * GiB,
|
|
runtimeImages: ['image'], libraries: [{ id: 'library', state: 'ready' }] };
|
|
const sample = { instanceId: 'instance-1', generation: 1, uptimeSeconds: 600, cpuCores: 0.1, memoryBytes: GiB / 4 };
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch);
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
hostId = hosts.createHost({ name: 'First' }).id;
|
|
hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory);
|
|
await instance(hostId, sample.instanceId);
|
|
});
|
|
afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); });
|
|
async function instance(host: string, id: string) {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
|
VALUES(?,?,?,'library','image',28960,?)`).run(id, 't5_zombie_theater_' + id, host, new Date().toISOString());
|
|
}
|
|
async function report(minute: number, overrides = {}, host = hostId) {
|
|
vi.setSystemTime(epoch + minute * 60_000);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(host, { ...inventory, instanceResources: [{ ...sample, ...overrides }] });
|
|
}
|
|
async function learned() {
|
|
// One real startup followed by twenty running minute peaks.
|
|
for (let minute = 0; minute < 4; minute++) await report(minute, { uptimeSeconds: minute * 60, cpuCores: 0.2, memoryBytes: 3 * GiB / 8 });
|
|
for (let minute = 5; minute < 25; minute++) await report(minute);
|
|
return (await import('../../src/lib/server/hosts/calibration.js')).capacityCalibration(hostId);
|
|
}
|
|
|
|
describe('measured instance capacity', () => {
|
|
it('lists retained samples with phase filters, host scope, stable ordering, and bounded pages', async () => {
|
|
for (let minute = 0; minute < 101; minute++) await report(minute, { uptimeSeconds: minute < 5 ? minute * 60 : 600 });
|
|
const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js');
|
|
const first = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 });
|
|
const second = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 2 });
|
|
expect(first).toMatchObject({ total: 101, page: 1, pageSize: 50 });
|
|
expect(first.samples).toHaveLength(50);
|
|
expect(first.samples[0]).toMatchObject({ hostId, hostName: 'First', instanceId: sample.instanceId,
|
|
server: 't5_zombie_theater_instance-1', generation: 1, phase: 'running',
|
|
cpuCores: 0.1, memoryBytes: GiB / 4, timestamp: new Date(epoch + 100 * 60_000).toISOString() });
|
|
expect(new Set([...first.samples, ...second.samples].map(row => row.timestamp)).size).toBe(100);
|
|
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 999 })).toMatchObject({ page: 3, samples: [expect.any(Object)] });
|
|
const startup = capacitySamples(hostId, { scope: 'host', phase: 'startup', page: 1 });
|
|
expect(startup.total).toBe(5);
|
|
expect(startup.samples.every(row => row.phase === 'startup')).toBe(true);
|
|
expect(capacitySamples(hostId, { scope: 'host', phase: 'running', page: 1 }).total).toBe(96);
|
|
vi.setSystemTime(epoch + 26 * 60 * 60_000);
|
|
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 })).toMatchObject({ total: 0, samples: [] });
|
|
});
|
|
|
|
it('supports fleet samples and retains instance identity after retirement', async () => {
|
|
await report(0);
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const other = hosts.createHost({ name: 'Second' });
|
|
hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
|
|
await instance(other.id, 'instance-2');
|
|
await report(1, { instanceId: 'instance-2' }, other.id);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId);
|
|
const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }).samples.map(row => row.instanceId)).toEqual([sample.instanceId]);
|
|
const fleet = capacitySamples(hostId, { scope: 'fleet', phase: 'all', page: 1 });
|
|
expect(fleet.total).toBe(2);
|
|
expect(fleet.samples.map(row => row.hostName)).toEqual(['Second', 'First']);
|
|
});
|
|
|
|
it('validates the samples API and rejects agent credentials and cross-origin reads', async () => {
|
|
await report(0);
|
|
const { GET } = await import('../../src/routes/api/hosts/[id]/samples/+server.js');
|
|
const request = (query = '', headers = {}, id = hostId) => GET({ request: new Request(`https://ec.test/api/hosts/${id}/samples${query}`, { headers }), params: { id } });
|
|
const response = request();
|
|
expect(response.headers.get('cache-control')).toBe('no-store');
|
|
expect(await response.json()).toMatchObject({ total: 1, samples: [expect.objectContaining({ cpuCores: 0.1 })] });
|
|
for (const query of ['?scope=wrong', '?phase=wrong', '?page=-1', '?page=1.5', '?page=NaN', '?page=Infinity']) {
|
|
expect(() => request(query)).toThrow(expect.objectContaining({ status: 400 }));
|
|
}
|
|
expect(() => request('', {}, 'missing')).toThrow(expect.objectContaining({ status: 404 }));
|
|
expect(() => request('', { origin: 'https://other.test' })).toThrow(expect.objectContaining({ status: 403 }));
|
|
expect(() => request('', { authorization: 'Bearer ec_host_test' })).toThrow(expect.objectContaining({ status: 403 }));
|
|
});
|
|
|
|
it('reports sampling failure without inventing measurements and clears it after recovery', async () => {
|
|
const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js');
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
const error = 'Container resource sampling failed.';
|
|
heartbeat(hostId, { ...inventory, instanceResourcesError: error });
|
|
expect(getHost(hostId)?.inventory.instanceResourcesError).toBe(error);
|
|
expect(capacityCalibration(hostId).running.samples).toBe(0);
|
|
for (const invalid of [42, {}, 'x'.repeat(501)]) {
|
|
expect(() => heartbeat(hostId, { ...inventory, instanceResourcesError: invalid })).toThrow('Invalid instance resource sampling error');
|
|
}
|
|
await report(0);
|
|
expect(getHost(hostId)?.inventory.instanceResourcesError).toBeUndefined();
|
|
expect(capacityCalibration(hostId).running.samples).toBe(1);
|
|
});
|
|
|
|
it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => {
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB });
|
|
const model = await learned();
|
|
expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.125, memoryBytes: GiB * 0.3125, samples: 20 });
|
|
expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.25, memoryBytes: GiB * 0.46875, samples: 4 });
|
|
});
|
|
it('raises allowances immediately on a measured spike, including during cold start', async () => {
|
|
await report(0, { cpuCores: 2, memoryBytes: 2 * GiB });
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2.5, memoryBytes: 2.5 * GiB });
|
|
await learned();
|
|
await report(25, { cpuCores: 3, memoryBytes: 3 * GiB });
|
|
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3.75, memoryBytes: 3.75 * GiB });
|
|
expect(capacityCalibration(hostId).startup.cpuCores).toBe(3.75);
|
|
});
|
|
it('uses recent fleet evidence on new hosts and survives a database restart', async () => {
|
|
const model = await learned();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const other = hosts.createHost({ name: 'Second' });
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(other.id).running).toMatchObject({ ...model.running, source: 'fleet' });
|
|
expect(capacityCalibration(hostId)).toEqual(model);
|
|
});
|
|
it('prefers mature local measurements when hosts have different CPU footprints', async () => {
|
|
await learned();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const other = hosts.createHost({ name: 'Slower CPU' });
|
|
hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
|
|
await instance(other.id, 'instance-2');
|
|
for (let minute = 25; minute < 45; minute++) await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id);
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.125 });
|
|
expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2.5 });
|
|
expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2.5 });
|
|
});
|
|
it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => {
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
for (let n = 0; n < 40; n++) {
|
|
vi.setSystemTime(epoch + n * 1000);
|
|
heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }] });
|
|
}
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([{ cpu_cores: 0.4 }]);
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', samples: 1 });
|
|
});
|
|
it('does not train from missing, foreign, retired or old-generation samples', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { heartbeat, createHost } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, inventory);
|
|
const other = createHost({ name: 'Second' });
|
|
await instance(other.id, 'foreign');
|
|
await report(0, { instanceId: 'foreign' });
|
|
await report(1, { generation: 2 });
|
|
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId);
|
|
await report(2);
|
|
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toEqual([]);
|
|
});
|
|
it('rejects impossible and malformed measurements before changing heartbeat state', async () => {
|
|
const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js');
|
|
for (const overrides of [{ cpuCores: -1 }, { cpuCores: NaN }, { cpuCores: 17 }, { memoryBytes: 0 },
|
|
{ memoryBytes: 65 * GiB }, { generation: 0 }, { uptimeSeconds: -1 }]) {
|
|
expect(() => heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, ...overrides }] })).toThrow('Invalid instance resource');
|
|
}
|
|
expect(() => heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] })).toThrow('Invalid instance resource');
|
|
expect(getHost(hostId)?.inventory.instanceResources).toBeUndefined();
|
|
});
|
|
it('expires stale models and prunes old measurements as new samples arrive', async () => {
|
|
await learned();
|
|
vi.setSystemTime(epoch + 26 * 60 * 60_000);
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
|
|
await report(26 * 60);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1);
|
|
expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
|
|
});
|
|
it('uses a running percentile and startup peaks with margin, keeping admission reads cached', async () => {
|
|
for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
const prepare = vi.spyOn(db(), 'prepare');
|
|
expect(capacityCalibration(hostId).running.cpuCores).toBe(0.125);
|
|
expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false);
|
|
prepare.mockRestore();
|
|
});
|
|
});
|