392 lines
17 KiB
TypeScript
392 lines
17 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
let hostId: string;
|
|
const epoch = Date.parse('2026-10-04T12:00:00Z');
|
|
const GiB = 1024 ** 3;
|
|
const inventory = {
|
|
cpuCount: 16,
|
|
cpuUsagePercent: 10,
|
|
memoryTotal: 64 * GiB,
|
|
memoryAvailable: 60 * GiB,
|
|
dockerRunning: true,
|
|
protocolVersion: 1,
|
|
directoriesWritable: true,
|
|
storageAvailable: 100 * GiB,
|
|
preparedRuntimeImages: ['image'],
|
|
libraries: [{ id: 'library', state: 'ready' }]
|
|
};
|
|
const sample = {
|
|
instanceId: 'instance-1',
|
|
generation: 1,
|
|
uptimeSeconds: 600,
|
|
cpuCores: 0.1,
|
|
memoryBytes: GiB / 4
|
|
};
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(epoch);
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
hostId = hosts.createHost({ name: 'First' }).id;
|
|
hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory);
|
|
await instance(hostId, sample.instanceId);
|
|
});
|
|
afterEach(async () => {
|
|
vi.useRealTimers();
|
|
await fixture.cleanup();
|
|
});
|
|
async function instance(host: string, id: string) {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
|
VALUES(?,?,?,'library','image',28960,?)`
|
|
)
|
|
.run(id, 't5_zombie_theater_' + id, host, new Date().toISOString());
|
|
}
|
|
async function report(minute: number, overrides = {}, host = hostId) {
|
|
vi.setSystemTime(epoch + minute * 60_000);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(host, { ...inventory, instanceResources: [{ ...sample, ...overrides }] });
|
|
}
|
|
async function learned() {
|
|
// One real startup followed by twenty running minute peaks.
|
|
for (let minute = 0; minute < 4; minute++)
|
|
await report(minute, {
|
|
uptimeSeconds: minute * 60,
|
|
cpuCores: 0.2,
|
|
memoryBytes: (3 * GiB) / 8
|
|
});
|
|
for (let minute = 5; minute < 25; minute++) await report(minute);
|
|
return (await import('../../src/lib/server/hosts/calibration.js')).capacityCalibration(hostId);
|
|
}
|
|
|
|
describe('measured instance capacity', () => {
|
|
it('lists retained samples with phase filters, host scope, stable ordering, and bounded pages', async () => {
|
|
for (let minute = 0; minute < 101; minute++)
|
|
await report(minute, { uptimeSeconds: minute < 5 ? minute * 60 : 600 });
|
|
const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js');
|
|
const first = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 });
|
|
const second = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 2 });
|
|
expect(first).toMatchObject({ total: 101, page: 1, pageSize: 50 });
|
|
expect(first.samples).toHaveLength(50);
|
|
expect(first.samples[0]).toMatchObject({
|
|
hostId,
|
|
hostName: 'First',
|
|
instanceId: sample.instanceId,
|
|
server: 't5_zombie_theater_instance-1',
|
|
generation: 1,
|
|
phase: 'running',
|
|
cpuCores: 0.1,
|
|
memoryBytes: GiB / 4,
|
|
timestamp: new Date(epoch + 100 * 60_000).toISOString()
|
|
});
|
|
expect(
|
|
new Set([...first.samples, ...second.samples].map((row) => row.timestamp)).size
|
|
).toBe(100);
|
|
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 999 })).toMatchObject({
|
|
page: 3,
|
|
samples: [expect.any(Object)]
|
|
});
|
|
const startup = capacitySamples(hostId, { scope: 'host', phase: 'startup', page: 1 });
|
|
expect(startup.total).toBe(5);
|
|
expect(startup.samples.every((row) => row.phase === 'startup')).toBe(true);
|
|
expect(capacitySamples(hostId, { scope: 'host', phase: 'running', page: 1 }).total).toBe(
|
|
96
|
|
);
|
|
vi.setSystemTime(epoch + 26 * 60 * 60_000);
|
|
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 })).toMatchObject({
|
|
total: 0,
|
|
samples: []
|
|
});
|
|
});
|
|
|
|
it('supports fleet samples and retains instance identity after retirement', async () => {
|
|
await report(0);
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const other = hosts.createHost({ name: 'Second' });
|
|
hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
|
|
await instance(other.id, 'instance-2');
|
|
await report(1, { instanceId: 'instance-2' }, other.id);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare('UPDATE native_instances SET retired_at=? WHERE id=?')
|
|
.run(new Date().toISOString(), sample.instanceId);
|
|
const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(
|
|
capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }).samples.map(
|
|
(row) => row.instanceId
|
|
)
|
|
).toEqual([sample.instanceId]);
|
|
const fleet = capacitySamples(hostId, { scope: 'fleet', phase: 'all', page: 1 });
|
|
expect(fleet.total).toBe(2);
|
|
expect(fleet.samples.map((row) => row.hostName)).toEqual(['Second', 'First']);
|
|
});
|
|
|
|
it('validates the samples API and rejects agent credentials and cross-origin reads', async () => {
|
|
await report(0);
|
|
const { GET } = await import('../../src/routes/api/hosts/[id]/samples/+server.js');
|
|
const request = (query = '', headers = {}, id = hostId) =>
|
|
GET({
|
|
request: new Request(`https://ec.test/api/hosts/${id}/samples${query}`, {
|
|
headers
|
|
}),
|
|
params: { id }
|
|
});
|
|
const response = request();
|
|
expect(response.headers.get('cache-control')).toBe('no-store');
|
|
expect(await response.json()).toMatchObject({
|
|
total: 1,
|
|
samples: [expect.objectContaining({ cpuCores: 0.1 })]
|
|
});
|
|
for (const query of [
|
|
'?scope=wrong',
|
|
'?phase=wrong',
|
|
'?page=-1',
|
|
'?page=1.5',
|
|
'?page=NaN',
|
|
'?page=Infinity'
|
|
]) {
|
|
expect(() => request(query)).toThrow(expect.objectContaining({ status: 400 }));
|
|
}
|
|
expect(() => request('', {}, 'missing')).toThrow(expect.objectContaining({ status: 404 }));
|
|
expect(() => request('', { origin: 'https://other.test' })).toThrow(
|
|
expect.objectContaining({ status: 403 })
|
|
);
|
|
expect(() => request('', { authorization: 'Bearer ec_host_test' })).toThrow(
|
|
expect.objectContaining({ status: 403 })
|
|
);
|
|
});
|
|
|
|
it('reports sampling failure without inventing measurements and clears it after recovery', async () => {
|
|
const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js');
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
const error = 'Container resource sampling failed.';
|
|
heartbeat(hostId, { ...inventory, instanceResourcesError: error });
|
|
expect(getHost(hostId)?.inventory.instanceResourcesError).toBe(error);
|
|
expect(capacityCalibration(hostId).running.samples).toBe(0);
|
|
for (const invalid of [42, {}, 'x'.repeat(501)]) {
|
|
expect(() =>
|
|
heartbeat(hostId, { ...inventory, instanceResourcesError: invalid })
|
|
).toThrow('Invalid instance resource sampling error');
|
|
}
|
|
await report(0);
|
|
expect(getHost(hostId)?.inventory.instanceResourcesError).toBeUndefined();
|
|
expect(capacityCalibration(hostId).running.samples).toBe(1);
|
|
});
|
|
|
|
it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => {
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).startup).toMatchObject({
|
|
source: 'bootstrap',
|
|
cpuCores: 0.5,
|
|
memoryBytes: GiB
|
|
});
|
|
const model = await learned();
|
|
expect(model.running).toMatchObject({
|
|
source: 'host',
|
|
cpuCores: 0.1,
|
|
memoryBytes: GiB / 4,
|
|
samples: 20
|
|
});
|
|
expect(model.startup).toMatchObject({
|
|
source: 'host',
|
|
cpuCores: 0.2,
|
|
memoryBytes: (3 * GiB) / 8,
|
|
samples: 4
|
|
});
|
|
});
|
|
it('raises allowances immediately on a measured spike, including during cold start', async () => {
|
|
await report(0, { cpuCores: 2, memoryBytes: 2 * GiB });
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).running).toMatchObject({
|
|
source: 'bootstrap',
|
|
cpuCores: 2,
|
|
memoryBytes: 2 * GiB
|
|
});
|
|
await learned();
|
|
await report(25, { cpuCores: 3, memoryBytes: 3 * GiB });
|
|
expect(capacityCalibration(hostId).running).toMatchObject({
|
|
source: 'host',
|
|
cpuCores: 3,
|
|
memoryBytes: 3 * GiB
|
|
});
|
|
expect(capacityCalibration(hostId).startup.cpuCores).toBe(3);
|
|
});
|
|
it('uses recent fleet evidence on new hosts and survives a database restart', async () => {
|
|
const model = await learned();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const other = hosts.createHost({ name: 'Second' });
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(other.id).running).toMatchObject({
|
|
...model.running,
|
|
source: 'fleet'
|
|
});
|
|
expect(capacityCalibration(hostId)).toEqual(model);
|
|
});
|
|
it('removes legacy cached padding once while preserving host and fleet evidence', async () => {
|
|
const model = await learned();
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
// Recreate the previous schema and its padded allowance values.
|
|
db().exec('ALTER TABLE ec_resource_allowances DROP COLUMN usage_margin');
|
|
db().exec(
|
|
'UPDATE ec_resource_allowances SET cpu_cores=cpu_cores*1.25, memory_bytes=memory_bytes*1.25'
|
|
);
|
|
db().pragma('user_version = 0');
|
|
closeDb();
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId)).toEqual(model);
|
|
expect(capacityCalibration('new-host').startup).toEqual({
|
|
...model.startup,
|
|
source: 'fleet'
|
|
});
|
|
closeDb();
|
|
expect(capacityCalibration(hostId)).toEqual(model);
|
|
});
|
|
it('prefers mature local measurements when hosts have different CPU footprints', async () => {
|
|
await learned();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const other = hosts.createHost({ name: 'Slower CPU' });
|
|
hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
|
|
await instance(other.id, 'instance-2');
|
|
for (let minute = 25; minute < 45; minute++)
|
|
await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id);
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).running).toMatchObject({
|
|
source: 'host',
|
|
cpuCores: 0.1
|
|
});
|
|
expect(capacityCalibration(other.id).running).toMatchObject({
|
|
source: 'host',
|
|
cpuCores: 2
|
|
});
|
|
expect(capacityCalibration('new-host').running).toMatchObject({
|
|
source: 'fleet',
|
|
cpuCores: 2
|
|
});
|
|
});
|
|
it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => {
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
for (let n = 0; n < 40; n++) {
|
|
vi.setSystemTime(epoch + n * 1000);
|
|
heartbeat(hostId, {
|
|
...inventory,
|
|
instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }]
|
|
});
|
|
}
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([
|
|
{ cpu_cores: 0.4 }
|
|
]);
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).running).toMatchObject({
|
|
source: 'bootstrap',
|
|
samples: 1
|
|
});
|
|
});
|
|
it('does not train from missing, foreign, retired or old-generation samples', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { heartbeat, createHost } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, inventory);
|
|
const other = createHost({ name: 'Second' });
|
|
await instance(other.id, 'foreign');
|
|
await report(0, { instanceId: 'foreign' });
|
|
await report(1, { generation: 2 });
|
|
db()
|
|
.prepare('UPDATE native_instances SET retired_at=? WHERE id=?')
|
|
.run(new Date().toISOString(), sample.instanceId);
|
|
await report(2);
|
|
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toEqual([]);
|
|
});
|
|
it('rejects impossible and malformed measurements before changing heartbeat state', async () => {
|
|
const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js');
|
|
for (const overrides of [
|
|
{ cpuCores: -1 },
|
|
{ cpuCores: NaN },
|
|
{ cpuCores: 17 },
|
|
{ memoryBytes: 0 },
|
|
{ memoryBytes: 65 * GiB },
|
|
{ generation: 0 },
|
|
{ uptimeSeconds: -1 }
|
|
]) {
|
|
expect(() =>
|
|
heartbeat(hostId, {
|
|
...inventory,
|
|
instanceResources: [{ ...sample, ...overrides }]
|
|
})
|
|
).toThrow('Invalid instance resource');
|
|
}
|
|
expect(() =>
|
|
heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] })
|
|
).toThrow('Invalid instance resource');
|
|
expect(getHost(hostId)?.inventory.instanceResources).toBeUndefined();
|
|
});
|
|
it('expires stale models and prunes old measurements as new samples arrive', async () => {
|
|
await learned();
|
|
vi.setSystemTime(epoch + 26 * 60 * 60_000);
|
|
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
|
|
await report(26 * 60);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1);
|
|
expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
|
|
});
|
|
it.each(['startup', 'running'] as const)(
|
|
'uses the %s 95th percentile for CPU and memory, keeping admission reads cached',
|
|
async (phase) => {
|
|
await learned();
|
|
// Repeated starts or running observations include isolated CPU and
|
|
// memory outliers in different minutes; neither should set the allowance.
|
|
const uptimeSeconds = phase === 'startup' ? 60 : 600;
|
|
for (let minute = 25; minute < 125; minute++)
|
|
await report(minute, {
|
|
uptimeSeconds,
|
|
cpuCores: minute === 25 ? 2 : 0.4,
|
|
memoryBytes: minute === 26 ? 3 * GiB : GiB / 2
|
|
});
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { capacityCalibration } =
|
|
await import('../../src/lib/server/hosts/calibration.js');
|
|
const prepare = vi.spyOn(db(), 'prepare');
|
|
expect(capacityCalibration(hostId)[phase]).toMatchObject({
|
|
source: 'host',
|
|
cpuCores: 0.4,
|
|
memoryBytes: GiB / 2
|
|
});
|
|
expect(capacityCalibration('new-host')[phase]).toMatchObject({
|
|
source: 'fleet',
|
|
cpuCores: 0.4,
|
|
memoryBytes: GiB / 2
|
|
});
|
|
expect(
|
|
prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))
|
|
).toBe(false);
|
|
prepare.mockRestore();
|
|
|
|
// Both phases still raise immediately for fresh spikes, hold that value
|
|
// within the minute, then return to P95 on the next recalculation.
|
|
await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB });
|
|
expect(capacityCalibration(hostId)[phase]).toMatchObject({
|
|
cpuCores: 3,
|
|
memoryBytes: 2 * GiB
|
|
});
|
|
await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
|
|
expect(capacityCalibration(hostId)[phase]).toMatchObject({
|
|
cpuCores: 3,
|
|
memoryBytes: 2 * GiB
|
|
});
|
|
await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
|
|
expect(capacityCalibration(hostId)[phase]).toMatchObject({
|
|
cpuCores: 0.4,
|
|
memoryBytes: GiB / 2
|
|
});
|
|
}
|
|
);
|
|
});
|