josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/host-capacity-calibration.test.ts
+238 -72
View File
@@ -5,25 +5,48 @@ let fixture: Fixture;
let hostId: string;
const epoch = Date.parse('2026-10-04T12:00:00Z');
const GiB = 1024 ** 3;
const inventory = { cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB,
dockerRunning: true, protocolVersion: 1, directoriesWritable: true, storageAvailable: 100 * GiB,
runtimeImages: ['image'], libraries: [{ id: 'library', state: 'ready' }] };
const sample = { instanceId: 'instance-1', generation: 1, uptimeSeconds: 600, cpuCores: 0.1, memoryBytes: GiB / 4 };
const inventory = {
cpuCount: 16,
cpuUsagePercent: 10,
memoryTotal: 64 * GiB,
memoryAvailable: 60 * GiB,
dockerRunning: true,
protocolVersion: 1,
directoriesWritable: true,
storageAvailable: 100 * GiB,
runtimeImages: ['image'],
libraries: [{ id: 'library', state: 'ready' }]
};
const sample = {
instanceId: 'instance-1',
generation: 1,
uptimeSeconds: 600,
cpuCores: 0.1,
memoryBytes: GiB / 4
};
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch);
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(epoch);
const hosts = await import('../../src/lib/server/hosts/index.js');
hostId = hosts.createHost({ name: 'First' }).id;
hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory);
await instance(hostId, sample.instanceId);
});
afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); });
afterEach(async () => {
vi.useRealTimers();
await fixture.cleanup();
});
async function instance(host: string, id: string) {
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
VALUES(?,?,?,'library','image',28960,?)`).run(id, 't5_zombie_theater_' + id, host, new Date().toISOString());
db()
.prepare(
`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
VALUES(?,?,?,'library','image',28960,?)`
)
.run(id, 't5_zombie_theater_' + id, host, new Date().toISOString());
}
async function report(minute: number, overrides = {}, host = hostId) {
vi.setSystemTime(epoch + minute * 60_000);
@@ -32,30 +55,54 @@ async function report(minute: number, overrides = {}, host = hostId) {
}
async function learned() {
// One real startup followed by twenty running minute peaks.
for (let minute = 0; minute < 4; minute++) await report(minute, { uptimeSeconds: minute * 60, cpuCores: 0.2, memoryBytes: 3 * GiB / 8 });
for (let minute = 0; minute < 4; minute++)
await report(minute, {
uptimeSeconds: minute * 60,
cpuCores: 0.2,
memoryBytes: (3 * GiB) / 8
});
for (let minute = 5; minute < 25; minute++) await report(minute);
return (await import('../../src/lib/server/hosts/calibration.js')).capacityCalibration(hostId);
}
describe('measured instance capacity', () => {
it('lists retained samples with phase filters, host scope, stable ordering, and bounded pages', async () => {
for (let minute = 0; minute < 101; minute++) await report(minute, { uptimeSeconds: minute < 5 ? minute * 60 : 600 });
for (let minute = 0; minute < 101; minute++)
await report(minute, { uptimeSeconds: minute < 5 ? minute * 60 : 600 });
const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js');
const first = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 });
const second = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 2 });
expect(first).toMatchObject({ total: 101, page: 1, pageSize: 50 });
expect(first.samples).toHaveLength(50);
expect(first.samples[0]).toMatchObject({ hostId, hostName: 'First', instanceId: sample.instanceId,
server: 't5_zombie_theater_instance-1', generation: 1, phase: 'running',
cpuCores: 0.1, memoryBytes: GiB / 4, timestamp: new Date(epoch + 100 * 60_000).toISOString() });
expect(new Set([...first.samples, ...second.samples].map(row => row.timestamp)).size).toBe(100);
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 999 })).toMatchObject({ page: 3, samples: [expect.any(Object)] });
expect(first.samples[0]).toMatchObject({
hostId,
hostName: 'First',
instanceId: sample.instanceId,
server: 't5_zombie_theater_instance-1',
generation: 1,
phase: 'running',
cpuCores: 0.1,
memoryBytes: GiB / 4,
timestamp: new Date(epoch + 100 * 60_000).toISOString()
});
expect(
new Set([...first.samples, ...second.samples].map((row) => row.timestamp)).size
).toBe(100);
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 999 })).toMatchObject({
page: 3,
samples: [expect.any(Object)]
});
const startup = capacitySamples(hostId, { scope: 'host', phase: 'startup', page: 1 });
expect(startup.total).toBe(5);
expect(startup.samples.every(row => row.phase === 'startup')).toBe(true);
expect(capacitySamples(hostId, { scope: 'host', phase: 'running', page: 1 }).total).toBe(96);
expect(startup.samples.every((row) => row.phase === 'startup')).toBe(true);
expect(capacitySamples(hostId, { scope: 'host', phase: 'running', page: 1 }).total).toBe(
96
);
vi.setSystemTime(epoch + 26 * 60 * 60_000);
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 })).toMatchObject({ total: 0, samples: [] });
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 })).toMatchObject({
total: 0,
samples: []
});
});
it('supports fleet samples and retains instance identity after retirement', async () => {
@@ -66,27 +113,53 @@ describe('measured instance capacity', () => {
await instance(other.id, 'instance-2');
await report(1, { instanceId: 'instance-2' }, other.id);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId);
db()
.prepare('UPDATE native_instances SET retired_at=? WHERE id=?')
.run(new Date().toISOString(), sample.instanceId);
const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }).samples.map(row => row.instanceId)).toEqual([sample.instanceId]);
expect(
capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }).samples.map(
(row) => row.instanceId
)
).toEqual([sample.instanceId]);
const fleet = capacitySamples(hostId, { scope: 'fleet', phase: 'all', page: 1 });
expect(fleet.total).toBe(2);
expect(fleet.samples.map(row => row.hostName)).toEqual(['Second', 'First']);
expect(fleet.samples.map((row) => row.hostName)).toEqual(['Second', 'First']);
});
it('validates the samples API and rejects agent credentials and cross-origin reads', async () => {
await report(0);
const { GET } = await import('../../src/routes/api/hosts/[id]/samples/+server.js');
const request = (query = '', headers = {}, id = hostId) => GET({ request: new Request(`https://ec.test/api/hosts/${id}/samples${query}`, { headers }), params: { id } });
const request = (query = '', headers = {}, id = hostId) =>
GET({
request: new Request(`https://ec.test/api/hosts/${id}/samples${query}`, {
headers
}),
params: { id }
});
const response = request();
expect(response.headers.get('cache-control')).toBe('no-store');
expect(await response.json()).toMatchObject({ total: 1, samples: [expect.objectContaining({ cpuCores: 0.1 })] });
for (const query of ['?scope=wrong', '?phase=wrong', '?page=-1', '?page=1.5', '?page=NaN', '?page=Infinity']) {
expect(await response.json()).toMatchObject({
total: 1,
samples: [expect.objectContaining({ cpuCores: 0.1 })]
});
for (const query of [
'?scope=wrong',
'?phase=wrong',
'?page=-1',
'?page=1.5',
'?page=NaN',
'?page=Infinity'
]) {
expect(() => request(query)).toThrow(expect.objectContaining({ status: 400 }));
}
expect(() => request('', {}, 'missing')).toThrow(expect.objectContaining({ status: 404 }));
expect(() => request('', { origin: 'https://other.test' })).toThrow(expect.objectContaining({ status: 403 }));
expect(() => request('', { authorization: 'Bearer ec_host_test' })).toThrow(expect.objectContaining({ status: 403 }));
expect(() => request('', { origin: 'https://other.test' })).toThrow(
expect.objectContaining({ status: 403 })
);
expect(() => request('', { authorization: 'Bearer ec_host_test' })).toThrow(
expect.objectContaining({ status: 403 })
);
});
it('reports sampling failure without inventing measurements and clears it after recovery', async () => {
@@ -97,7 +170,9 @@ describe('measured instance capacity', () => {
expect(getHost(hostId)?.inventory.instanceResourcesError).toBe(error);
expect(capacityCalibration(hostId).running.samples).toBe(0);
for (const invalid of [42, {}, 'x'.repeat(501)]) {
expect(() => heartbeat(hostId, { ...inventory, instanceResourcesError: invalid })).toThrow('Invalid instance resource sampling error');
expect(() =>
heartbeat(hostId, { ...inventory, instanceResourcesError: invalid })
).toThrow('Invalid instance resource sampling error');
}
await report(0);
expect(getHost(hostId)?.inventory.instanceResourcesError).toBeUndefined();
@@ -106,27 +181,53 @@ describe('measured instance capacity', () => {
it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => {
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB });
expect(capacityCalibration(hostId).startup).toMatchObject({
source: 'bootstrap',
cpuCores: 0.5,
memoryBytes: GiB
});
const model = await learned();
expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.1, memoryBytes: GiB / 4, samples: 20 });
expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.2, memoryBytes: 3 * GiB / 8, samples: 4 });
expect(model.running).toMatchObject({
source: 'host',
cpuCores: 0.1,
memoryBytes: GiB / 4,
samples: 20
});
expect(model.startup).toMatchObject({
source: 'host',
cpuCores: 0.2,
memoryBytes: (3 * GiB) / 8,
samples: 4
});
});
it('raises allowances immediately on a measured spike, including during cold start', async () => {
await report(0, { cpuCores: 2, memoryBytes: 2 * GiB });
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2, memoryBytes: 2 * GiB });
expect(capacityCalibration(hostId).running).toMatchObject({
source: 'bootstrap',
cpuCores: 2,
memoryBytes: 2 * GiB
});
await learned();
await report(25, { cpuCores: 3, memoryBytes: 3 * GiB });
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3, memoryBytes: 3 * GiB });
expect(capacityCalibration(hostId).running).toMatchObject({
source: 'host',
cpuCores: 3,
memoryBytes: 3 * GiB
});
expect(capacityCalibration(hostId).startup.cpuCores).toBe(3);
});
it('uses recent fleet evidence on new hosts and survives a database restart', async () => {
const model = await learned();
const hosts = await import('../../src/lib/server/hosts/index.js');
const other = hosts.createHost({ name: 'Second' });
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(other.id).running).toMatchObject({ ...model.running, source: 'fleet' });
expect(capacityCalibration(other.id).running).toMatchObject({
...model.running,
source: 'fleet'
});
expect(capacityCalibration(hostId)).toEqual(model);
});
it('removes legacy cached padding once while preserving host and fleet evidence', async () => {
@@ -134,11 +235,16 @@ describe('measured instance capacity', () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
// Recreate the previous schema and its padded allowance values.
db().exec('ALTER TABLE ec_resource_allowances DROP COLUMN usage_margin');
db().exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores*1.25, memory_bytes=memory_bytes*1.25');
db().exec(
'UPDATE ec_resource_allowances SET cpu_cores=cpu_cores*1.25, memory_bytes=memory_bytes*1.25'
);
closeDb();
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId)).toEqual(model);
expect(capacityCalibration('new-host').startup).toEqual({ ...model.startup, source: 'fleet' });
expect(capacityCalibration('new-host').startup).toEqual({
...model.startup,
source: 'fleet'
});
closeDb();
expect(capacityCalibration(hostId)).toEqual(model);
});
@@ -148,22 +254,40 @@ describe('measured instance capacity', () => {
const other = hosts.createHost({ name: 'Slower CPU' });
hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
await instance(other.id, 'instance-2');
for (let minute = 25; minute < 45; minute++) await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id);
for (let minute = 25; minute < 45; minute++)
await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id);
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.1 });
expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2 });
expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2 });
expect(capacityCalibration(hostId).running).toMatchObject({
source: 'host',
cpuCores: 0.1
});
expect(capacityCalibration(other.id).running).toMatchObject({
source: 'host',
cpuCores: 2
});
expect(capacityCalibration('new-host').running).toMatchObject({
source: 'fleet',
cpuCores: 2
});
});
it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => {
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
for (let n = 0; n < 40; n++) {
vi.setSystemTime(epoch + n * 1000);
heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }] });
heartbeat(hostId, {
...inventory,
instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }]
});
}
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([{ cpu_cores: 0.4 }]);
expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([
{ cpu_cores: 0.4 }
]);
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', samples: 1 });
expect(capacityCalibration(hostId).running).toMatchObject({
source: 'bootstrap',
samples: 1
});
});
it('does not train from missing, foreign, retired or old-generation samples', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
@@ -173,17 +297,33 @@ describe('measured instance capacity', () => {
await instance(other.id, 'foreign');
await report(0, { instanceId: 'foreign' });
await report(1, { generation: 2 });
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId);
db()
.prepare('UPDATE native_instances SET retired_at=? WHERE id=?')
.run(new Date().toISOString(), sample.instanceId);
await report(2);
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toEqual([]);
});
it('rejects impossible and malformed measurements before changing heartbeat state', async () => {
const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js');
for (const overrides of [{ cpuCores: -1 }, { cpuCores: NaN }, { cpuCores: 17 }, { memoryBytes: 0 },
{ memoryBytes: 65 * GiB }, { generation: 0 }, { uptimeSeconds: -1 }]) {
expect(() => heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, ...overrides }] })).toThrow('Invalid instance resource');
for (const overrides of [
{ cpuCores: -1 },
{ cpuCores: NaN },
{ cpuCores: 17 },
{ memoryBytes: 0 },
{ memoryBytes: 65 * GiB },
{ generation: 0 },
{ uptimeSeconds: -1 }
]) {
expect(() =>
heartbeat(hostId, {
...inventory,
instanceResources: [{ ...sample, ...overrides }]
})
).toThrow('Invalid instance resource');
}
expect(() => heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] })).toThrow('Invalid instance resource');
expect(() =>
heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] })
).toThrow('Invalid instance resource');
expect(getHost(hostId)?.inventory.instanceResources).toBeUndefined();
});
it('expires stale models and prunes old measurements as new samples arrive', async () => {
@@ -196,29 +336,55 @@ describe('measured instance capacity', () => {
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1);
expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
});
it.each(['startup', 'running'] as const)('uses the %s 95th percentile for CPU and memory, keeping admission reads cached', async phase => {
await learned();
// Repeated starts or running observations include isolated CPU and
// memory outliers in different minutes; neither should set the allowance.
const uptimeSeconds = phase === 'startup' ? 60 : 600;
for (let minute = 25; minute < 125; minute++) await report(minute, {
uptimeSeconds, cpuCores: minute === 25 ? 2 : 0.4, memoryBytes: minute === 26 ? 3 * GiB : GiB / 2
});
const { db } = await import('../../src/lib/server/db/index.js');
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
const prepare = vi.spyOn(db(), 'prepare');
expect(capacityCalibration(hostId)[phase]).toMatchObject({ source: 'host', cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(capacityCalibration('new-host')[phase]).toMatchObject({ source: 'fleet', cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false);
prepare.mockRestore();
it.each(['startup', 'running'] as const)(
'uses the %s 95th percentile for CPU and memory, keeping admission reads cached',
async (phase) => {
await learned();
// Repeated starts or running observations include isolated CPU and
// memory outliers in different minutes; neither should set the allowance.
const uptimeSeconds = phase === 'startup' ? 60 : 600;
for (let minute = 25; minute < 125; minute++)
await report(minute, {
uptimeSeconds,
cpuCores: minute === 25 ? 2 : 0.4,
memoryBytes: minute === 26 ? 3 * GiB : GiB / 2
});
const { db } = await import('../../src/lib/server/db/index.js');
const { capacityCalibration } =
await import('../../src/lib/server/hosts/calibration.js');
const prepare = vi.spyOn(db(), 'prepare');
expect(capacityCalibration(hostId)[phase]).toMatchObject({
source: 'host',
cpuCores: 0.4,
memoryBytes: GiB / 2
});
expect(capacityCalibration('new-host')[phase]).toMatchObject({
source: 'fleet',
cpuCores: 0.4,
memoryBytes: GiB / 2
});
expect(
prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))
).toBe(false);
prepare.mockRestore();
// Both phases still raise immediately for fresh spikes, hold that value
// within the minute, then return to P95 on the next recalculation.
await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB });
expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB });
await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB });
await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 0.4, memoryBytes: GiB / 2 });
});
// Both phases still raise immediately for fresh spikes, hold that value
// within the minute, then return to P95 on the next recalculation.
await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB });
expect(capacityCalibration(hostId)[phase]).toMatchObject({
cpuCores: 3,
memoryBytes: 2 * GiB
});
await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(capacityCalibration(hostId)[phase]).toMatchObject({
cpuCores: 3,
memoryBytes: 2 * GiB
});
await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 });
expect(capacityCalibration(hostId)[phase]).toMatchObject({
cpuCores: 0.4,
memoryBytes: GiB / 2
});
}
);
});