Add host temperature telemetry and history

This commit is contained in:
2026-10-13 23:39:04 -04:00
parent 09396ab584
commit 171f657876
28 changed files with 1466 additions and 32 deletions
+95
View File
@@ -34,6 +34,101 @@ async function sample(id: string, at: number, cpu = 25, memory = 4 * GiB, cores
}
describe('capacity history', () => {
it('keeps separate CPU series, device drill-down, real zero and missing temperature gaps', async () => {
await host('one');
await host('two');
await host('unsupported');
await host('removed');
const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
const cpu = (celsius: number) => ({
id: 'package',
kind: 'cpu-package' as const,
device: 'coretemp',
label: 'Package id 0',
celsius
});
const drive = {
id: 'drive',
kind: 'nvme' as const,
device: 'nvme',
label: 'Composite',
celsius: 50.9
};
recordTemperatures(
'one',
{ temperatures: [cpu(65), drive] },
new Date(epoch - 55_000).toISOString()
);
recordTemperatures(
'one',
{ temperatures: [cpu(75)] },
new Date(epoch - 30_000).toISOString()
);
recordTemperatures(
'one',
{ temperatures: [cpu(0)] },
new Date(epoch - 180_000).toISOString()
);
recordTemperatures(
'two',
{ temperatures: [cpu(60)] },
new Date(epoch - 40_000).toISOString()
);
recordTemperatures(
'removed',
{ temperatures: [cpu(99)] },
new Date(epoch - 40_000).toISOString()
);
const { db } = await import('../../src/lib/server/db/index.js');
db()
.prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?')
.run(new Date(epoch).toISOString(), 'removed');
const { readCapacityHistory } =
await import('../../src/lib/server/hosts/capacity-history.js');
const fleet = readCapacityHistory(query, epoch).temperatures;
expect(fleet.map((s) => s.label)).toEqual(['one', 'two', 'unsupported']);
expect(fleet[0].points.slice(-3).map((p) => p.celsius)).toEqual([0, null, 75]);
expect(fleet[1].points.at(-1)?.celsius).toBe(60);
expect(fleet[2].points.every((p) => p.celsius === null)).toBe(true);
const selected = readCapacityHistory(
{ ...query, hostId: 'one', game: 't5' },
epoch
).temperatures;
expect(selected.map((s) => s.kind)).toEqual(['cpu', 'nvme']);
expect(selected[1].points.at(-1)?.celsius).toBe(50.9);
expect(
readCapacityHistory({ ...query, hostId: 'one', game: 't6' }, epoch).temperatures
).toEqual(selected);
expect(readCapacityHistory({ ...query, hostId: 'missing' }, epoch).temperatures).toEqual(
[]
);
});
it('preserves temperature peaks for long ranges and includes a host enrolled during the current minute', async () => {
await host('new', epoch - 20_000);
const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
recordTemperatures(
'new',
{
temperatures: [
{
id: 'pkg',
device: 'coretemp',
kind: 'cpu-package',
label: 'Package id 0',
celsius: 65
}
]
},
new Date(epoch - 10_000).toISOString()
);
const { readCapacityHistory } =
await import('../../src/lib/server/hosts/capacity-history.js');
expect(readCapacityHistory(query, epoch).temperatures[0].points.at(-1)?.celsius).toBe(65);
const long = readCapacityHistory({ ...query, range: '7d' }, epoch).temperatures[0];
expect(long.points).toHaveLength(168);
expect(long.points.at(-1)?.celsius).toBe(65);
});
it('averages each host before summing fleet headroom and exposes missing observations as gaps', async () => {
await host('one');
await host('two');
+151
View File
@@ -0,0 +1,151 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import {
cpuTemperature,
historyTemperatures,
validateTemperatures,
type HostTemperatureSensor
} from '../../src/lib/host-temperatures.js';
const sensor = (changes: Partial<HostTemperatureSensor> = {}): HostTemperatureSensor => ({
id: 'package',
device: 'coretemp',
label: 'Package id 0',
kind: 'cpu-package',
celsius: 65,
...changes
});
describe('temperature readings', () => {
it('keeps valid sensors while dropping malformed, duplicate and out-of-range entries', () => {
const good = sensor();
const readings = [
good,
good,
null,
'bad',
...[NaN, Infinity, -Infinity, null, '65', true, -101, 201].map((celsius) => ({
...good,
id: 'invalid',
celsius
})),
{ ...good, id: 'x'.repeat(101) },
{ ...good, label: '' },
{ ...good, kind: 'cpu-control' },
sensor({ id: 'zero', celsius: 0 })
];
expect(validateTemperatures(readings)).toEqual([good, sensor({ id: 'zero', celsius: 0 })]);
expect(validateTemperatures({})).toEqual([]);
expect(validateTemperatures(undefined)).toEqual([]);
});
it('chooses the hottest package and uses die/core readings only as fallbacks', () => {
const packages = [sensor(), sensor({ id: 'socket2', label: 'Package id 1', celsius: 72 })];
const core = sensor({ id: 'core', label: 'Core 0', kind: 'cpu-core', celsius: 80 });
const die = sensor({ id: 'die', label: 'Tdie', kind: 'cpu-die', celsius: 64 });
expect(cpuTemperature([...packages, core])?.celsius).toBe(72);
expect(cpuTemperature([die, core])?.celsius).toBe(64);
expect(cpuTemperature([core])?.celsius).toBe(80);
expect(cpuTemperature()).toBeNull();
const drive = sensor({
id: 'drive',
kind: 'nvme',
device: 'nvme',
label: 'Composite',
celsius: 50.9
});
expect(
historyTemperatures([
...packages,
core,
drive,
{ ...drive, id: 'internal', label: 'Sensor 1' }
])
).toEqual([{ ...packages[1], id: 'cpu', label: 'CPU temperature' }, drive]);
});
});
describe('temperature persistence', () => {
let fixture: Fixture;
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
});
afterEach(async () => {
vi.restoreAllMocks();
await fixture.cleanup();
});
it('accepts older agents and faulty temperature payloads without losing host metrics', async () => {
const { createHost, heartbeat, validateInventory } =
await import('../../src/lib/server/hosts/index.js');
const host = createHost({ name: 'ec-2' });
expect(validateInventory({ cpuUsagePercent: 25 }).temperatures).toBeUndefined();
expect(
heartbeat(host.id, {
cpuUsagePercent: 25,
temperatures: [sensor(), { celsius: 'bad' }]
}).inventory
).toMatchObject({ cpuUsagePercent: 25, temperatures: [sensor()] });
expect(
heartbeat(host.id, { cpuUsagePercent: 30, temperatures: [] }).inventory.temperatures
).toEqual([]);
expect(
heartbeat(host.id, { cpuUsagePercent: 35, temperatures: 'bad' }).inventory.temperatures
).toEqual([]);
const { db } = await import('../../src/lib/server/db/index.js');
expect(
db()
.prepare('SELECT celsius FROM ec_host_temperature_samples WHERE host_id=?')
.all(host.id)
).toEqual([{ celsius: 65 }]);
});
it('retains one peak per minute and applies host history retention, including offline hosts', async () => {
const { createHost } = await import('../../src/lib/server/hosts/index.js');
const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { fleetSettings, saveFleetSettings } =
await import('../../src/lib/server/settings.js');
const { pruneSampleHistory } = await import('../../src/lib/server/retention.js');
const host = createHost({ name: 'ec-2' });
const minute = Math.floor((Date.now() - 3 * 86400_000) / 60_000) * 60_000;
for (const [i, celsius] of [65, 78, 67].entries())
recordTemperatures(
host.id,
{ temperatures: [sensor({ celsius })] },
new Date(minute + i * 15_000).toISOString()
);
expect(db().prepare('SELECT celsius FROM ec_host_temperature_samples').all()).toEqual([
{ celsius: 78 }
]);
saveFleetSettings({ ...fleetSettings(), hostResourceKeepDays: 0 });
pruneSampleHistory();
expect(db().prepare('SELECT COUNT(*) AS n FROM ec_host_temperature_samples').get()).toEqual(
{ n: 1 }
);
recordTemperatures(host.id, { temperatures: [sensor()] }, new Date().toISOString());
saveFleetSettings({ ...fleetSettings(), hostResourceKeepDays: 1 });
pruneSampleHistory();
expect(db().prepare('SELECT celsius FROM ec_host_temperature_samples').all()).toEqual([
{ celsius: 65 }
]);
});
it('upgrades an existing database without rewriting its resource samples', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { createHost } = await import('../../src/lib/server/hosts/index.js');
const host = createHost({ name: 'old' });
db()
.prepare(
'INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_usage_percent) VALUES(?,?,?)'
)
.run(host.id, '2026-10-13T12:00:00Z', 25);
db().exec('DROP TABLE ec_host_temperature_samples; PRAGMA user_version=22;');
closeDb();
expect(db().prepare('SELECT * FROM ec_host_temperature_samples').all()).toEqual([]);
expect(
db().prepare('SELECT cpu_usage_percent FROM ec_host_resource_samples').all()
).toEqual([{ cpu_usage_percent: 25 }]);
expect(db().pragma('foreign_key_check')).toEqual([]);
});
});
+5
View File
@@ -43,6 +43,7 @@ const host = (changes: Partial<CapacityHost> = {}): CapacityHost => ({
...changes
});
const history: CapacityHistory = {
temperatures: [],
range: '24h',
startAt: 1000,
endAt: 5000,
@@ -112,6 +113,10 @@ it('keeps current estimates and host comparison before history with scoped histo
expect(body).not.toContain('Manage hosts');
expect(body).toContain('Updates every 30 seconds');
expect(body).not.toContain('>Refresh<');
expect(body.indexOf('id="temperature-history-title"')).toBeGreaterThan(
body.indexOf('id="capacity-history-title"')
);
expect(body.match(/aria-label="History range"/g)).toHaveLength(2);
});
it('keeps fleet-wide queue scope explicit and preserves unavailable estimates', () => {
+35
View File
@@ -9,6 +9,7 @@ import {
stableHostOrder
} from '../../src/lib/ui/host-overview.js';
import HostList from '../../src/lib/ui/components/HostList.svelte';
import HostTemperatures from '../../src/lib/ui/components/HostTemperatures.svelte';
const now = Date.parse('2026-10-06T12:00:00Z');
function host(id = 'ec-1', changes: Partial<Host> = {}): Host {
@@ -43,6 +44,40 @@ function host(id = 'ec-1', changes: Partial<Host> = {}): Host {
};
}
it('shows CPU temperature with stale status and keeps sensor details expandable', () => {
const target = host('ec-2', {
connected: false,
inventory: {
cpuUsagePercent: 25,
temperatures: [
{
id: 'cpu',
kind: 'cpu-package',
device: 'coretemp',
label: 'Package id 0',
celsius: 65
}
]
}
});
const list = render(HostList, {
props: {
hosts: [{ ...target, serverCount: 0, playerCount: 0, unreportedInstances: 0 }],
now
}
}).body;
expect(list).toContain('CPU temp: 65 °C');
expect(list).toContain('Resource readings stale');
const detail = render(HostTemperatures, { props: { host: target } }).body;
expect(detail).toContain('(last reported)');
expect(detail).toContain('Temperature sensors');
expect(detail).toContain('CPU package 0');
expect(detail).not.toMatch(/<details[^>]*open/);
const missing = render(HostTemperatures, { props: { host: host() } }).body;
expect(missing).toContain('Unavailable');
expect(missing).not.toContain('0 °C');
});
it('does not show an old Ready state as available when the heartbeat becomes stale', () => {
expect(hostListStatus(host(), now).label).toBe('Ready');
expect(hostReadingsStale(host(), now + 90_001)).toBe(true);