Add host temperature telemetry and history
This commit is contained in:
@@ -1,5 +1,24 @@
|
||||
# Agent changelog
|
||||
|
||||
## 2026-10-13
|
||||
|
||||
### 1.10.0 — Host temperature telemetry
|
||||
|
||||
- Read supported Linux hwmon CPU package/core, AMD physical die/CCD, NVMe and
|
||||
Intel PCH temperatures with the Python standard library. Sensor failures,
|
||||
faults and disabled inputs are skipped independently; unsupported hosts report
|
||||
an empty list. AMD Tctl control offsets are excluded from physical readings.
|
||||
- Include labeled, bounded sensors in heartbeats with the `host.temperatures`
|
||||
capability. Physical device/channel identities survive hwmon renumbering.
|
||||
- The controller shows current CPU readings and expandable sensor details, and
|
||||
adds temperature history to Capacity. Fleet history has one CPU line per host;
|
||||
individual hosts can add NVMe composite and chipset lines. History retains
|
||||
minute peaks using the host-resource retention setting; missing data stays
|
||||
unavailable and no thermal alerts or placement rules are introduced.
|
||||
- Adds database migration 23. Existing agents remain compatible. Release
|
||||
artifacts are packaged and the stale signature is removed. Offline signing
|
||||
and host rollout are still required; neither was performed here.
|
||||
|
||||
## 2026-10-12
|
||||
|
||||
### 1.9.3 — Protect private bank configuration (EC #152)
|
||||
|
||||
+58
-3
@@ -35,7 +35,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.9.3"
|
||||
VERSION = "1.10.0"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
MANIFEST_MAX_ENTRIES = 10000
|
||||
@@ -63,6 +63,7 @@ UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-ch
|
||||
UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
|
||||
UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
|
||||
AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
|
||||
HWMON_ROOT = Path("/sys/class/hwmon")
|
||||
ROOT_MARKER = ".electric-cherry.json"
|
||||
|
||||
|
||||
@@ -1308,6 +1309,59 @@ class Agent:
|
||||
return None
|
||||
return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
|
||||
|
||||
def temperatures(self):
|
||||
"""Best-effort, read-only samples from known millidegree-Celsius drivers."""
|
||||
samples = []
|
||||
try:
|
||||
chips = sorted(HWMON_ROOT.glob("hwmon*"))[:64]
|
||||
except OSError:
|
||||
return samples
|
||||
for chip in chips:
|
||||
try:
|
||||
device = (chip / "name").read_text().strip()
|
||||
if device not in ("coretemp", "k10temp", "nvme") and not device.startswith("pch_"):
|
||||
continue
|
||||
# hwmon numbers can change on reboot. The physical sysfs path
|
||||
# and channel identify separate CPU sockets and NVMe devices.
|
||||
identity = re.sub(r"/(?:hwmon/)?hwmon\d+$", "", str(chip.resolve()))
|
||||
inputs = sorted(chip.glob("temp[0-9]*_input"))[:128]
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
for source in inputs:
|
||||
try:
|
||||
channel = source.name.removesuffix("_input")
|
||||
label_path = chip / (channel + "_label")
|
||||
label = label_path.read_text().strip() if label_path.exists() else channel
|
||||
if not label or len(label) > 120:
|
||||
continue
|
||||
fault = chip / (channel + "_fault")
|
||||
enabled = chip / (channel + "_enable")
|
||||
if (fault.exists() and int(fault.read_text()) != 0) or (enabled.exists() and int(enabled.read_text()) == 0):
|
||||
continue
|
||||
if device == "coretemp":
|
||||
kind = "cpu-package" if label.startswith("Package id ") else "cpu-core" if label.startswith("Core ") else None
|
||||
elif device == "k10temp":
|
||||
# Tctl can contain a fan-control offset. Never present
|
||||
# it as the physical die temperature or a fleet metric.
|
||||
kind = "cpu-die" if label == "Tdie" or re.fullmatch(r"Tccd\d+", label) else None
|
||||
else:
|
||||
kind = "nvme" if device == "nvme" else "chipset"
|
||||
if kind is None:
|
||||
continue
|
||||
celsius = int(source.read_text()) / 1000
|
||||
if not -100 <= celsius <= 200:
|
||||
continue
|
||||
sensor_id = hashlib.sha256((identity + "/" + channel).encode()).hexdigest()[:24]
|
||||
samples.append({"id": sensor_id, "device": device, "label": label,
|
||||
"kind": kind, "celsius": round(celsius, 1)})
|
||||
if len(samples) >= 128:
|
||||
return samples
|
||||
except (OSError, ValueError):
|
||||
# Read permission alone does not imply data is available
|
||||
# (e.g. ENODATA), and a device can disappear during a read.
|
||||
continue
|
||||
return samples
|
||||
|
||||
def instance_resources(self, observations=None):
|
||||
"""One bounded Docker stats batch; unavailable samples are never zero load."""
|
||||
ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true",
|
||||
@@ -1392,6 +1446,7 @@ class Agent:
|
||||
|
||||
def inventory(self):
|
||||
cpu_usage = self.cpu_usage_percent()
|
||||
temperatures = self.temperatures()
|
||||
usage = shutil.disk_usage(self.root)
|
||||
memory = {}
|
||||
try:
|
||||
@@ -1516,9 +1571,9 @@ class Agent:
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
update = self.update_inventory()
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
capabilities = ["host.temperatures", "library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
plutonium_cache = self.plutonium_cache_inventory()
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "temperatures": temperatures, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
||||
health = self.inspect(instance_id, generation)
|
||||
|
||||
@@ -778,6 +778,12 @@ class AgentTests(unittest.TestCase):
|
||||
else:
|
||||
self.assertEqual(inventory["cpuUsagePercent"], usage)
|
||||
|
||||
def test_inventory_reports_optional_temperature_capability_and_empty_samples(self):
|
||||
with patch.object(self.agent, "temperatures", return_value=[]):
|
||||
inventory = self.agent.inventory()
|
||||
self.assertEqual(inventory["temperatures"], [])
|
||||
self.assertIn("host.temperatures", inventory["capabilities"])
|
||||
|
||||
def resource_container(self):
|
||||
return {"Name": self.agent.container_name(INSTANCE, 1),
|
||||
"Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}},
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import errno
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("ec_agent_temperatures", Path(__file__).resolve().parents[1] / "ec_agent.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
|
||||
class TemperatureTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.hwmon = self.root / "class"
|
||||
self.hwmon.mkdir()
|
||||
self.patch = patch.object(module, "HWMON_ROOT", self.hwmon)
|
||||
self.patch.start()
|
||||
self.addCleanup(self.patch.stop)
|
||||
self.agent = object.__new__(module.Agent)
|
||||
|
||||
def chip(self, number, device, readings):
|
||||
physical = self.root / (device + str(number)) / "hwmon" / ("hwmon" + str(number))
|
||||
physical.mkdir(parents=True)
|
||||
(self.hwmon / physical.name).symlink_to(physical, target_is_directory=True)
|
||||
(physical / "name").write_text(device)
|
||||
for channel, label, value in readings:
|
||||
(physical / (channel + "_input")).write_text(str(value))
|
||||
if label is not None:
|
||||
(physical / (channel + "_label")).write_text(label)
|
||||
return physical
|
||||
|
||||
def test_ec2_sensors_and_units_without_guessing_unknown_devices(self):
|
||||
self.chip(0, "nvme", [("temp1", "Composite", 50900), ("temp2", "Sensor 1", 50900)])
|
||||
self.chip(1, "pch_cometlake", [("temp1", None, 62000)])
|
||||
self.chip(2, "coretemp", [("temp1", "Package id 0", 65000), ("temp2", "Core 0", 63000)])
|
||||
self.chip(3, "iwlwifi_1", [("temp1", None, "unavailable")])
|
||||
samples = self.agent.temperatures()
|
||||
self.assertEqual([s["celsius"] for s in samples], [50.9, 50.9, 62.0, 65.0, 63.0])
|
||||
self.assertEqual([s["kind"] for s in samples], ["nvme", "nvme", "chipset", "cpu-package", "cpu-core"])
|
||||
self.assertEqual(len({s["id"] for s in samples}), len(samples))
|
||||
|
||||
def test_fault_disabled_bad_value_and_enodata_do_not_lose_other_sensors(self):
|
||||
chip = self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), value)
|
||||
for i, value in enumerate([65000, 60000, "bad", 500000, 61000, 63000, 0], 1)])
|
||||
(chip / "temp1_fault").write_text("1")
|
||||
(chip / "temp2_enable").write_text("0")
|
||||
original = Path.read_text
|
||||
def read(path, *args, **kwargs):
|
||||
if path.name == "temp5_input":
|
||||
raise OSError(errno.ENODATA, "No data available")
|
||||
return original(path, *args, **kwargs)
|
||||
with patch.object(Path, "read_text", read):
|
||||
self.assertEqual([s["celsius"] for s in self.agent.temperatures()], [63.0, 0.0])
|
||||
|
||||
def test_amd_keeps_physical_die_readings_and_excludes_control_offsets(self):
|
||||
self.chip(0, "k10temp", [("temp1", "Tctl", 85000), ("temp2", "Tdie", 65000), ("temp3", "Tccd1", 64000)])
|
||||
self.assertEqual([s["label"] for s in self.agent.temperatures()], ["Tdie", "Tccd1"])
|
||||
|
||||
def test_sensor_identity_survives_hwmon_renumbering_and_separates_drives(self):
|
||||
physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
|
||||
self.chip(2, "nvme", [("temp1", "Composite", 51000)])
|
||||
before = {s["celsius"]: s["id"] for s in self.agent.temperatures()}
|
||||
renamed = physical.with_name("hwmon9")
|
||||
physical.rename(renamed)
|
||||
(self.hwmon / "hwmon1").unlink()
|
||||
(self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
|
||||
self.assertEqual(before, {s["celsius"]: s["id"] for s in self.agent.temperatures()})
|
||||
self.assertEqual(len(set(before.values())), 2)
|
||||
|
||||
def test_missing_hwmon_and_disappearing_devices_are_optional(self):
|
||||
self.assertEqual(self.agent.temperatures(), [])
|
||||
(self.hwmon / "hwmon0").symlink_to(self.root / "missing")
|
||||
self.assertEqual(self.agent.temperatures(), [])
|
||||
with patch.object(Path, "glob", side_effect=PermissionError()):
|
||||
self.assertEqual(self.agent.temperatures(), [])
|
||||
|
||||
def test_direct_hwmon_child_keeps_identity_when_renumbered(self):
|
||||
physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
|
||||
direct = physical.parent.parent / "hwmon8"
|
||||
physical.rename(direct)
|
||||
(self.hwmon / "hwmon1").unlink()
|
||||
(self.hwmon / "hwmon8").symlink_to(direct, target_is_directory=True)
|
||||
before = self.agent.temperatures()[0]["id"]
|
||||
renamed = direct.with_name("hwmon9")
|
||||
direct.rename(renamed)
|
||||
(self.hwmon / "hwmon8").unlink()
|
||||
(self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
|
||||
self.assertEqual(self.agent.temperatures()[0]["id"], before)
|
||||
|
||||
def test_inventory_size_is_bounded(self):
|
||||
self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), 65000) for i in range(1, 200)])
|
||||
self.assertEqual(len(self.agent.temperatures()), 128)
|
||||
@@ -354,6 +354,40 @@ journalctl -u electric-cherry-update.service -n 100
|
||||
|
||||
## Placement and resource capacity
|
||||
|
||||
### Host temperature
|
||||
|
||||
Agent **1.10.0** reports temperature sensors exposed by supported Linux hwmon
|
||||
drivers without installing additional packages. **Hosts** shows CPU temperature
|
||||
beneath CPU usage. A host's **Overview → Inventory** shows the selected reading
|
||||
and expandable **Temperature sensors**, including per-core and storage details.
|
||||
Stale readings are marked as last reported; absent or failed readings are
|
||||
**Unavailable**, never zero or a readiness failure.
|
||||
|
||||
**Fleet → Capacity → Temperature history** uses the existing host selector and
|
||||
Last hour / 24 hours / 7 days controls. **All hosts** displays a separate CPU line
|
||||
for each host. Select one host to add its NVMe composite and chipset lines using
|
||||
the legend. Hover, tap, or use the arrow keys on the chart to inspect a period.
|
||||
Each host page's **Capacity** button opens the view filtered to that host.
|
||||
The game filter affects launch estimates, not temperatures.
|
||||
|
||||
The CPU reading is the hottest reported package, falling back to a physical
|
||||
AMD die/CCD and then a core reading. AMD Tctl cooling-control values are excluded
|
||||
because they may include an offset. Supported device drivers are `coretemp`,
|
||||
`k10temp`, `nvme` and `pch_*`; sensor availability depends on hardware and drivers.
|
||||
Virtual machines and older agents may have no supported readings. Missing,
|
||||
faulty and disabled sensors are skipped individually.
|
||||
|
||||
Temperature history starts after the agent upgrade. The controller stores one
|
||||
peak per sensor per minute, limited to the CPU summary and up to sixteen NVMe
|
||||
composite/chipset inputs per host. Longer chart periods show the maximum of
|
||||
those peaks; a period without readings remains a gap. Brief gaps within an
|
||||
aggregated period may still leave a peak for that period. **Host resource history
|
||||
retention** applies to CPU, memory and temperature samples (seven days by default;
|
||||
zero keeps them indefinitely). Series colors identify hosts/sensors and do not
|
||||
indicate thermal safety. Temperature does not change placement or trigger alerts.
|
||||
|
||||
### CPU and memory capacity
|
||||
|
||||
### Host inventory and eligibility
|
||||
|
||||
The Inventory panel shows CPU usage with agent 1.4.2 and later. It measures
|
||||
|
||||
@@ -30,6 +30,14 @@ export interface CapacityHistory {
|
||||
endAt: number;
|
||||
bucketMs: number;
|
||||
points: CapacityHistoryPoint[];
|
||||
temperatures: TemperatureHistorySeries[];
|
||||
}
|
||||
export interface TemperatureHistorySeries {
|
||||
id: string;
|
||||
hostId: string;
|
||||
label: string;
|
||||
kind: 'cpu' | 'nvme' | 'chipset';
|
||||
points: { at: number; celsius: number | null }[];
|
||||
}
|
||||
export interface CapacityHost {
|
||||
id: string;
|
||||
|
||||
@@ -305,7 +305,7 @@ export const settingHelp: Partial<Record<keyof FleetSettings, string>> = {
|
||||
'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.',
|
||||
recoveryKeepDays:
|
||||
'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.',
|
||||
hostResourceKeepDays: 'Host CPU and memory history.',
|
||||
hostResourceKeepDays: 'Host CPU, memory and temperature history.',
|
||||
instanceResourceKeepDays:
|
||||
'Minute resource samples. Capacity learning continues to use its independent 24-hour window.',
|
||||
metricsKeepDays: 'Historical fleet occupancy and utilization samples.',
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
export const temperatureKinds = ['cpu-package', 'cpu-die', 'cpu-core', 'nvme', 'chipset'] as const;
|
||||
export type TemperatureKind = (typeof temperatureKinds)[number];
|
||||
export interface HostTemperatureSensor {
|
||||
id: string;
|
||||
device: string;
|
||||
label: string;
|
||||
kind: TemperatureKind;
|
||||
celsius: number;
|
||||
}
|
||||
|
||||
/** Optional telemetry must never make an otherwise valid heartbeat fail. */
|
||||
export function validateTemperatures(value: unknown): HostTemperatureSensor[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
const seen = new Set<string>();
|
||||
return value.slice(0, 128).flatMap((sensor) => {
|
||||
if (
|
||||
!sensor ||
|
||||
typeof sensor !== 'object' ||
|
||||
Array.isArray(sensor) ||
|
||||
typeof sensor.id !== 'string' ||
|
||||
!/^[a-zA-Z0-9_-]{1,100}$/.test(sensor.id) ||
|
||||
seen.has(sensor.id) ||
|
||||
!['device', 'label'].every(
|
||||
(key) =>
|
||||
typeof sensor[key] === 'string' &&
|
||||
sensor[key].length > 0 &&
|
||||
sensor[key].length <= 120
|
||||
) ||
|
||||
!temperatureKinds.includes(sensor.kind) ||
|
||||
typeof sensor.celsius !== 'number' ||
|
||||
!Number.isFinite(sensor.celsius) ||
|
||||
sensor.celsius < -100 ||
|
||||
sensor.celsius > 200
|
||||
)
|
||||
return [];
|
||||
seen.add(sensor.id);
|
||||
return [
|
||||
{
|
||||
id: sensor.id,
|
||||
device: sensor.device,
|
||||
label: sensor.label,
|
||||
kind: sensor.kind,
|
||||
celsius: sensor.celsius
|
||||
}
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
export function cpuTemperature(
|
||||
sensors: HostTemperatureSensor[] = []
|
||||
): HostTemperatureSensor | null {
|
||||
// Prefer physical package/die readings. Cores are a fallback, never mixed
|
||||
// into an average that might hide a hot package on a multi-socket host.
|
||||
for (const kind of ['cpu-package', 'cpu-die', 'cpu-core'] as const) {
|
||||
const candidates = sensors.filter((sensor) => sensor.kind === kind);
|
||||
if (candidates.length)
|
||||
return candidates.reduce((hot, sensor) =>
|
||||
sensor.celsius > hot.celsius ? sensor : hot
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function temperatureLabel(sensor: HostTemperatureSensor): string {
|
||||
if (sensor.kind === 'cpu-package')
|
||||
return `CPU package ${sensor.label.replace(/^Package id /, '')}`;
|
||||
if (sensor.kind === 'cpu-core') return `CPU ${sensor.label.toLowerCase()}`;
|
||||
if (sensor.kind === 'cpu-die') return `CPU die · ${sensor.label}`;
|
||||
if (sensor.kind === 'chipset') return 'Chipset';
|
||||
return `NVMe · ${sensor.label}`;
|
||||
}
|
||||
|
||||
export function formatTemperature(celsius: number | null | undefined): string {
|
||||
return typeof celsius === 'number' && Number.isFinite(celsius)
|
||||
? `${Number(celsius.toFixed(1))} °C`
|
||||
: 'Unavailable';
|
||||
}
|
||||
|
||||
/** One CPU series per host, plus bounded device series for the host drill-down. */
|
||||
export function historyTemperatures(
|
||||
sensors: HostTemperatureSensor[] = []
|
||||
): HostTemperatureSensor[] {
|
||||
const cpu = cpuTemperature(sensors);
|
||||
const devices = sensors
|
||||
.filter(
|
||||
(sensor) =>
|
||||
sensor.kind === 'chipset' ||
|
||||
(sensor.kind === 'nvme' && sensor.label.toLowerCase() === 'composite')
|
||||
)
|
||||
.slice(0, 16);
|
||||
return [...(cpu ? [{ ...cpu, id: 'cpu', label: 'CPU temperature' }] : []), ...devices];
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { HostLibraryCache } from './game-files.js';
|
||||
import type { HostTemperatureSensor } from './host-temperatures.js';
|
||||
|
||||
export type HostMode = 'active' | 'draining' | 'disabled';
|
||||
export type HostReadiness =
|
||||
@@ -149,6 +150,7 @@ export interface HostInventory {
|
||||
cpu?: string;
|
||||
cpuCount?: number;
|
||||
cpuUsagePercent?: number;
|
||||
temperatures?: HostTemperatureSensor[];
|
||||
memoryTotal?: number;
|
||||
memoryAvailable?: number;
|
||||
storageTotal?: number;
|
||||
|
||||
@@ -24,6 +24,7 @@ import { migrateLeaderboardReports } from './leaderboard-reports.js';
|
||||
import { migratePlayerScoreTotal } from './player-score-total.js';
|
||||
import { migrateBank } from './bank.js';
|
||||
import { migrateBankCredentials } from './bank-credentials.js';
|
||||
import { migrateTemperatures } from '../hosts/temperatures.js';
|
||||
import type { Migration } from './migration-runner.js';
|
||||
|
||||
// Append only: never renumber entries or change an applied migration to add an
|
||||
@@ -118,7 +119,8 @@ export const migrations: readonly Migration[] = [
|
||||
},
|
||||
{ version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank },
|
||||
{ version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials },
|
||||
{ version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions }
|
||||
{ version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions },
|
||||
{ version: 23, name: 'host temperature history', run: migrateTemperatures }
|
||||
];
|
||||
|
||||
export const databaseVersion = migrations.length;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { db } from '../db/index.js';
|
||||
import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js';
|
||||
import { readTemperatureHistory } from './temperatures.js';
|
||||
|
||||
const windows: Record<CapacityRange, { duration: number; bucketMs: number }> = {
|
||||
'1h': { duration: 3_600_000, bucketMs: 60_000 },
|
||||
@@ -15,10 +16,14 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
|
||||
const count = duration / bucketMs;
|
||||
const hosts = handle
|
||||
.prepare(
|
||||
`SELECT id,created_at FROM ec_hosts
|
||||
`SELECT id,name,created_at FROM ec_hosts
|
||||
WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`
|
||||
)
|
||||
.all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[];
|
||||
.all(...(query.hostId ? [query.hostId] : [])) as {
|
||||
id: string;
|
||||
name: string;
|
||||
created_at: string;
|
||||
}[];
|
||||
// The primary key bounds each host scan to the selected interval. Aggregate
|
||||
// before crossing the worker boundary; do not return raw heartbeat rows.
|
||||
const samples = handle.prepare(`SELECT
|
||||
@@ -69,5 +74,12 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
|
||||
point.memoryBytesAvailable = null;
|
||||
}
|
||||
}
|
||||
return { range: query.range, startAt, endAt: now, bucketMs, points };
|
||||
return {
|
||||
range: query.range,
|
||||
startAt,
|
||||
endAt: now,
|
||||
bucketMs,
|
||||
points,
|
||||
temperatures: readTemperatureHistory(query, hosts, startAt, now, bucketMs)
|
||||
};
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ import { logResultMetadata } from './log-results.js';
|
||||
import { parseHostInventory } from './request-context.js';
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { recordInstanceResources, validateInstanceResources } from './calibration.js';
|
||||
import { validateTemperatures } from '../../host-temperatures.js';
|
||||
import { recordTemperatures } from './temperatures.js';
|
||||
import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
|
||||
import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js';
|
||||
import type {
|
||||
@@ -622,6 +624,8 @@ export function validateInventory(value: unknown): HostInventory {
|
||||
throw new Error('Host inventory must be an object.');
|
||||
const inventory = value as HostInventory;
|
||||
if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.');
|
||||
if (inventory.temperatures !== undefined)
|
||||
inventory.temperatures = validateTemperatures(inventory.temperatures);
|
||||
for (const key of ['hostname', 'agentVersion', 'os', 'architecture', 'cpu', 'dockerVersion']) {
|
||||
if (
|
||||
inventory[key] !== undefined &&
|
||||
@@ -924,6 +928,7 @@ export function heartbeat(hostId: string, value: unknown): Host {
|
||||
|
||||
function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
|
||||
recordInstanceResources(hostId, inventory, timestamp);
|
||||
recordTemperatures(hostId, inventory, timestamp);
|
||||
database()
|
||||
.prepare(
|
||||
`INSERT OR REPLACE INTO ec_host_resource_samples
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
import { db } from '../db/index.js';
|
||||
import { historyTemperatures } from '../../host-temperatures.js';
|
||||
import type { HostInventory } from '../../hosts.js';
|
||||
import type { CapacityQuery, TemperatureHistorySeries } from '../../capacity.js';
|
||||
|
||||
export function migrateTemperatures(handle: Database.Database): void {
|
||||
handle.exec(`CREATE TABLE IF NOT EXISTS ec_host_temperature_samples (
|
||||
host_id TEXT NOT NULL REFERENCES ec_hosts(id), bucket TEXT NOT NULL,
|
||||
sensor_id TEXT NOT NULL, kind TEXT NOT NULL, label TEXT NOT NULL,
|
||||
celsius REAL NOT NULL CHECK(celsius BETWEEN -100 AND 200),
|
||||
PRIMARY KEY(host_id,bucket,sensor_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ec_host_temperatures_retention ON ec_host_temperature_samples(bucket);`);
|
||||
}
|
||||
|
||||
export function recordTemperatures(
|
||||
hostId: string,
|
||||
inventory: HostInventory,
|
||||
timestamp: string
|
||||
): void {
|
||||
const sensors = historyTemperatures(inventory.temperatures);
|
||||
if (!sensors.length) return;
|
||||
const bucket = new Date(Math.floor(Date.parse(timestamp) / 60_000) * 60_000).toISOString();
|
||||
const insert = db().prepare(`INSERT INTO ec_host_temperature_samples
|
||||
(host_id,bucket,sensor_id,kind,label,celsius) VALUES(?,?,?,?,?,?)
|
||||
ON CONFLICT(host_id,bucket,sensor_id) DO UPDATE SET
|
||||
celsius=MAX(celsius,excluded.celsius),kind=excluded.kind,label=excluded.label`);
|
||||
for (const sensor of sensors)
|
||||
insert.run(hostId, bucket, sensor.id, sensor.kind, sensor.label, sensor.celsius);
|
||||
}
|
||||
|
||||
/** Runs on the capacity report worker, returning at most 168 points per series. */
|
||||
export function readTemperatureHistory(
|
||||
query: CapacityQuery,
|
||||
hosts: { id: string; name: string; created_at: string }[],
|
||||
startAt: number,
|
||||
endAt: number,
|
||||
bucketMs: number
|
||||
): TemperatureHistorySeries[] {
|
||||
const count = Math.round((endAt - startAt) / bucketMs);
|
||||
const statement = db().prepare(`SELECT sensor_id,kind,label,
|
||||
MIN(?,MAX(0,CAST((CAST(strftime('%s',bucket) AS INTEGER)*1000-?) / ? AS INTEGER))) AS period,
|
||||
MAX(celsius) AS celsius
|
||||
FROM ec_host_temperature_samples WHERE host_id=? AND bucket>=? AND bucket<=?
|
||||
${query.hostId ? '' : "AND sensor_id='cpu'"}
|
||||
GROUP BY sensor_id,period ORDER BY sensor_id,period`);
|
||||
const result: TemperatureHistorySeries[] = [];
|
||||
for (const host of hosts) {
|
||||
const bySensor = new Map<string, TemperatureHistorySeries>();
|
||||
const add = (id: string, kind: TemperatureHistorySeries['kind'], label: string) => {
|
||||
const series = {
|
||||
id: `${host.id}:${id}`,
|
||||
hostId: host.id,
|
||||
kind,
|
||||
label,
|
||||
points: Array.from({ length: count }, (_, i) => ({
|
||||
at: startAt + i * bucketMs,
|
||||
celsius: null as number | null
|
||||
}))
|
||||
};
|
||||
bySensor.set(id, series);
|
||||
result.push(series);
|
||||
return series;
|
||||
};
|
||||
add('cpu', 'cpu', query.hostId ? 'CPU temperature' : host.name);
|
||||
const rows = statement.all(
|
||||
count - 1,
|
||||
startAt,
|
||||
bucketMs,
|
||||
host.id,
|
||||
new Date(
|
||||
Math.max(
|
||||
Math.floor(Date.parse(host.created_at) / 60_000) * 60_000,
|
||||
Math.floor(startAt / 60_000) * 60_000
|
||||
)
|
||||
).toISOString(),
|
||||
new Date(endAt).toISOString()
|
||||
) as { sensor_id: string; kind: string; label: string; period: number; celsius: number }[];
|
||||
for (const row of rows) {
|
||||
let series = bySensor.get(row.sensor_id);
|
||||
if (!series) {
|
||||
if (!['nvme', 'chipset'].includes(row.kind) || bySensor.size >= 17) continue;
|
||||
const kind = row.kind as 'nvme' | 'chipset';
|
||||
const number = [...bySensor.values()].filter((s) => s.kind === kind).length + 1;
|
||||
series = add(
|
||||
row.sensor_id,
|
||||
kind,
|
||||
`${kind === 'nvme' ? 'NVMe drive' : 'Chipset'} ${number}`
|
||||
);
|
||||
}
|
||||
if (series.points[row.period]) series.points[row.period].celsius = row.celsius;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -556,6 +556,14 @@ function samplePolicies(s = fleetSettings()) {
|
||||
value: iso(s.hostResourceKeepDays),
|
||||
enabled: s.hostResourceKeepDays > 0
|
||||
},
|
||||
{
|
||||
category: 'host-temperatures',
|
||||
label: 'Host temperature samples',
|
||||
table: 'ec_host_temperature_samples',
|
||||
condition: 'bucket < ?',
|
||||
value: iso(s.hostResourceKeepDays),
|
||||
enabled: s.hostResourceKeepDays > 0
|
||||
},
|
||||
{
|
||||
category: 'instance-resources',
|
||||
label: 'Instance resource samples',
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<script lang="ts">
|
||||
import type { Host } from '$lib/hosts.js';
|
||||
import { cpuTemperature, formatTemperature } from '$lib/host-temperatures.js';
|
||||
import { contextHref } from '$lib/navigation-context.js';
|
||||
import {
|
||||
hostListStatus,
|
||||
@@ -68,7 +69,13 @@
|
||||
? '—'
|
||||
: `${host.playerCount}${host.unreportedInstances ? '+' : ''}`}</td
|
||||
>
|
||||
<td class:stale>{hostCpuUsage(host.inventory.cpuUsagePercent)}</td>
|
||||
<td class:stale
|
||||
>{hostCpuUsage(host.inventory.cpuUsagePercent)}<span class="secondary"
|
||||
>CPU temp: {formatTemperature(
|
||||
cpuTemperature(host.inventory.temperatures)?.celsius
|
||||
)}</span
|
||||
></td
|
||||
>
|
||||
<td class:stale
|
||||
>{hostUsedSpace(
|
||||
host.inventory.memoryTotal,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
<script lang="ts">
|
||||
import type { Host } from '$lib/hosts.js';
|
||||
import { cpuTemperature, formatTemperature, temperatureLabel } from '$lib/host-temperatures.js';
|
||||
import { hostReadingsStale } from '$ui/host-overview.js';
|
||||
import { onMount } from 'svelte';
|
||||
let { host }: { host: Host } = $props();
|
||||
let now = $state(Date.now());
|
||||
const stale = $derived(hostReadingsStale(host, now));
|
||||
const sensors = $derived(host.inventory.temperatures ?? []);
|
||||
const cpu = $derived(cpuTemperature(sensors));
|
||||
onMount(() => {
|
||||
const clock = setInterval(() => (now = Date.now()), 15_000);
|
||||
return () => clearInterval(clock);
|
||||
});
|
||||
</script>
|
||||
|
||||
<div class:stale>
|
||||
{formatTemperature(cpu?.celsius)}{#if stale && cpu}
|
||||
(last reported){/if}
|
||||
{#if cpu}<span class="source">{temperatureLabel(cpu)}</span>{/if}
|
||||
{#if sensors.length}
|
||||
<details>
|
||||
<summary>Temperature sensors</summary>
|
||||
<ul>
|
||||
{#each sensors as sensor (sensor.id)}
|
||||
<li>
|
||||
<span
|
||||
>{temperatureLabel(sensor)}<small
|
||||
>{sensor.device} · {sensor.label} · {sensor.id.slice(-6)}</small
|
||||
></span
|
||||
><span>{formatTemperature(sensor.celsius)}</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
{#if stale}<p>Last reported readings; waiting for a fresh host heartbeat.</p>{/if}
|
||||
</details>
|
||||
{:else}<span class="source">No supported temperature sensors reported.</span>{/if}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.source,
|
||||
small {
|
||||
display: block;
|
||||
color: var(--text-secondary);
|
||||
font-size: var(--text-xs);
|
||||
margin-top: var(--space-1);
|
||||
}
|
||||
.stale {
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
details {
|
||||
margin-top: var(--space-2);
|
||||
}
|
||||
summary {
|
||||
cursor: pointer;
|
||||
font-size: var(--text-sm);
|
||||
}
|
||||
summary:focus-visible {
|
||||
outline: 2px solid var(--accent-primary);
|
||||
outline-offset: 3px;
|
||||
}
|
||||
ul {
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
margin: var(--space-3) 0;
|
||||
display: grid;
|
||||
gap: var(--space-3);
|
||||
}
|
||||
li {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-2);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
p {
|
||||
font-size: var(--text-xs);
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,456 @@
|
||||
<script lang="ts">
|
||||
import type { Snippet } from 'svelte';
|
||||
import type { CapacityHistory, TemperatureHistorySeries } from '$lib/capacity.js';
|
||||
import { formatTemperature } from '$lib/host-temperatures.js';
|
||||
import { timeFormat } from '$ui/stores/time-zone.js';
|
||||
|
||||
let {
|
||||
history,
|
||||
scope,
|
||||
controls
|
||||
}: { history: CapacityHistory; scope: string; controls?: Snippet } = $props();
|
||||
let visibility = $state<Record<string, boolean>>({});
|
||||
let selectedAt = $state<number | null>(null);
|
||||
const series = $derived(history.temperatures);
|
||||
const visible = $derived(series.filter((s) => visibility[s.id] ?? s.kind === 'cpu'));
|
||||
const values = $derived(
|
||||
series.flatMap((s) => s.points.flatMap((p) => (p.celsius === null ? [] : [p.celsius])))
|
||||
);
|
||||
const minimum = $derived(
|
||||
values.length
|
||||
? Math.floor((values.reduce((a, b) => Math.min(a, b), Infinity) - 5) / 10) * 10
|
||||
: 0
|
||||
);
|
||||
const maximum = $derived(
|
||||
values.length
|
||||
? Math.ceil((values.reduce((a, b) => Math.max(a, b), -Infinity) + 5) / 10) * 10
|
||||
: 100
|
||||
);
|
||||
const times = $derived(history.points.map((p) => p.at));
|
||||
const measured = $derived(visible.some((s) => s.points.some((p) => p.celsius !== null)));
|
||||
const selectedIndex = $derived.by(() => {
|
||||
if (selectedAt !== null)
|
||||
return times.reduce(
|
||||
(best, at, i) =>
|
||||
Math.abs(at - selectedAt!) < Math.abs(times[best] - selectedAt!) ? i : best,
|
||||
0
|
||||
);
|
||||
for (let i = times.length - 1; i >= 0; i--)
|
||||
if (visible.some((s) => s.points[i]?.celsius != null)) return i;
|
||||
return Math.max(0, times.length - 1);
|
||||
});
|
||||
const selectedTime = $derived(times[selectedIndex]);
|
||||
const readout = $derived(
|
||||
visible
|
||||
.map((s) => `${s.label}: ${formatTemperature(s.points[selectedIndex]?.celsius)}`)
|
||||
.join('. ')
|
||||
);
|
||||
const ticks = [0, 0.25, 0.5, 0.75, 1];
|
||||
function color(s: TemperatureHistorySeries) {
|
||||
return `var(--chart-series-${(series.indexOf(s) % 6) + 1})`;
|
||||
}
|
||||
function dash(s: TemperatureHistorySeries) {
|
||||
return ['', '7 4', '2 4'][Math.floor(series.indexOf(s) / 6) % 3];
|
||||
}
|
||||
function x(at: number) {
|
||||
return (
|
||||
8 +
|
||||
984 *
|
||||
Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
1,
|
||||
(at - history.startAt) / Math.max(1, history.endAt - history.startAt)
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
function y(celsius: number) {
|
||||
return 230 - (220 * (celsius - minimum)) / (maximum - minimum);
|
||||
}
|
||||
function segments(s: TemperatureHistorySeries) {
|
||||
const result: { x: number; y: number }[][] = [];
|
||||
let segment: { x: number; y: number }[] = [];
|
||||
for (const point of s.points) {
|
||||
if (point.celsius === null) {
|
||||
if (segment.length) result.push(segment);
|
||||
segment = [];
|
||||
} else segment.push({ x: x(point.at), y: y(point.celsius) });
|
||||
}
|
||||
if (segment.length) result.push(segment);
|
||||
return result;
|
||||
}
|
||||
function inspect(event: PointerEvent) {
|
||||
if (!times.length || (event.type === 'pointermove' && event.pointerType === 'touch'))
|
||||
return;
|
||||
const rect = (event.currentTarget as HTMLElement).getBoundingClientRect();
|
||||
selectedAt =
|
||||
history.startAt +
|
||||
Math.max(
|
||||
0,
|
||||
Math.min(1, (((event.clientX - rect.left) / rect.width) * 1000 - 8) / 984)
|
||||
) *
|
||||
(history.endAt - history.startAt);
|
||||
}
|
||||
function move(event: KeyboardEvent) {
|
||||
if (!times.length || !['ArrowLeft', 'ArrowRight', 'Home', 'End'].includes(event.key))
|
||||
return;
|
||||
event.preventDefault();
|
||||
const next =
|
||||
event.key === 'Home'
|
||||
? 0
|
||||
: event.key === 'End'
|
||||
? times.length - 1
|
||||
: Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
times.length - 1,
|
||||
selectedIndex + (event.key === 'ArrowLeft' ? -1 : 1)
|
||||
)
|
||||
);
|
||||
selectedAt = times[next];
|
||||
}
|
||||
</script>
|
||||
|
||||
<section class="panel" id="temperatures" aria-labelledby="temperature-history-title">
|
||||
<div class="heading">
|
||||
<div>
|
||||
<h2 id="temperature-history-title">Temperature history</h2>
|
||||
<p>{scope} · Peak temperature per period</p>
|
||||
</div>
|
||||
{@render controls?.()}
|
||||
</div>
|
||||
<div class="legend" role="group" aria-label="Temperature series">
|
||||
{#each series as sensor (sensor.id)}
|
||||
<button
|
||||
type="button"
|
||||
aria-pressed={visibility[sensor.id] ?? sensor.kind === 'cpu'}
|
||||
onclick={() =>
|
||||
(visibility[sensor.id] = !(visibility[sensor.id] ?? sensor.kind === 'cpu'))}
|
||||
>
|
||||
<svg viewBox="0 0 24 8" aria-hidden="true"
|
||||
><line
|
||||
x1="0"
|
||||
x2="24"
|
||||
y1="4"
|
||||
y2="4"
|
||||
stroke={color(sensor)}
|
||||
stroke-dasharray={dash(sensor)}
|
||||
/></svg
|
||||
>
|
||||
{sensor.label}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
<div class="axis-title">Temperature (°C)</div>
|
||||
<div class="chart-grid">
|
||||
<div class="y-axis" aria-hidden="true">
|
||||
{#each ticks as tick}<span style:top={`${(1 - tick) * 100}%`}
|
||||
>{minimum + (maximum - minimum) * tick}</span
|
||||
>{/each}
|
||||
</div>
|
||||
<div
|
||||
class="plot"
|
||||
role="slider"
|
||||
tabindex="0"
|
||||
aria-label="Temperature history. Use left and right arrows to inspect a period."
|
||||
aria-valuemin="0"
|
||||
aria-valuemax={Math.max(0, times.length - 1)}
|
||||
aria-valuenow={selectedIndex}
|
||||
aria-valuetext={selectedTime === undefined
|
||||
? 'No temperature history'
|
||||
: `${$timeFormat.period(selectedTime)}. ${readout || 'No sensors selected'}`}
|
||||
onpointermove={inspect}
|
||||
onpointerdown={inspect}
|
||||
onkeydown={move}
|
||||
>
|
||||
<svg viewBox="0 0 1000 240" preserveAspectRatio="none" aria-hidden="true">
|
||||
{#each ticks as tick}<line
|
||||
class="grid-line"
|
||||
x1="0"
|
||||
x2="1000"
|
||||
y1={y(minimum + (maximum - minimum) * tick)}
|
||||
y2={y(minimum + (maximum - minimum) * tick)}
|
||||
/>{/each}
|
||||
{#each visible as sensor (sensor.id)}
|
||||
<g data-series={sensor.id}>
|
||||
{#each segments(sensor) as segment}
|
||||
<path
|
||||
class="trend"
|
||||
stroke={color(sensor)}
|
||||
stroke-dasharray={dash(sensor)}
|
||||
d={segment
|
||||
.map((point, i) => `${i ? 'L' : 'M'}${point.x},${point.y}`)
|
||||
.join(' ')}
|
||||
/>
|
||||
{#if segment.length === 1}<circle
|
||||
cx={segment[0].x}
|
||||
cy={segment[0].y}
|
||||
r="3"
|
||||
fill={color(sensor)}
|
||||
/>{/if}
|
||||
{/each}
|
||||
{#if sensor.points[selectedIndex]?.celsius != null}<circle
|
||||
class="selected-point"
|
||||
cx={x(selectedTime)}
|
||||
cy={y(sensor.points[selectedIndex].celsius!)}
|
||||
r="4"
|
||||
fill={color(sensor)}
|
||||
/>{/if}
|
||||
</g>
|
||||
{/each}
|
||||
{#if selectedTime !== undefined && measured}<line
|
||||
class="cursor"
|
||||
x1={x(selectedTime)}
|
||||
x2={x(selectedTime)}
|
||||
y1="5"
|
||||
y2="235"
|
||||
/>{/if}
|
||||
</svg>
|
||||
{#if !measured}<div class="empty-chart">
|
||||
<strong
|
||||
>{visible.length || !series.length
|
||||
? 'No temperature readings in this period'
|
||||
: 'Select a sensor to show history'}</strong
|
||||
>
|
||||
{#if visible.length || !series.length}<span
|
||||
>Readings appear when an agent reports supported temperature sensors.</span
|
||||
>{/if}
|
||||
</div>{/if}
|
||||
</div>
|
||||
<div class="x-axis" aria-hidden="true">
|
||||
<span>{$timeFormat.period(history.startAt)}</span><span
|
||||
>{$timeFormat.period(history.endAt)}</span
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
{#if selectedTime !== undefined && visible.length}
|
||||
<div class="readout" aria-label="Selected temperature period">
|
||||
<span>{$timeFormat.period(selectedTime)}</span>
|
||||
<dl>
|
||||
{#each visible as sensor (sensor.id)}<div>
|
||||
<dt>{sensor.label}</dt>
|
||||
<dd>{formatTemperature(sensor.points[selectedIndex]?.celsius)}</dd>
|
||||
</div>{/each}
|
||||
</dl>
|
||||
</div>
|
||||
{/if}
|
||||
<details>
|
||||
<summary>About this chart</summary>
|
||||
<p>
|
||||
Each CPU line represents one host: its hottest reported package or physical die, falling
|
||||
back to its hottest core. Select a host to compare its CPU, NVMe drives and chipset.
|
||||
Per-minute peaks are retained; longer periods show their highest reading. Gaps mean no
|
||||
measurement was received in that period. Temperatures include all workloads and are
|
||||
unaffected by the game filter. Colors identify series; they do not indicate safe or
|
||||
unsafe temperatures.
|
||||
</p>
|
||||
</details>
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.panel {
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-lg);
|
||||
padding: var(--space-5);
|
||||
min-width: 0;
|
||||
}
|
||||
h2 {
|
||||
margin: 0;
|
||||
font-size: var(--text-lg);
|
||||
}
|
||||
.heading {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-4);
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.heading p {
|
||||
color: var(--text-secondary);
|
||||
font-size: var(--text-sm);
|
||||
margin: var(--space-2) 0 0;
|
||||
}
|
||||
.legend {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-2) var(--space-5);
|
||||
margin: var(--space-4) 0;
|
||||
}
|
||||
.legend button {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
background: none;
|
||||
border: 0;
|
||||
padding: var(--space-2) 0;
|
||||
color: var(--text-primary);
|
||||
font: inherit;
|
||||
font-size: var(--text-sm);
|
||||
cursor: pointer;
|
||||
overflow-wrap: anywhere;
|
||||
max-width: 100%;
|
||||
}
|
||||
.legend button[aria-pressed='false'] {
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
.legend button[aria-pressed='false'] svg {
|
||||
opacity: 0.35;
|
||||
}
|
||||
.legend svg {
|
||||
width: 24px;
|
||||
height: 8px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.legend line {
|
||||
stroke-width: 3;
|
||||
}
|
||||
button:focus-visible,
|
||||
.plot:focus-visible,
|
||||
summary:focus-visible {
|
||||
outline: 2px solid var(--accent-primary);
|
||||
outline-offset: 3px;
|
||||
}
|
||||
.axis-title,
|
||||
.x-axis,
|
||||
.y-axis {
|
||||
color: var(--text-tertiary);
|
||||
font-size: var(--text-xs);
|
||||
}
|
||||
.chart-grid {
|
||||
display: grid;
|
||||
grid-template-columns: 42px minmax(0, 1fr);
|
||||
gap: 10px;
|
||||
margin-top: 14px;
|
||||
}
|
||||
.y-axis {
|
||||
position: relative;
|
||||
height: 220px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
.y-axis span {
|
||||
position: absolute;
|
||||
right: 0;
|
||||
transform: translateY(-50%);
|
||||
}
|
||||
.plot {
|
||||
position: relative;
|
||||
height: 240px;
|
||||
cursor: crosshair;
|
||||
touch-action: pan-y;
|
||||
}
|
||||
.plot > svg {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
overflow: visible;
|
||||
}
|
||||
.plot path,
|
||||
.plot line {
|
||||
vector-effect: non-scaling-stroke;
|
||||
}
|
||||
.grid-line {
|
||||
stroke: var(--border-subtle);
|
||||
stroke-width: 1;
|
||||
}
|
||||
.trend {
|
||||
fill: none;
|
||||
stroke-width: 2;
|
||||
stroke-linejoin: round;
|
||||
stroke-linecap: round;
|
||||
}
|
||||
.selected-point {
|
||||
stroke: var(--surface);
|
||||
stroke-width: 2;
|
||||
}
|
||||
.cursor {
|
||||
stroke: var(--text-tertiary);
|
||||
stroke-width: 1;
|
||||
stroke-dasharray: 4 5;
|
||||
}
|
||||
.x-axis {
|
||||
grid-column: 2;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-4);
|
||||
}
|
||||
.x-axis span:last-child {
|
||||
text-align: right;
|
||||
}
|
||||
.empty-chart {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex-direction: column;
|
||||
gap: var(--space-2);
|
||||
padding: var(--space-4);
|
||||
text-align: center;
|
||||
pointer-events: none;
|
||||
}
|
||||
.empty-chart strong {
|
||||
font-size: var(--text-sm);
|
||||
font-weight: var(--weight-medium);
|
||||
}
|
||||
.empty-chart span {
|
||||
font-size: var(--text-xs);
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
.readout {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-4);
|
||||
font-size: var(--text-xs);
|
||||
margin: var(--space-4) 0;
|
||||
}
|
||||
.readout > span,
|
||||
dt {
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
dl {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-3) var(--space-5);
|
||||
margin: 0;
|
||||
}
|
||||
dl > div {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
min-width: 0;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
dt {
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
dd {
|
||||
margin: 0;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
summary {
|
||||
cursor: pointer;
|
||||
font-size: var(--text-xs);
|
||||
color: var(--text-secondary);
|
||||
padding: var(--space-2) 0;
|
||||
}
|
||||
details p {
|
||||
color: var(--text-secondary);
|
||||
font-size: var(--text-xs);
|
||||
line-height: 1.65;
|
||||
}
|
||||
@media (max-width: 640px) {
|
||||
.panel {
|
||||
padding: var(--space-4);
|
||||
}
|
||||
.chart-grid {
|
||||
grid-template-columns: 32px minmax(0, 1fr);
|
||||
gap: 8px;
|
||||
}
|
||||
}
|
||||
@media (pointer: coarse) {
|
||||
.legend button {
|
||||
min-height: 44px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -78,6 +78,12 @@
|
||||
|
||||
/* Metrics: unknown uses a pattern as well as color. */
|
||||
--chart-active: var(--accent-primary);
|
||||
--chart-series-1: var(--accent-primary);
|
||||
--chart-series-2: #b99af7;
|
||||
--chart-series-3: #69c6ba;
|
||||
--chart-series-4: #dfb76c;
|
||||
--chart-series-5: #df91b6;
|
||||
--chart-series-6: #bbc7db;
|
||||
--chart-idle: #2b3442;
|
||||
--chart-unknown-base: #605b64;
|
||||
--chart-unknown-stripe: #a7a0ad;
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
import Select from '$ui/components/primitives/Select.svelte';
|
||||
import CapacityHosts from '$ui/components/capacity/CapacityHosts.svelte';
|
||||
import CapacityChart from '$ui/components/capacity/CapacityChart.svelte';
|
||||
import TemperatureChart from '$ui/components/capacity/TemperatureChart.svelte';
|
||||
import RebalancePanel from '$ui/components/capacity/RebalancePanel.svelte';
|
||||
import Freshness from '$ui/components/Freshness.svelte';
|
||||
import { plural } from '$ui/plural.js';
|
||||
@@ -319,8 +320,7 @@
|
||||
: 'Waiting for host reports.'} Refreshes every 30 seconds while visible.
|
||||
</p>
|
||||
</details>
|
||||
<CapacityChart history={report.history} scope={scopeName}>
|
||||
{#snippet controls()}<nav class="ranges" aria-label="History range">
|
||||
{#snippet historyRanges()}<nav class="ranges" aria-label="History range">
|
||||
{#each ranges as range}<a
|
||||
href={scopeUrl({ range: range.value })}
|
||||
aria-current={report.query.range === range.value ? 'page' : undefined}
|
||||
@@ -328,7 +328,12 @@
|
||||
data-sveltekit-noscroll>{range.label}</a
|
||||
>{/each}
|
||||
</nav>{/snippet}
|
||||
</CapacityChart>
|
||||
<CapacityChart history={report.history} scope={scopeName} controls={historyRanges} />
|
||||
{#key report.query.hostId}<TemperatureChart
|
||||
history={report.history}
|
||||
scope={scopeName}
|
||||
controls={historyRanges}
|
||||
/>{/key}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
|
||||
import HostPreparation from '$ui/components/HostPreparation.svelte';
|
||||
import CapacitySamples from '$ui/components/CapacitySamples.svelte';
|
||||
import HostTemperatures from '$ui/components/HostTemperatures.svelte';
|
||||
import ServerCard from '$ui/components/ServerCard.svelte';
|
||||
import type { ServerObservation } from '$lib/server-observation.js';
|
||||
import { syncServerTime } from '$ui/stores/servers.js';
|
||||
@@ -300,6 +301,8 @@
|
||||
: `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined}
|
||||
(last reported){/if}
|
||||
</dd>
|
||||
<dt>CPU temperature</dt>
|
||||
<dd><HostTemperatures {host} /></dd>
|
||||
<dt>Memory available / total</dt>
|
||||
<dd>
|
||||
{bytes(host.inventory.memoryAvailable)} / {bytes(
|
||||
|
||||
@@ -35,7 +35,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.9.3"
|
||||
VERSION = "1.10.0"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
MANIFEST_MAX_ENTRIES = 10000
|
||||
@@ -63,6 +63,7 @@ UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-ch
|
||||
UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
|
||||
UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
|
||||
AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
|
||||
HWMON_ROOT = Path("/sys/class/hwmon")
|
||||
ROOT_MARKER = ".electric-cherry.json"
|
||||
|
||||
|
||||
@@ -1308,6 +1309,59 @@ class Agent:
|
||||
return None
|
||||
return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
|
||||
|
||||
def temperatures(self):
|
||||
"""Best-effort, read-only samples from known millidegree-Celsius drivers."""
|
||||
samples = []
|
||||
try:
|
||||
chips = sorted(HWMON_ROOT.glob("hwmon*"))[:64]
|
||||
except OSError:
|
||||
return samples
|
||||
for chip in chips:
|
||||
try:
|
||||
device = (chip / "name").read_text().strip()
|
||||
if device not in ("coretemp", "k10temp", "nvme") and not device.startswith("pch_"):
|
||||
continue
|
||||
# hwmon numbers can change on reboot. The physical sysfs path
|
||||
# and channel identify separate CPU sockets and NVMe devices.
|
||||
identity = re.sub(r"/(?:hwmon/)?hwmon\d+$", "", str(chip.resolve()))
|
||||
inputs = sorted(chip.glob("temp[0-9]*_input"))[:128]
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
for source in inputs:
|
||||
try:
|
||||
channel = source.name.removesuffix("_input")
|
||||
label_path = chip / (channel + "_label")
|
||||
label = label_path.read_text().strip() if label_path.exists() else channel
|
||||
if not label or len(label) > 120:
|
||||
continue
|
||||
fault = chip / (channel + "_fault")
|
||||
enabled = chip / (channel + "_enable")
|
||||
if (fault.exists() and int(fault.read_text()) != 0) or (enabled.exists() and int(enabled.read_text()) == 0):
|
||||
continue
|
||||
if device == "coretemp":
|
||||
kind = "cpu-package" if label.startswith("Package id ") else "cpu-core" if label.startswith("Core ") else None
|
||||
elif device == "k10temp":
|
||||
# Tctl can contain a fan-control offset. Never present
|
||||
# it as the physical die temperature or a fleet metric.
|
||||
kind = "cpu-die" if label == "Tdie" or re.fullmatch(r"Tccd\d+", label) else None
|
||||
else:
|
||||
kind = "nvme" if device == "nvme" else "chipset"
|
||||
if kind is None:
|
||||
continue
|
||||
celsius = int(source.read_text()) / 1000
|
||||
if not -100 <= celsius <= 200:
|
||||
continue
|
||||
sensor_id = hashlib.sha256((identity + "/" + channel).encode()).hexdigest()[:24]
|
||||
samples.append({"id": sensor_id, "device": device, "label": label,
|
||||
"kind": kind, "celsius": round(celsius, 1)})
|
||||
if len(samples) >= 128:
|
||||
return samples
|
||||
except (OSError, ValueError):
|
||||
# Read permission alone does not imply data is available
|
||||
# (e.g. ENODATA), and a device can disappear during a read.
|
||||
continue
|
||||
return samples
|
||||
|
||||
def instance_resources(self, observations=None):
|
||||
"""One bounded Docker stats batch; unavailable samples are never zero load."""
|
||||
ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true",
|
||||
@@ -1392,6 +1446,7 @@ class Agent:
|
||||
|
||||
def inventory(self):
|
||||
cpu_usage = self.cpu_usage_percent()
|
||||
temperatures = self.temperatures()
|
||||
usage = shutil.disk_usage(self.root)
|
||||
memory = {}
|
||||
try:
|
||||
@@ -1516,9 +1571,9 @@ class Agent:
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
update = self.update_inventory()
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
capabilities = ["host.temperatures", "library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
plutonium_cache = self.plutonium_cache_inventory()
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "temperatures": temperatures, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
||||
health = self.inspect(instance_id, generation)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"version": "1.9.3",
|
||||
"version": "1.10.0",
|
||||
"protocolVersion": 1,
|
||||
"platform": "debian-amd64",
|
||||
"sha256": "a0baaab0f0a60e848300e63f31eeefff6293d8b749b5ca9397c36e8e75ff4a93",
|
||||
"sha256": "1a110c92ae875ee7cd060ac7de82260b63d776d2a865e57356d629e0a99573c7",
|
||||
"updaterProtocolVersion": 2,
|
||||
"updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb",
|
||||
"installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231"
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
-----BEGIN PGP SIGNATURE-----
|
||||
|
||||
iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCas2mehsUgAAAAAAEAA5t
|
||||
YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfpvyAD/THg0KsH9kDSCF/I2vdOV
|
||||
h2BBe9r1JWk1F0mFy2njfvAA/R9mMAMdJRkFIxMNgkIl17nQrUq4TpXQsKSSujXZ
|
||||
u5oK
|
||||
=GspF
|
||||
-----END PGP SIGNATURE-----
|
||||
@@ -0,0 +1,110 @@
|
||||
import { afterEach, expect, it } from 'vitest';
|
||||
import { flushSync, mount, unmount } from 'svelte';
|
||||
import TemperatureChart from '../../src/lib/ui/components/capacity/TemperatureChart.svelte';
|
||||
import type { CapacityHistory } from '../../src/lib/capacity.js';
|
||||
|
||||
let component: ReturnType<typeof mount> | undefined;
|
||||
afterEach(async () => {
|
||||
if (component) await unmount(component);
|
||||
component = undefined;
|
||||
document.body.replaceChildren();
|
||||
});
|
||||
function history(): CapacityHistory {
|
||||
return {
|
||||
range: '1h',
|
||||
startAt: 1000,
|
||||
endAt: 4000,
|
||||
bucketMs: 1000,
|
||||
points: [1000, 2000, 3000].map((at) => ({
|
||||
at,
|
||||
cpuCoresAvailable: null,
|
||||
memoryBytesAvailable: null,
|
||||
reportingHosts: 0,
|
||||
expectedHosts: 1
|
||||
})),
|
||||
temperatures: [
|
||||
{
|
||||
id: 'host:cpu',
|
||||
hostId: 'host',
|
||||
label: 'CPU temperature',
|
||||
kind: 'cpu',
|
||||
points: [65, null, 67].map((celsius, i) => ({ at: 1000 + i * 1000, celsius }))
|
||||
},
|
||||
{
|
||||
id: 'host:drive',
|
||||
hostId: 'host',
|
||||
label: 'NVMe drive 1',
|
||||
kind: 'nvme',
|
||||
points: [49, 50.9, 52].map((celsius, i) => ({ at: 1000 + i * 1000, celsius }))
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
function show(report = history()) {
|
||||
component = mount(TemperatureChart, {
|
||||
target: document.body,
|
||||
props: { history: report, scope: 'ec-2' }
|
||||
});
|
||||
flushSync();
|
||||
}
|
||||
|
||||
it('defaults to CPU and toggles device lines and readouts together', () => {
|
||||
show();
|
||||
const buttons = document.querySelectorAll<HTMLButtonElement>('.legend button');
|
||||
expect(buttons[0].getAttribute('aria-pressed')).toBe('true');
|
||||
expect(buttons[1].getAttribute('aria-pressed')).toBe('false');
|
||||
expect(document.querySelectorAll('[data-series]')).toHaveLength(1);
|
||||
// A missing middle bucket splits the CPU line into two isolated segments.
|
||||
expect(document.querySelectorAll('[data-series="host:cpu"] path')).toHaveLength(2);
|
||||
buttons[1].click();
|
||||
flushSync();
|
||||
expect(document.querySelectorAll('[data-series]')).toHaveLength(2);
|
||||
expect(document.querySelector('.readout')?.textContent).toContain('52 °C');
|
||||
buttons[0].click();
|
||||
buttons[1].click();
|
||||
flushSync();
|
||||
expect(document.querySelectorAll('[data-series]')).toHaveLength(0);
|
||||
expect(document.body.textContent).toContain('Select a sensor to show history');
|
||||
expect(document.querySelector('.readout')).toBeNull();
|
||||
});
|
||||
|
||||
it('supports keyboard and pointer inspection without turning missing readings into zero', () => {
|
||||
show();
|
||||
const plot = document.querySelector<HTMLElement>('[role="slider"]')!;
|
||||
expect(plot.getAttribute('aria-valuenow')).toBe('2');
|
||||
plot.dispatchEvent(new KeyboardEvent('keydown', { key: 'Home', bubbles: true }));
|
||||
flushSync();
|
||||
expect(plot.getAttribute('aria-valuetext')).toContain('65 °C');
|
||||
plot.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowRight', bubbles: true }));
|
||||
flushSync();
|
||||
expect(plot.getAttribute('aria-valuetext')).toContain('Unavailable');
|
||||
expect(document.querySelector('.readout')?.textContent).not.toContain('0 °C');
|
||||
Object.defineProperty(plot, 'getBoundingClientRect', {
|
||||
value: () => ({ left: 0, width: 1000 })
|
||||
});
|
||||
plot.dispatchEvent(
|
||||
new PointerEvent('pointerdown', { clientX: 8, pointerType: 'touch', bubbles: true })
|
||||
);
|
||||
flushSync();
|
||||
expect(plot.getAttribute('aria-valuenow')).toBe('0');
|
||||
expect(document.querySelector('.readout')?.textContent).toContain('65 °C');
|
||||
});
|
||||
|
||||
it('shows unsupported hosts as unavailable', () => {
|
||||
const report = history();
|
||||
report.temperatures[0].points = report.temperatures[0].points.map((point) => ({
|
||||
...point,
|
||||
celsius: null
|
||||
}));
|
||||
show(report);
|
||||
expect(document.body.textContent).toContain('No temperature readings in this period');
|
||||
expect(document.body.textContent).toContain('Unavailable');
|
||||
});
|
||||
|
||||
it('plots a genuine zero-degree reading', () => {
|
||||
const report = history();
|
||||
report.temperatures[0].points[2].celsius = 0;
|
||||
show(report);
|
||||
expect(document.querySelector('.readout')?.textContent).toContain('0 °C');
|
||||
expect(document.querySelector('.empty-chart')).toBeNull();
|
||||
});
|
||||
@@ -34,6 +34,101 @@ async function sample(id: string, at: number, cpu = 25, memory = 4 * GiB, cores
|
||||
}
|
||||
|
||||
describe('capacity history', () => {
|
||||
it('keeps separate CPU series, device drill-down, real zero and missing temperature gaps', async () => {
|
||||
await host('one');
|
||||
await host('two');
|
||||
await host('unsupported');
|
||||
await host('removed');
|
||||
const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
|
||||
const cpu = (celsius: number) => ({
|
||||
id: 'package',
|
||||
kind: 'cpu-package' as const,
|
||||
device: 'coretemp',
|
||||
label: 'Package id 0',
|
||||
celsius
|
||||
});
|
||||
const drive = {
|
||||
id: 'drive',
|
||||
kind: 'nvme' as const,
|
||||
device: 'nvme',
|
||||
label: 'Composite',
|
||||
celsius: 50.9
|
||||
};
|
||||
recordTemperatures(
|
||||
'one',
|
||||
{ temperatures: [cpu(65), drive] },
|
||||
new Date(epoch - 55_000).toISOString()
|
||||
);
|
||||
recordTemperatures(
|
||||
'one',
|
||||
{ temperatures: [cpu(75)] },
|
||||
new Date(epoch - 30_000).toISOString()
|
||||
);
|
||||
recordTemperatures(
|
||||
'one',
|
||||
{ temperatures: [cpu(0)] },
|
||||
new Date(epoch - 180_000).toISOString()
|
||||
);
|
||||
recordTemperatures(
|
||||
'two',
|
||||
{ temperatures: [cpu(60)] },
|
||||
new Date(epoch - 40_000).toISOString()
|
||||
);
|
||||
recordTemperatures(
|
||||
'removed',
|
||||
{ temperatures: [cpu(99)] },
|
||||
new Date(epoch - 40_000).toISOString()
|
||||
);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db()
|
||||
.prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?')
|
||||
.run(new Date(epoch).toISOString(), 'removed');
|
||||
const { readCapacityHistory } =
|
||||
await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const fleet = readCapacityHistory(query, epoch).temperatures;
|
||||
expect(fleet.map((s) => s.label)).toEqual(['one', 'two', 'unsupported']);
|
||||
expect(fleet[0].points.slice(-3).map((p) => p.celsius)).toEqual([0, null, 75]);
|
||||
expect(fleet[1].points.at(-1)?.celsius).toBe(60);
|
||||
expect(fleet[2].points.every((p) => p.celsius === null)).toBe(true);
|
||||
const selected = readCapacityHistory(
|
||||
{ ...query, hostId: 'one', game: 't5' },
|
||||
epoch
|
||||
).temperatures;
|
||||
expect(selected.map((s) => s.kind)).toEqual(['cpu', 'nvme']);
|
||||
expect(selected[1].points.at(-1)?.celsius).toBe(50.9);
|
||||
expect(
|
||||
readCapacityHistory({ ...query, hostId: 'one', game: 't6' }, epoch).temperatures
|
||||
).toEqual(selected);
|
||||
expect(readCapacityHistory({ ...query, hostId: 'missing' }, epoch).temperatures).toEqual(
|
||||
[]
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves temperature peaks for long ranges and includes a host enrolled during the current minute', async () => {
|
||||
await host('new', epoch - 20_000);
|
||||
const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
|
||||
recordTemperatures(
|
||||
'new',
|
||||
{
|
||||
temperatures: [
|
||||
{
|
||||
id: 'pkg',
|
||||
device: 'coretemp',
|
||||
kind: 'cpu-package',
|
||||
label: 'Package id 0',
|
||||
celsius: 65
|
||||
}
|
||||
]
|
||||
},
|
||||
new Date(epoch - 10_000).toISOString()
|
||||
);
|
||||
const { readCapacityHistory } =
|
||||
await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
expect(readCapacityHistory(query, epoch).temperatures[0].points.at(-1)?.celsius).toBe(65);
|
||||
const long = readCapacityHistory({ ...query, range: '7d' }, epoch).temperatures[0];
|
||||
expect(long.points).toHaveLength(168);
|
||||
expect(long.points.at(-1)?.celsius).toBe(65);
|
||||
});
|
||||
it('averages each host before summing fleet headroom and exposes missing observations as gaps', async () => {
|
||||
await host('one');
|
||||
await host('two');
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import {
|
||||
cpuTemperature,
|
||||
historyTemperatures,
|
||||
validateTemperatures,
|
||||
type HostTemperatureSensor
|
||||
} from '../../src/lib/host-temperatures.js';
|
||||
|
||||
const sensor = (changes: Partial<HostTemperatureSensor> = {}): HostTemperatureSensor => ({
|
||||
id: 'package',
|
||||
device: 'coretemp',
|
||||
label: 'Package id 0',
|
||||
kind: 'cpu-package',
|
||||
celsius: 65,
|
||||
...changes
|
||||
});
|
||||
|
||||
describe('temperature readings', () => {
|
||||
it('keeps valid sensors while dropping malformed, duplicate and out-of-range entries', () => {
|
||||
const good = sensor();
|
||||
const readings = [
|
||||
good,
|
||||
good,
|
||||
null,
|
||||
'bad',
|
||||
...[NaN, Infinity, -Infinity, null, '65', true, -101, 201].map((celsius) => ({
|
||||
...good,
|
||||
id: 'invalid',
|
||||
celsius
|
||||
})),
|
||||
{ ...good, id: 'x'.repeat(101) },
|
||||
{ ...good, label: '' },
|
||||
{ ...good, kind: 'cpu-control' },
|
||||
sensor({ id: 'zero', celsius: 0 })
|
||||
];
|
||||
expect(validateTemperatures(readings)).toEqual([good, sensor({ id: 'zero', celsius: 0 })]);
|
||||
expect(validateTemperatures({})).toEqual([]);
|
||||
expect(validateTemperatures(undefined)).toEqual([]);
|
||||
});
|
||||
it('chooses the hottest package and uses die/core readings only as fallbacks', () => {
|
||||
const packages = [sensor(), sensor({ id: 'socket2', label: 'Package id 1', celsius: 72 })];
|
||||
const core = sensor({ id: 'core', label: 'Core 0', kind: 'cpu-core', celsius: 80 });
|
||||
const die = sensor({ id: 'die', label: 'Tdie', kind: 'cpu-die', celsius: 64 });
|
||||
expect(cpuTemperature([...packages, core])?.celsius).toBe(72);
|
||||
expect(cpuTemperature([die, core])?.celsius).toBe(64);
|
||||
expect(cpuTemperature([core])?.celsius).toBe(80);
|
||||
expect(cpuTemperature()).toBeNull();
|
||||
const drive = sensor({
|
||||
id: 'drive',
|
||||
kind: 'nvme',
|
||||
device: 'nvme',
|
||||
label: 'Composite',
|
||||
celsius: 50.9
|
||||
});
|
||||
expect(
|
||||
historyTemperatures([
|
||||
...packages,
|
||||
core,
|
||||
drive,
|
||||
{ ...drive, id: 'internal', label: 'Sensor 1' }
|
||||
])
|
||||
).toEqual([{ ...packages[1], id: 'cpu', label: 'CPU temperature' }, drive]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('temperature persistence', () => {
|
||||
let fixture: Fixture;
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture([]);
|
||||
});
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await fixture.cleanup();
|
||||
});
|
||||
|
||||
it('accepts older agents and faulty temperature payloads without losing host metrics', async () => {
|
||||
const { createHost, heartbeat, validateInventory } =
|
||||
await import('../../src/lib/server/hosts/index.js');
|
||||
const host = createHost({ name: 'ec-2' });
|
||||
expect(validateInventory({ cpuUsagePercent: 25 }).temperatures).toBeUndefined();
|
||||
expect(
|
||||
heartbeat(host.id, {
|
||||
cpuUsagePercent: 25,
|
||||
temperatures: [sensor(), { celsius: 'bad' }]
|
||||
}).inventory
|
||||
).toMatchObject({ cpuUsagePercent: 25, temperatures: [sensor()] });
|
||||
expect(
|
||||
heartbeat(host.id, { cpuUsagePercent: 30, temperatures: [] }).inventory.temperatures
|
||||
).toEqual([]);
|
||||
expect(
|
||||
heartbeat(host.id, { cpuUsagePercent: 35, temperatures: 'bad' }).inventory.temperatures
|
||||
).toEqual([]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(
|
||||
db()
|
||||
.prepare('SELECT celsius FROM ec_host_temperature_samples WHERE host_id=?')
|
||||
.all(host.id)
|
||||
).toEqual([{ celsius: 65 }]);
|
||||
});
|
||||
|
||||
it('retains one peak per minute and applies host history retention, including offline hosts', async () => {
|
||||
const { createHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { fleetSettings, saveFleetSettings } =
|
||||
await import('../../src/lib/server/settings.js');
|
||||
const { pruneSampleHistory } = await import('../../src/lib/server/retention.js');
|
||||
const host = createHost({ name: 'ec-2' });
|
||||
const minute = Math.floor((Date.now() - 3 * 86400_000) / 60_000) * 60_000;
|
||||
for (const [i, celsius] of [65, 78, 67].entries())
|
||||
recordTemperatures(
|
||||
host.id,
|
||||
{ temperatures: [sensor({ celsius })] },
|
||||
new Date(minute + i * 15_000).toISOString()
|
||||
);
|
||||
expect(db().prepare('SELECT celsius FROM ec_host_temperature_samples').all()).toEqual([
|
||||
{ celsius: 78 }
|
||||
]);
|
||||
saveFleetSettings({ ...fleetSettings(), hostResourceKeepDays: 0 });
|
||||
pruneSampleHistory();
|
||||
expect(db().prepare('SELECT COUNT(*) AS n FROM ec_host_temperature_samples').get()).toEqual(
|
||||
{ n: 1 }
|
||||
);
|
||||
recordTemperatures(host.id, { temperatures: [sensor()] }, new Date().toISOString());
|
||||
saveFleetSettings({ ...fleetSettings(), hostResourceKeepDays: 1 });
|
||||
pruneSampleHistory();
|
||||
expect(db().prepare('SELECT celsius FROM ec_host_temperature_samples').all()).toEqual([
|
||||
{ celsius: 65 }
|
||||
]);
|
||||
});
|
||||
|
||||
it('upgrades an existing database without rewriting its resource samples', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { createHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = createHost({ name: 'old' });
|
||||
db()
|
||||
.prepare(
|
||||
'INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_usage_percent) VALUES(?,?,?)'
|
||||
)
|
||||
.run(host.id, '2026-10-13T12:00:00Z', 25);
|
||||
db().exec('DROP TABLE ec_host_temperature_samples; PRAGMA user_version=22;');
|
||||
closeDb();
|
||||
expect(db().prepare('SELECT * FROM ec_host_temperature_samples').all()).toEqual([]);
|
||||
expect(
|
||||
db().prepare('SELECT cpu_usage_percent FROM ec_host_resource_samples').all()
|
||||
).toEqual([{ cpu_usage_percent: 25 }]);
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -43,6 +43,7 @@ const host = (changes: Partial<CapacityHost> = {}): CapacityHost => ({
|
||||
...changes
|
||||
});
|
||||
const history: CapacityHistory = {
|
||||
temperatures: [],
|
||||
range: '24h',
|
||||
startAt: 1000,
|
||||
endAt: 5000,
|
||||
@@ -112,6 +113,10 @@ it('keeps current estimates and host comparison before history with scoped histo
|
||||
expect(body).not.toContain('Manage hosts');
|
||||
expect(body).toContain('Updates every 30 seconds');
|
||||
expect(body).not.toContain('>Refresh<');
|
||||
expect(body.indexOf('id="temperature-history-title"')).toBeGreaterThan(
|
||||
body.indexOf('id="capacity-history-title"')
|
||||
);
|
||||
expect(body.match(/aria-label="History range"/g)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('keeps fleet-wide queue scope explicit and preserves unavailable estimates', () => {
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
stableHostOrder
|
||||
} from '../../src/lib/ui/host-overview.js';
|
||||
import HostList from '../../src/lib/ui/components/HostList.svelte';
|
||||
import HostTemperatures from '../../src/lib/ui/components/HostTemperatures.svelte';
|
||||
|
||||
const now = Date.parse('2026-10-06T12:00:00Z');
|
||||
function host(id = 'ec-1', changes: Partial<Host> = {}): Host {
|
||||
@@ -43,6 +44,40 @@ function host(id = 'ec-1', changes: Partial<Host> = {}): Host {
|
||||
};
|
||||
}
|
||||
|
||||
it('shows CPU temperature with stale status and keeps sensor details expandable', () => {
|
||||
const target = host('ec-2', {
|
||||
connected: false,
|
||||
inventory: {
|
||||
cpuUsagePercent: 25,
|
||||
temperatures: [
|
||||
{
|
||||
id: 'cpu',
|
||||
kind: 'cpu-package',
|
||||
device: 'coretemp',
|
||||
label: 'Package id 0',
|
||||
celsius: 65
|
||||
}
|
||||
]
|
||||
}
|
||||
});
|
||||
const list = render(HostList, {
|
||||
props: {
|
||||
hosts: [{ ...target, serverCount: 0, playerCount: 0, unreportedInstances: 0 }],
|
||||
now
|
||||
}
|
||||
}).body;
|
||||
expect(list).toContain('CPU temp: 65 °C');
|
||||
expect(list).toContain('Resource readings stale');
|
||||
const detail = render(HostTemperatures, { props: { host: target } }).body;
|
||||
expect(detail).toContain('(last reported)');
|
||||
expect(detail).toContain('Temperature sensors');
|
||||
expect(detail).toContain('CPU package 0');
|
||||
expect(detail).not.toMatch(/<details[^>]*open/);
|
||||
const missing = render(HostTemperatures, { props: { host: host() } }).body;
|
||||
expect(missing).toContain('Unavailable');
|
||||
expect(missing).not.toContain('0 °C');
|
||||
});
|
||||
|
||||
it('does not show an old Ready state as available when the heartbeat becomes stale', () => {
|
||||
expect(hostListStatus(host(), now).label).toBe('Ready');
|
||||
expect(hostReadingsStale(host(), now + 90_001)).toBe(true);
|
||||
|
||||
Reference in New Issue
Block a user