Add host temperature telemetry and history
This commit is contained in:
@@ -1,5 +1,24 @@
|
||||
# Agent changelog
|
||||
|
||||
## 2026-10-13
|
||||
|
||||
### 1.10.0 — Host temperature telemetry
|
||||
|
||||
- Read supported Linux hwmon CPU package/core, AMD physical die/CCD, NVMe and
|
||||
Intel PCH temperatures with the Python standard library. Sensor failures,
|
||||
faults and disabled inputs are skipped independently; unsupported hosts report
|
||||
an empty list. AMD Tctl control offsets are excluded from physical readings.
|
||||
- Include labeled, bounded sensors in heartbeats with the `host.temperatures`
|
||||
capability. Physical device/channel identities survive hwmon renumbering.
|
||||
- The controller shows current CPU readings and expandable sensor details, and
|
||||
adds temperature history to Capacity. Fleet history has one CPU line per host;
|
||||
individual hosts can add NVMe composite and chipset lines. History retains
|
||||
minute peaks using the host-resource retention setting; missing data stays
|
||||
unavailable and no thermal alerts or placement rules are introduced.
|
||||
- Adds database migration 23. Existing agents remain compatible. Release
|
||||
artifacts are packaged and the stale signature is removed. Offline signing
|
||||
and host rollout are still required; neither was performed here.
|
||||
|
||||
## 2026-10-12
|
||||
|
||||
### 1.9.3 — Protect private bank configuration (EC #152)
|
||||
|
||||
+58
-3
@@ -35,7 +35,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.9.3"
|
||||
VERSION = "1.10.0"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
MANIFEST_MAX_ENTRIES = 10000
|
||||
@@ -63,6 +63,7 @@ UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-ch
|
||||
UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
|
||||
UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
|
||||
AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
|
||||
HWMON_ROOT = Path("/sys/class/hwmon")
|
||||
ROOT_MARKER = ".electric-cherry.json"
|
||||
|
||||
|
||||
@@ -1308,6 +1309,59 @@ class Agent:
|
||||
return None
|
||||
return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
|
||||
|
||||
def temperatures(self):
|
||||
"""Best-effort, read-only samples from known millidegree-Celsius drivers."""
|
||||
samples = []
|
||||
try:
|
||||
chips = sorted(HWMON_ROOT.glob("hwmon*"))[:64]
|
||||
except OSError:
|
||||
return samples
|
||||
for chip in chips:
|
||||
try:
|
||||
device = (chip / "name").read_text().strip()
|
||||
if device not in ("coretemp", "k10temp", "nvme") and not device.startswith("pch_"):
|
||||
continue
|
||||
# hwmon numbers can change on reboot. The physical sysfs path
|
||||
# and channel identify separate CPU sockets and NVMe devices.
|
||||
identity = re.sub(r"/(?:hwmon/)?hwmon\d+$", "", str(chip.resolve()))
|
||||
inputs = sorted(chip.glob("temp[0-9]*_input"))[:128]
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
for source in inputs:
|
||||
try:
|
||||
channel = source.name.removesuffix("_input")
|
||||
label_path = chip / (channel + "_label")
|
||||
label = label_path.read_text().strip() if label_path.exists() else channel
|
||||
if not label or len(label) > 120:
|
||||
continue
|
||||
fault = chip / (channel + "_fault")
|
||||
enabled = chip / (channel + "_enable")
|
||||
if (fault.exists() and int(fault.read_text()) != 0) or (enabled.exists() and int(enabled.read_text()) == 0):
|
||||
continue
|
||||
if device == "coretemp":
|
||||
kind = "cpu-package" if label.startswith("Package id ") else "cpu-core" if label.startswith("Core ") else None
|
||||
elif device == "k10temp":
|
||||
# Tctl can contain a fan-control offset. Never present
|
||||
# it as the physical die temperature or a fleet metric.
|
||||
kind = "cpu-die" if label == "Tdie" or re.fullmatch(r"Tccd\d+", label) else None
|
||||
else:
|
||||
kind = "nvme" if device == "nvme" else "chipset"
|
||||
if kind is None:
|
||||
continue
|
||||
celsius = int(source.read_text()) / 1000
|
||||
if not -100 <= celsius <= 200:
|
||||
continue
|
||||
sensor_id = hashlib.sha256((identity + "/" + channel).encode()).hexdigest()[:24]
|
||||
samples.append({"id": sensor_id, "device": device, "label": label,
|
||||
"kind": kind, "celsius": round(celsius, 1)})
|
||||
if len(samples) >= 128:
|
||||
return samples
|
||||
except (OSError, ValueError):
|
||||
# Read permission alone does not imply data is available
|
||||
# (e.g. ENODATA), and a device can disappear during a read.
|
||||
continue
|
||||
return samples
|
||||
|
||||
def instance_resources(self, observations=None):
|
||||
"""One bounded Docker stats batch; unavailable samples are never zero load."""
|
||||
ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true",
|
||||
@@ -1392,6 +1446,7 @@ class Agent:
|
||||
|
||||
def inventory(self):
|
||||
cpu_usage = self.cpu_usage_percent()
|
||||
temperatures = self.temperatures()
|
||||
usage = shutil.disk_usage(self.root)
|
||||
memory = {}
|
||||
try:
|
||||
@@ -1516,9 +1571,9 @@ class Agent:
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
update = self.update_inventory()
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
capabilities = ["host.temperatures", "library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
plutonium_cache = self.plutonium_cache_inventory()
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "temperatures": temperatures, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
||||
health = self.inspect(instance_id, generation)
|
||||
|
||||
@@ -778,6 +778,12 @@ class AgentTests(unittest.TestCase):
|
||||
else:
|
||||
self.assertEqual(inventory["cpuUsagePercent"], usage)
|
||||
|
||||
def test_inventory_reports_optional_temperature_capability_and_empty_samples(self):
|
||||
with patch.object(self.agent, "temperatures", return_value=[]):
|
||||
inventory = self.agent.inventory()
|
||||
self.assertEqual(inventory["temperatures"], [])
|
||||
self.assertIn("host.temperatures", inventory["capabilities"])
|
||||
|
||||
def resource_container(self):
|
||||
return {"Name": self.agent.container_name(INSTANCE, 1),
|
||||
"Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}},
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import errno
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("ec_agent_temperatures", Path(__file__).resolve().parents[1] / "ec_agent.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
|
||||
class TemperatureTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.hwmon = self.root / "class"
|
||||
self.hwmon.mkdir()
|
||||
self.patch = patch.object(module, "HWMON_ROOT", self.hwmon)
|
||||
self.patch.start()
|
||||
self.addCleanup(self.patch.stop)
|
||||
self.agent = object.__new__(module.Agent)
|
||||
|
||||
def chip(self, number, device, readings):
|
||||
physical = self.root / (device + str(number)) / "hwmon" / ("hwmon" + str(number))
|
||||
physical.mkdir(parents=True)
|
||||
(self.hwmon / physical.name).symlink_to(physical, target_is_directory=True)
|
||||
(physical / "name").write_text(device)
|
||||
for channel, label, value in readings:
|
||||
(physical / (channel + "_input")).write_text(str(value))
|
||||
if label is not None:
|
||||
(physical / (channel + "_label")).write_text(label)
|
||||
return physical
|
||||
|
||||
def test_ec2_sensors_and_units_without_guessing_unknown_devices(self):
|
||||
self.chip(0, "nvme", [("temp1", "Composite", 50900), ("temp2", "Sensor 1", 50900)])
|
||||
self.chip(1, "pch_cometlake", [("temp1", None, 62000)])
|
||||
self.chip(2, "coretemp", [("temp1", "Package id 0", 65000), ("temp2", "Core 0", 63000)])
|
||||
self.chip(3, "iwlwifi_1", [("temp1", None, "unavailable")])
|
||||
samples = self.agent.temperatures()
|
||||
self.assertEqual([s["celsius"] for s in samples], [50.9, 50.9, 62.0, 65.0, 63.0])
|
||||
self.assertEqual([s["kind"] for s in samples], ["nvme", "nvme", "chipset", "cpu-package", "cpu-core"])
|
||||
self.assertEqual(len({s["id"] for s in samples}), len(samples))
|
||||
|
||||
def test_fault_disabled_bad_value_and_enodata_do_not_lose_other_sensors(self):
|
||||
chip = self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), value)
|
||||
for i, value in enumerate([65000, 60000, "bad", 500000, 61000, 63000, 0], 1)])
|
||||
(chip / "temp1_fault").write_text("1")
|
||||
(chip / "temp2_enable").write_text("0")
|
||||
original = Path.read_text
|
||||
def read(path, *args, **kwargs):
|
||||
if path.name == "temp5_input":
|
||||
raise OSError(errno.ENODATA, "No data available")
|
||||
return original(path, *args, **kwargs)
|
||||
with patch.object(Path, "read_text", read):
|
||||
self.assertEqual([s["celsius"] for s in self.agent.temperatures()], [63.0, 0.0])
|
||||
|
||||
def test_amd_keeps_physical_die_readings_and_excludes_control_offsets(self):
|
||||
self.chip(0, "k10temp", [("temp1", "Tctl", 85000), ("temp2", "Tdie", 65000), ("temp3", "Tccd1", 64000)])
|
||||
self.assertEqual([s["label"] for s in self.agent.temperatures()], ["Tdie", "Tccd1"])
|
||||
|
||||
def test_sensor_identity_survives_hwmon_renumbering_and_separates_drives(self):
|
||||
physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
|
||||
self.chip(2, "nvme", [("temp1", "Composite", 51000)])
|
||||
before = {s["celsius"]: s["id"] for s in self.agent.temperatures()}
|
||||
renamed = physical.with_name("hwmon9")
|
||||
physical.rename(renamed)
|
||||
(self.hwmon / "hwmon1").unlink()
|
||||
(self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
|
||||
self.assertEqual(before, {s["celsius"]: s["id"] for s in self.agent.temperatures()})
|
||||
self.assertEqual(len(set(before.values())), 2)
|
||||
|
||||
def test_missing_hwmon_and_disappearing_devices_are_optional(self):
|
||||
self.assertEqual(self.agent.temperatures(), [])
|
||||
(self.hwmon / "hwmon0").symlink_to(self.root / "missing")
|
||||
self.assertEqual(self.agent.temperatures(), [])
|
||||
with patch.object(Path, "glob", side_effect=PermissionError()):
|
||||
self.assertEqual(self.agent.temperatures(), [])
|
||||
|
||||
def test_direct_hwmon_child_keeps_identity_when_renumbered(self):
|
||||
physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
|
||||
direct = physical.parent.parent / "hwmon8"
|
||||
physical.rename(direct)
|
||||
(self.hwmon / "hwmon1").unlink()
|
||||
(self.hwmon / "hwmon8").symlink_to(direct, target_is_directory=True)
|
||||
before = self.agent.temperatures()[0]["id"]
|
||||
renamed = direct.with_name("hwmon9")
|
||||
direct.rename(renamed)
|
||||
(self.hwmon / "hwmon8").unlink()
|
||||
(self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
|
||||
self.assertEqual(self.agent.temperatures()[0]["id"], before)
|
||||
|
||||
def test_inventory_size_is_bounded(self):
|
||||
self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), 65000) for i in range(1, 200)])
|
||||
self.assertEqual(len(self.agent.temperatures()), 128)
|
||||
Reference in New Issue
Block a user