Add host temperature telemetry and history

agent/ec_agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.9.3"
+VERSION = "1.10.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -63,6 +63,7 @@ UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-ch
 UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
 UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
 AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
+HWMON_ROOT = Path("/sys/class/hwmon")
 ROOT_MARKER = ".electric-cherry.json"
 
 
@@ -1308,6 +1309,59 @@ class Agent:
                 return None
             return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
 
+    def temperatures(self):
+        """Best-effort, read-only samples from known millidegree-Celsius drivers."""
+        samples = []
+        try:
+            chips = sorted(HWMON_ROOT.glob("hwmon*"))[:64]
+        except OSError:
+            return samples
+        for chip in chips:
+            try:
+                device = (chip / "name").read_text().strip()
+                if device not in ("coretemp", "k10temp", "nvme") and not device.startswith("pch_"):
+                    continue
+                # hwmon numbers can change on reboot. The physical sysfs path
+                # and channel identify separate CPU sockets and NVMe devices.
+                identity = re.sub(r"/(?:hwmon/)?hwmon\d+$", "", str(chip.resolve()))
+                inputs = sorted(chip.glob("temp[0-9]*_input"))[:128]
+            except (OSError, ValueError):
+                continue
+            for source in inputs:
+                try:
+                    channel = source.name.removesuffix("_input")
+                    label_path = chip / (channel + "_label")
+                    label = label_path.read_text().strip() if label_path.exists() else channel
+                    if not label or len(label) > 120:
+                        continue
+                    fault = chip / (channel + "_fault")
+                    enabled = chip / (channel + "_enable")
+                    if (fault.exists() and int(fault.read_text()) != 0) or (enabled.exists() and int(enabled.read_text()) == 0):
+                        continue
+                    if device == "coretemp":
+                        kind = "cpu-package" if label.startswith("Package id ") else "cpu-core" if label.startswith("Core ") else None
+                    elif device == "k10temp":
+                        # Tctl can contain a fan-control offset. Never present
+                        # it as the physical die temperature or a fleet metric.
+                        kind = "cpu-die" if label == "Tdie" or re.fullmatch(r"Tccd\d+", label) else None
+                    else:
+                        kind = "nvme" if device == "nvme" else "chipset"
+                    if kind is None:
+                        continue
+                    celsius = int(source.read_text()) / 1000
+                    if not -100 <= celsius <= 200:
+                        continue
+                    sensor_id = hashlib.sha256((identity + "/" + channel).encode()).hexdigest()[:24]
+                    samples.append({"id": sensor_id, "device": device, "label": label,
+                                    "kind": kind, "celsius": round(celsius, 1)})
+                    if len(samples) >= 128:
+                        return samples
+                except (OSError, ValueError):
+                    # Read permission alone does not imply data is available
+                    # (e.g. ENODATA), and a device can disappear during a read.
+                    continue
+        return samples
+
     def instance_resources(self, observations=None):
         """One bounded Docker stats batch; unavailable samples are never zero load."""
         ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true",
@@ -1392,6 +1446,7 @@ class Agent:
 
     def inventory(self):
         cpu_usage = self.cpu_usage_percent()
+        temperatures = self.temperatures()
         usage = shutil.disk_usage(self.root)
         memory = {}
         try:
@@ -1516,9 +1571,9 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["host.temperatures", "library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         plutonium_cache = self.plutonium_cache_inventory()
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "temperatures": temperatures, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
         health = self.inspect(instance_id, generation)