From 171f6578762942987ce5573f5bcc244acc5cff2d Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Tue, 13 Oct 2026 23:39:04 -0400 Subject: [PATCH] Add host temperature telemetry and history --- agent/CHANGELOG.md | 19 + agent/ec_agent.py | 61 ++- agent/tests/test_agent.py | 6 + agent/tests/test_temperatures.py | 96 ++++ docs/guides/native-hosts.md | 34 ++ src/lib/capacity.ts | 8 + src/lib/fleet-settings.ts | 2 +- src/lib/host-temperatures.ts | 92 ++++ src/lib/hosts.ts | 2 + src/lib/server/db/migrations.ts | 4 +- src/lib/server/hosts/capacity-history.ts | 18 +- src/lib/server/hosts/index.ts | 5 + src/lib/server/hosts/temperatures.ts | 96 ++++ src/lib/server/retention.ts | 8 + src/lib/ui/components/HostList.svelte | 9 +- src/lib/ui/components/HostTemperatures.svelte | 79 +++ .../capacity/TemperatureChart.svelte | 456 ++++++++++++++++++ src/lib/ui/styles/tokens.css | 6 + src/routes/capacity/+page.svelte | 25 +- src/routes/hosts/[id]/+page.svelte | 3 + static/agent/ec-agent.py | 61 ++- static/agent/manifest.json | 4 +- static/agent/manifest.json.sig | 8 - tests/client/temperature-chart.test.ts | 110 +++++ tests/unit/capacity-history.test.ts | 95 ++++ tests/unit/host-temperatures.test.ts | 151 ++++++ tests/unit/ui-capacity.test.ts | 5 + tests/unit/ui-host-overview.test.ts | 35 ++ 28 files changed, 1466 insertions(+), 32 deletions(-) create mode 100644 agent/tests/test_temperatures.py create mode 100644 src/lib/host-temperatures.ts create mode 100644 src/lib/server/hosts/temperatures.ts create mode 100644 src/lib/ui/components/HostTemperatures.svelte create mode 100644 src/lib/ui/components/capacity/TemperatureChart.svelte delete mode 100644 static/agent/manifest.json.sig create mode 100644 tests/client/temperature-chart.test.ts create mode 100644 tests/unit/host-temperatures.test.ts diff --git a/agent/CHANGELOG.md b/agent/CHANGELOG.md index fb23cab..b59cdc9 100644 --- a/agent/CHANGELOG.md +++ b/agent/CHANGELOG.md @@ -1,5 +1,24 @@ # Agent changelog +## 2026-10-13 + +### 1.10.0 — Host temperature telemetry + +- Read supported Linux hwmon CPU package/core, AMD physical die/CCD, NVMe and + Intel PCH temperatures with the Python standard library. Sensor failures, + faults and disabled inputs are skipped independently; unsupported hosts report + an empty list. AMD Tctl control offsets are excluded from physical readings. +- Include labeled, bounded sensors in heartbeats with the `host.temperatures` + capability. Physical device/channel identities survive hwmon renumbering. +- The controller shows current CPU readings and expandable sensor details, and + adds temperature history to Capacity. Fleet history has one CPU line per host; + individual hosts can add NVMe composite and chipset lines. History retains + minute peaks using the host-resource retention setting; missing data stays + unavailable and no thermal alerts or placement rules are introduced. +- Adds database migration 23. Existing agents remain compatible. Release + artifacts are packaged and the stale signature is removed. Offline signing + and host rollout are still required; neither was performed here. + ## 2026-10-12 ### 1.9.3 — Protect private bank configuration (EC #152) diff --git a/agent/ec_agent.py b/agent/ec_agent.py index e8515d6..a420f94 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.9.3" +VERSION = "1.10.0" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -63,6 +63,7 @@ UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-ch UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json") UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json") AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json") +HWMON_ROOT = Path("/sys/class/hwmon") ROOT_MARKER = ".electric-cherry.json" @@ -1308,6 +1309,59 @@ class Agent: return None return round(100 * (total - deltas[3] - deltas[4]) / total, 1) + def temperatures(self): + """Best-effort, read-only samples from known millidegree-Celsius drivers.""" + samples = [] + try: + chips = sorted(HWMON_ROOT.glob("hwmon*"))[:64] + except OSError: + return samples + for chip in chips: + try: + device = (chip / "name").read_text().strip() + if device not in ("coretemp", "k10temp", "nvme") and not device.startswith("pch_"): + continue + # hwmon numbers can change on reboot. The physical sysfs path + # and channel identify separate CPU sockets and NVMe devices. + identity = re.sub(r"/(?:hwmon/)?hwmon\d+$", "", str(chip.resolve())) + inputs = sorted(chip.glob("temp[0-9]*_input"))[:128] + except (OSError, ValueError): + continue + for source in inputs: + try: + channel = source.name.removesuffix("_input") + label_path = chip / (channel + "_label") + label = label_path.read_text().strip() if label_path.exists() else channel + if not label or len(label) > 120: + continue + fault = chip / (channel + "_fault") + enabled = chip / (channel + "_enable") + if (fault.exists() and int(fault.read_text()) != 0) or (enabled.exists() and int(enabled.read_text()) == 0): + continue + if device == "coretemp": + kind = "cpu-package" if label.startswith("Package id ") else "cpu-core" if label.startswith("Core ") else None + elif device == "k10temp": + # Tctl can contain a fan-control offset. Never present + # it as the physical die temperature or a fleet metric. + kind = "cpu-die" if label == "Tdie" or re.fullmatch(r"Tccd\d+", label) else None + else: + kind = "nvme" if device == "nvme" else "chipset" + if kind is None: + continue + celsius = int(source.read_text()) / 1000 + if not -100 <= celsius <= 200: + continue + sensor_id = hashlib.sha256((identity + "/" + channel).encode()).hexdigest()[:24] + samples.append({"id": sensor_id, "device": device, "label": label, + "kind": kind, "celsius": round(celsius, 1)}) + if len(samples) >= 128: + return samples + except (OSError, ValueError): + # Read permission alone does not imply data is available + # (e.g. ENODATA), and a device can disappear during a read. + continue + return samples + def instance_resources(self, observations=None): """One bounded Docker stats batch; unavailable samples are never zero load.""" ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true", @@ -1392,6 +1446,7 @@ class Agent: def inventory(self): cpu_usage = self.cpu_usage_percent() + temperatures = self.temperatures() usage = shutil.disk_usage(self.root) memory = {} try: @@ -1516,9 +1571,9 @@ class Agent: if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() - capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) + capabilities = ["host.temperatures", "library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) plutonium_cache = self.plutonium_cache_inventory() - return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} + return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "temperatures": temperatures, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False): health = self.inspect(instance_id, generation) diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 0547107..8388315 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -778,6 +778,12 @@ class AgentTests(unittest.TestCase): else: self.assertEqual(inventory["cpuUsagePercent"], usage) + def test_inventory_reports_optional_temperature_capability_and_empty_samples(self): + with patch.object(self.agent, "temperatures", return_value=[]): + inventory = self.agent.inventory() + self.assertEqual(inventory["temperatures"], []) + self.assertIn("host.temperatures", inventory["capabilities"]) + def resource_container(self): return {"Name": self.agent.container_name(INSTANCE, 1), "Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}}, diff --git a/agent/tests/test_temperatures.py b/agent/tests/test_temperatures.py new file mode 100644 index 0000000..c46eaa0 --- /dev/null +++ b/agent/tests/test_temperatures.py @@ -0,0 +1,96 @@ +import errno +import importlib.util +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("ec_agent_temperatures", Path(__file__).resolve().parents[1] / "ec_agent.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class TemperatureTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.hwmon = self.root / "class" + self.hwmon.mkdir() + self.patch = patch.object(module, "HWMON_ROOT", self.hwmon) + self.patch.start() + self.addCleanup(self.patch.stop) + self.agent = object.__new__(module.Agent) + + def chip(self, number, device, readings): + physical = self.root / (device + str(number)) / "hwmon" / ("hwmon" + str(number)) + physical.mkdir(parents=True) + (self.hwmon / physical.name).symlink_to(physical, target_is_directory=True) + (physical / "name").write_text(device) + for channel, label, value in readings: + (physical / (channel + "_input")).write_text(str(value)) + if label is not None: + (physical / (channel + "_label")).write_text(label) + return physical + + def test_ec2_sensors_and_units_without_guessing_unknown_devices(self): + self.chip(0, "nvme", [("temp1", "Composite", 50900), ("temp2", "Sensor 1", 50900)]) + self.chip(1, "pch_cometlake", [("temp1", None, 62000)]) + self.chip(2, "coretemp", [("temp1", "Package id 0", 65000), ("temp2", "Core 0", 63000)]) + self.chip(3, "iwlwifi_1", [("temp1", None, "unavailable")]) + samples = self.agent.temperatures() + self.assertEqual([s["celsius"] for s in samples], [50.9, 50.9, 62.0, 65.0, 63.0]) + self.assertEqual([s["kind"] for s in samples], ["nvme", "nvme", "chipset", "cpu-package", "cpu-core"]) + self.assertEqual(len({s["id"] for s in samples}), len(samples)) + + def test_fault_disabled_bad_value_and_enodata_do_not_lose_other_sensors(self): + chip = self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), value) + for i, value in enumerate([65000, 60000, "bad", 500000, 61000, 63000, 0], 1)]) + (chip / "temp1_fault").write_text("1") + (chip / "temp2_enable").write_text("0") + original = Path.read_text + def read(path, *args, **kwargs): + if path.name == "temp5_input": + raise OSError(errno.ENODATA, "No data available") + return original(path, *args, **kwargs) + with patch.object(Path, "read_text", read): + self.assertEqual([s["celsius"] for s in self.agent.temperatures()], [63.0, 0.0]) + + def test_amd_keeps_physical_die_readings_and_excludes_control_offsets(self): + self.chip(0, "k10temp", [("temp1", "Tctl", 85000), ("temp2", "Tdie", 65000), ("temp3", "Tccd1", 64000)]) + self.assertEqual([s["label"] for s in self.agent.temperatures()], ["Tdie", "Tccd1"]) + + def test_sensor_identity_survives_hwmon_renumbering_and_separates_drives(self): + physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)]) + self.chip(2, "nvme", [("temp1", "Composite", 51000)]) + before = {s["celsius"]: s["id"] for s in self.agent.temperatures()} + renamed = physical.with_name("hwmon9") + physical.rename(renamed) + (self.hwmon / "hwmon1").unlink() + (self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True) + self.assertEqual(before, {s["celsius"]: s["id"] for s in self.agent.temperatures()}) + self.assertEqual(len(set(before.values())), 2) + + def test_missing_hwmon_and_disappearing_devices_are_optional(self): + self.assertEqual(self.agent.temperatures(), []) + (self.hwmon / "hwmon0").symlink_to(self.root / "missing") + self.assertEqual(self.agent.temperatures(), []) + with patch.object(Path, "glob", side_effect=PermissionError()): + self.assertEqual(self.agent.temperatures(), []) + + def test_direct_hwmon_child_keeps_identity_when_renumbered(self): + physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)]) + direct = physical.parent.parent / "hwmon8" + physical.rename(direct) + (self.hwmon / "hwmon1").unlink() + (self.hwmon / "hwmon8").symlink_to(direct, target_is_directory=True) + before = self.agent.temperatures()[0]["id"] + renamed = direct.with_name("hwmon9") + direct.rename(renamed) + (self.hwmon / "hwmon8").unlink() + (self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True) + self.assertEqual(self.agent.temperatures()[0]["id"], before) + + def test_inventory_size_is_bounded(self): + self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), 65000) for i in range(1, 200)]) + self.assertEqual(len(self.agent.temperatures()), 128) diff --git a/docs/guides/native-hosts.md b/docs/guides/native-hosts.md index a4e537b..2703fee 100644 --- a/docs/guides/native-hosts.md +++ b/docs/guides/native-hosts.md @@ -354,6 +354,40 @@ journalctl -u electric-cherry-update.service -n 100 ## Placement and resource capacity +### Host temperature + +Agent **1.10.0** reports temperature sensors exposed by supported Linux hwmon +drivers without installing additional packages. **Hosts** shows CPU temperature +beneath CPU usage. A host's **Overview → Inventory** shows the selected reading +and expandable **Temperature sensors**, including per-core and storage details. +Stale readings are marked as last reported; absent or failed readings are +**Unavailable**, never zero or a readiness failure. + +**Fleet → Capacity → Temperature history** uses the existing host selector and +Last hour / 24 hours / 7 days controls. **All hosts** displays a separate CPU line +for each host. Select one host to add its NVMe composite and chipset lines using +the legend. Hover, tap, or use the arrow keys on the chart to inspect a period. +Each host page's **Capacity** button opens the view filtered to that host. +The game filter affects launch estimates, not temperatures. + +The CPU reading is the hottest reported package, falling back to a physical +AMD die/CCD and then a core reading. AMD Tctl cooling-control values are excluded +because they may include an offset. Supported device drivers are `coretemp`, +`k10temp`, `nvme` and `pch_*`; sensor availability depends on hardware and drivers. +Virtual machines and older agents may have no supported readings. Missing, +faulty and disabled sensors are skipped individually. + +Temperature history starts after the agent upgrade. The controller stores one +peak per sensor per minute, limited to the CPU summary and up to sixteen NVMe +composite/chipset inputs per host. Longer chart periods show the maximum of +those peaks; a period without readings remains a gap. Brief gaps within an +aggregated period may still leave a peak for that period. **Host resource history +retention** applies to CPU, memory and temperature samples (seven days by default; +zero keeps them indefinitely). Series colors identify hosts/sensors and do not +indicate thermal safety. Temperature does not change placement or trigger alerts. + +### CPU and memory capacity + ### Host inventory and eligibility The Inventory panel shows CPU usage with agent 1.4.2 and later. It measures diff --git a/src/lib/capacity.ts b/src/lib/capacity.ts index 52d3a73..569627a 100644 --- a/src/lib/capacity.ts +++ b/src/lib/capacity.ts @@ -30,6 +30,14 @@ export interface CapacityHistory { endAt: number; bucketMs: number; points: CapacityHistoryPoint[]; + temperatures: TemperatureHistorySeries[]; +} +export interface TemperatureHistorySeries { + id: string; + hostId: string; + label: string; + kind: 'cpu' | 'nvme' | 'chipset'; + points: { at: number; celsius: number | null }[]; } export interface CapacityHost { id: string; diff --git a/src/lib/fleet-settings.ts b/src/lib/fleet-settings.ts index 79e3c69..23183d2 100644 --- a/src/lib/fleet-settings.ts +++ b/src/lib/fleet-settings.ts @@ -305,7 +305,7 @@ export const settingHelp: Partial> = { 'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.', recoveryKeepDays: 'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.', - hostResourceKeepDays: 'Host CPU and memory history.', + hostResourceKeepDays: 'Host CPU, memory and temperature history.', instanceResourceKeepDays: 'Minute resource samples. Capacity learning continues to use its independent 24-hour window.', metricsKeepDays: 'Historical fleet occupancy and utilization samples.', diff --git a/src/lib/host-temperatures.ts b/src/lib/host-temperatures.ts new file mode 100644 index 0000000..9f51b66 --- /dev/null +++ b/src/lib/host-temperatures.ts @@ -0,0 +1,92 @@ +export const temperatureKinds = ['cpu-package', 'cpu-die', 'cpu-core', 'nvme', 'chipset'] as const; +export type TemperatureKind = (typeof temperatureKinds)[number]; +export interface HostTemperatureSensor { + id: string; + device: string; + label: string; + kind: TemperatureKind; + celsius: number; +} + +/** Optional telemetry must never make an otherwise valid heartbeat fail. */ +export function validateTemperatures(value: unknown): HostTemperatureSensor[] { + if (!Array.isArray(value)) return []; + const seen = new Set(); + return value.slice(0, 128).flatMap((sensor) => { + if ( + !sensor || + typeof sensor !== 'object' || + Array.isArray(sensor) || + typeof sensor.id !== 'string' || + !/^[a-zA-Z0-9_-]{1,100}$/.test(sensor.id) || + seen.has(sensor.id) || + !['device', 'label'].every( + (key) => + typeof sensor[key] === 'string' && + sensor[key].length > 0 && + sensor[key].length <= 120 + ) || + !temperatureKinds.includes(sensor.kind) || + typeof sensor.celsius !== 'number' || + !Number.isFinite(sensor.celsius) || + sensor.celsius < -100 || + sensor.celsius > 200 + ) + return []; + seen.add(sensor.id); + return [ + { + id: sensor.id, + device: sensor.device, + label: sensor.label, + kind: sensor.kind, + celsius: sensor.celsius + } + ]; + }); +} + +export function cpuTemperature( + sensors: HostTemperatureSensor[] = [] +): HostTemperatureSensor | null { + // Prefer physical package/die readings. Cores are a fallback, never mixed + // into an average that might hide a hot package on a multi-socket host. + for (const kind of ['cpu-package', 'cpu-die', 'cpu-core'] as const) { + const candidates = sensors.filter((sensor) => sensor.kind === kind); + if (candidates.length) + return candidates.reduce((hot, sensor) => + sensor.celsius > hot.celsius ? sensor : hot + ); + } + return null; +} + +export function temperatureLabel(sensor: HostTemperatureSensor): string { + if (sensor.kind === 'cpu-package') + return `CPU package ${sensor.label.replace(/^Package id /, '')}`; + if (sensor.kind === 'cpu-core') return `CPU ${sensor.label.toLowerCase()}`; + if (sensor.kind === 'cpu-die') return `CPU die · ${sensor.label}`; + if (sensor.kind === 'chipset') return 'Chipset'; + return `NVMe · ${sensor.label}`; +} + +export function formatTemperature(celsius: number | null | undefined): string { + return typeof celsius === 'number' && Number.isFinite(celsius) + ? `${Number(celsius.toFixed(1))} °C` + : 'Unavailable'; +} + +/** One CPU series per host, plus bounded device series for the host drill-down. */ +export function historyTemperatures( + sensors: HostTemperatureSensor[] = [] +): HostTemperatureSensor[] { + const cpu = cpuTemperature(sensors); + const devices = sensors + .filter( + (sensor) => + sensor.kind === 'chipset' || + (sensor.kind === 'nvme' && sensor.label.toLowerCase() === 'composite') + ) + .slice(0, 16); + return [...(cpu ? [{ ...cpu, id: 'cpu', label: 'CPU temperature' }] : []), ...devices]; +} diff --git a/src/lib/hosts.ts b/src/lib/hosts.ts index 65a3d05..e11c0ef 100644 --- a/src/lib/hosts.ts +++ b/src/lib/hosts.ts @@ -1,4 +1,5 @@ import type { HostLibraryCache } from './game-files.js'; +import type { HostTemperatureSensor } from './host-temperatures.js'; export type HostMode = 'active' | 'draining' | 'disabled'; export type HostReadiness = @@ -149,6 +150,7 @@ export interface HostInventory { cpu?: string; cpuCount?: number; cpuUsagePercent?: number; + temperatures?: HostTemperatureSensor[]; memoryTotal?: number; memoryAvailable?: number; storageTotal?: number; diff --git a/src/lib/server/db/migrations.ts b/src/lib/server/db/migrations.ts index 43c7a3d..b019540 100644 --- a/src/lib/server/db/migrations.ts +++ b/src/lib/server/db/migrations.ts @@ -24,6 +24,7 @@ import { migrateLeaderboardReports } from './leaderboard-reports.js'; import { migratePlayerScoreTotal } from './player-score-total.js'; import { migrateBank } from './bank.js'; import { migrateBankCredentials } from './bank-credentials.js'; +import { migrateTemperatures } from '../hosts/temperatures.js'; import type { Migration } from './migration-runner.js'; // Append only: never renumber entries or change an applied migration to add an @@ -118,7 +119,8 @@ export const migrations: readonly Migration[] = [ }, { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank }, { version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials }, - { version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions } + { version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions }, + { version: 23, name: 'host temperature history', run: migrateTemperatures } ]; export const databaseVersion = migrations.length; diff --git a/src/lib/server/hosts/capacity-history.ts b/src/lib/server/hosts/capacity-history.ts index 163775a..9e78514 100644 --- a/src/lib/server/hosts/capacity-history.ts +++ b/src/lib/server/hosts/capacity-history.ts @@ -1,5 +1,6 @@ import { db } from '../db/index.js'; import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js'; +import { readTemperatureHistory } from './temperatures.js'; const windows: Record = { '1h': { duration: 3_600_000, bucketMs: 60_000 }, @@ -15,10 +16,14 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap const count = duration / bucketMs; const hosts = handle .prepare( - `SELECT id,created_at FROM ec_hosts + `SELECT id,name,created_at FROM ec_hosts WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id` ) - .all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[]; + .all(...(query.hostId ? [query.hostId] : [])) as { + id: string; + name: string; + created_at: string; + }[]; // The primary key bounds each host scan to the selected interval. Aggregate // before crossing the worker boundary; do not return raw heartbeat rows. const samples = handle.prepare(`SELECT @@ -69,5 +74,12 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap point.memoryBytesAvailable = null; } } - return { range: query.range, startAt, endAt: now, bucketMs, points }; + return { + range: query.range, + startAt, + endAt: now, + bucketMs, + points, + temperatures: readTemperatureHistory(query, hosts, startAt, now, bucketMs) + }; } diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index ce16d40..cb1c91d 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -23,6 +23,8 @@ import { logResultMetadata } from './log-results.js'; import { parseHostInventory } from './request-context.js'; import { fleetSettings } from '../settings.js'; import { recordInstanceResources, validateInstanceResources } from './calibration.js'; +import { validateTemperatures } from '../../host-temperatures.js'; +import { recordTemperatures } from './temperatures.js'; import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js'; import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js'; import type { @@ -622,6 +624,8 @@ export function validateInventory(value: unknown): HostInventory { throw new Error('Host inventory must be an object.'); const inventory = value as HostInventory; if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.'); + if (inventory.temperatures !== undefined) + inventory.temperatures = validateTemperatures(inventory.temperatures); for (const key of ['hostname', 'agentVersion', 'os', 'architecture', 'cpu', 'dockerVersion']) { if ( inventory[key] !== undefined && @@ -924,6 +928,7 @@ export function heartbeat(hostId: string, value: unknown): Host { function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void { recordInstanceResources(hostId, inventory, timestamp); + recordTemperatures(hostId, inventory, timestamp); database() .prepare( `INSERT OR REPLACE INTO ec_host_resource_samples diff --git a/src/lib/server/hosts/temperatures.ts b/src/lib/server/hosts/temperatures.ts new file mode 100644 index 0000000..51c6c36 --- /dev/null +++ b/src/lib/server/hosts/temperatures.ts @@ -0,0 +1,96 @@ +import type Database from 'better-sqlite3'; +import { db } from '../db/index.js'; +import { historyTemperatures } from '../../host-temperatures.js'; +import type { HostInventory } from '../../hosts.js'; +import type { CapacityQuery, TemperatureHistorySeries } from '../../capacity.js'; + +export function migrateTemperatures(handle: Database.Database): void { + handle.exec(`CREATE TABLE IF NOT EXISTS ec_host_temperature_samples ( + host_id TEXT NOT NULL REFERENCES ec_hosts(id), bucket TEXT NOT NULL, + sensor_id TEXT NOT NULL, kind TEXT NOT NULL, label TEXT NOT NULL, + celsius REAL NOT NULL CHECK(celsius BETWEEN -100 AND 200), + PRIMARY KEY(host_id,bucket,sensor_id) + ); + CREATE INDEX IF NOT EXISTS ec_host_temperatures_retention ON ec_host_temperature_samples(bucket);`); +} + +export function recordTemperatures( + hostId: string, + inventory: HostInventory, + timestamp: string +): void { + const sensors = historyTemperatures(inventory.temperatures); + if (!sensors.length) return; + const bucket = new Date(Math.floor(Date.parse(timestamp) / 60_000) * 60_000).toISOString(); + const insert = db().prepare(`INSERT INTO ec_host_temperature_samples + (host_id,bucket,sensor_id,kind,label,celsius) VALUES(?,?,?,?,?,?) + ON CONFLICT(host_id,bucket,sensor_id) DO UPDATE SET + celsius=MAX(celsius,excluded.celsius),kind=excluded.kind,label=excluded.label`); + for (const sensor of sensors) + insert.run(hostId, bucket, sensor.id, sensor.kind, sensor.label, sensor.celsius); +} + +/** Runs on the capacity report worker, returning at most 168 points per series. */ +export function readTemperatureHistory( + query: CapacityQuery, + hosts: { id: string; name: string; created_at: string }[], + startAt: number, + endAt: number, + bucketMs: number +): TemperatureHistorySeries[] { + const count = Math.round((endAt - startAt) / bucketMs); + const statement = db().prepare(`SELECT sensor_id,kind,label, + MIN(?,MAX(0,CAST((CAST(strftime('%s',bucket) AS INTEGER)*1000-?) / ? AS INTEGER))) AS period, + MAX(celsius) AS celsius + FROM ec_host_temperature_samples WHERE host_id=? AND bucket>=? AND bucket<=? + ${query.hostId ? '' : "AND sensor_id='cpu'"} + GROUP BY sensor_id,period ORDER BY sensor_id,period`); + const result: TemperatureHistorySeries[] = []; + for (const host of hosts) { + const bySensor = new Map(); + const add = (id: string, kind: TemperatureHistorySeries['kind'], label: string) => { + const series = { + id: `${host.id}:${id}`, + hostId: host.id, + kind, + label, + points: Array.from({ length: count }, (_, i) => ({ + at: startAt + i * bucketMs, + celsius: null as number | null + })) + }; + bySensor.set(id, series); + result.push(series); + return series; + }; + add('cpu', 'cpu', query.hostId ? 'CPU temperature' : host.name); + const rows = statement.all( + count - 1, + startAt, + bucketMs, + host.id, + new Date( + Math.max( + Math.floor(Date.parse(host.created_at) / 60_000) * 60_000, + Math.floor(startAt / 60_000) * 60_000 + ) + ).toISOString(), + new Date(endAt).toISOString() + ) as { sensor_id: string; kind: string; label: string; period: number; celsius: number }[]; + for (const row of rows) { + let series = bySensor.get(row.sensor_id); + if (!series) { + if (!['nvme', 'chipset'].includes(row.kind) || bySensor.size >= 17) continue; + const kind = row.kind as 'nvme' | 'chipset'; + const number = [...bySensor.values()].filter((s) => s.kind === kind).length + 1; + series = add( + row.sensor_id, + kind, + `${kind === 'nvme' ? 'NVMe drive' : 'Chipset'} ${number}` + ); + } + if (series.points[row.period]) series.points[row.period].celsius = row.celsius; + } + } + return result; +} diff --git a/src/lib/server/retention.ts b/src/lib/server/retention.ts index 8b7e051..ace9ab1 100644 --- a/src/lib/server/retention.ts +++ b/src/lib/server/retention.ts @@ -556,6 +556,14 @@ function samplePolicies(s = fleetSettings()) { value: iso(s.hostResourceKeepDays), enabled: s.hostResourceKeepDays > 0 }, + { + category: 'host-temperatures', + label: 'Host temperature samples', + table: 'ec_host_temperature_samples', + condition: 'bucket < ?', + value: iso(s.hostResourceKeepDays), + enabled: s.hostResourceKeepDays > 0 + }, { category: 'instance-resources', label: 'Instance resource samples', diff --git a/src/lib/ui/components/HostList.svelte b/src/lib/ui/components/HostList.svelte index d952f31..1f825a7 100644 --- a/src/lib/ui/components/HostList.svelte +++ b/src/lib/ui/components/HostList.svelte @@ -1,5 +1,6 @@ + +
+ {formatTemperature(cpu?.celsius)}{#if stale && cpu} + (last reported){/if} + {#if cpu}{temperatureLabel(cpu)}{/if} + {#if sensors.length} +
+ Temperature sensors +
    + {#each sensors as sensor (sensor.id)} +
  • + {temperatureLabel(sensor)}{sensor.device} · {sensor.label} · {sensor.id.slice(-6)}{formatTemperature(sensor.celsius)} +
    • + {/each} +
+ {#if stale}

Last reported readings; waiting for a fresh host heartbeat.

{/if} +
+ {:else}No supported temperature sensors reported.{/if} +
+ + diff --git a/src/lib/ui/components/capacity/TemperatureChart.svelte b/src/lib/ui/components/capacity/TemperatureChart.svelte new file mode 100644 index 0000000..c34ef29 --- /dev/null +++ b/src/lib/ui/components/capacity/TemperatureChart.svelte @@ -0,0 +1,456 @@ + + +
+
+
+

Temperature history

+

{scope} · Peak temperature per period

+
+ {@render controls?.()} +
+
+ {#each series as sensor (sensor.id)} + + {/each} +
+
Temperature (°C)
+
+ +
+ + {#if !measured}
+ {visible.length || !series.length + ? 'No temperature readings in this period' + : 'Select a sensor to show history'} + {#if visible.length || !series.length}Readings appear when an agent reports supported temperature sensors.{/if} +
{/if} +
+ +
+ {#if selectedTime !== undefined && visible.length} +
+ {$timeFormat.period(selectedTime)} +
+ {#each visible as sensor (sensor.id)}
+
{sensor.label}
+
{formatTemperature(sensor.points[selectedIndex]?.celsius)}
+
{/each} +
+
+ {/if} +
+ About this chart +

+ Each CPU line represents one host: its hottest reported package or physical die, falling + back to its hottest core. Select a host to compare its CPU, NVMe drives and chipset. + Per-minute peaks are retained; longer periods show their highest reading. Gaps mean no + measurement was received in that period. Temperatures include all workloads and are + unaffected by the game filter. Colors identify series; they do not indicate safe or + unsafe temperatures. +

+
+
+ + diff --git a/src/lib/ui/styles/tokens.css b/src/lib/ui/styles/tokens.css index 5fc01c8..e9dbcd1 100644 --- a/src/lib/ui/styles/tokens.css +++ b/src/lib/ui/styles/tokens.css @@ -78,6 +78,12 @@ /* Metrics: unknown uses a pattern as well as color. */ --chart-active: var(--accent-primary); + --chart-series-1: var(--accent-primary); + --chart-series-2: #b99af7; + --chart-series-3: #69c6ba; + --chart-series-4: #dfb76c; + --chart-series-5: #df91b6; + --chart-series-6: #bbc7db; --chart-idle: #2b3442; --chart-unknown-base: #605b64; --chart-unknown-stripe: #a7a0ad; diff --git a/src/routes/capacity/+page.svelte b/src/routes/capacity/+page.svelte index 9db849f..a4a387b 100644 --- a/src/routes/capacity/+page.svelte +++ b/src/routes/capacity/+page.svelte @@ -17,6 +17,7 @@ import Select from '$ui/components/primitives/Select.svelte'; import CapacityHosts from '$ui/components/capacity/CapacityHosts.svelte'; import CapacityChart from '$ui/components/capacity/CapacityChart.svelte'; + import TemperatureChart from '$ui/components/capacity/TemperatureChart.svelte'; import RebalancePanel from '$ui/components/capacity/RebalancePanel.svelte'; import Freshness from '$ui/components/Freshness.svelte'; import { plural } from '$ui/plural.js'; @@ -319,16 +320,20 @@ : 'Waiting for host reports.'} Refreshes every 30 seconds while visible.

- - {#snippet controls()}{/snippet} - + {#snippet historyRanges()}{/snippet} + + {#key report.query.hostId}{/key}