Add host temperature telemetry and history

This commit is contained in:
2026-10-13 23:39:04 -04:00
parent 09396ab584
commit 171f657876
28 changed files with 1466 additions and 32 deletions
+19
View File
@@ -1,5 +1,24 @@
# Agent changelog # Agent changelog
## 2026-10-13
### 1.10.0 — Host temperature telemetry
- Read supported Linux hwmon CPU package/core, AMD physical die/CCD, NVMe and
Intel PCH temperatures with the Python standard library. Sensor failures,
faults and disabled inputs are skipped independently; unsupported hosts report
an empty list. AMD Tctl control offsets are excluded from physical readings.
- Include labeled, bounded sensors in heartbeats with the `host.temperatures`
capability. Physical device/channel identities survive hwmon renumbering.
- The controller shows current CPU readings and expandable sensor details, and
adds temperature history to Capacity. Fleet history has one CPU line per host;
individual hosts can add NVMe composite and chipset lines. History retains
minute peaks using the host-resource retention setting; missing data stays
unavailable and no thermal alerts or placement rules are introduced.
- Adds database migration 23. Existing agents remain compatible. Release
artifacts are packaged and the stale signature is removed. Offline signing
and host rollout are still required; neither was performed here.
## 2026-10-12 ## 2026-10-12
### 1.9.3 — Protect private bank configuration (EC #152) ### 1.9.3 — Protect private bank configuration (EC #152)
+58 -3
View File
@@ -35,7 +35,7 @@ import urllib.parse
import urllib.request import urllib.request
import uuid import uuid
VERSION = "1.9.3" VERSION = "1.10.0"
LABEL = "io.electric-cherry" LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024 MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000 MANIFEST_MAX_ENTRIES = 10000
@@ -63,6 +63,7 @@ UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-ch
UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json") UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json") UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json") AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
HWMON_ROOT = Path("/sys/class/hwmon")
ROOT_MARKER = ".electric-cherry.json" ROOT_MARKER = ".electric-cherry.json"
@@ -1308,6 +1309,59 @@ class Agent:
return None return None
return round(100 * (total - deltas[3] - deltas[4]) / total, 1) return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
def temperatures(self):
"""Best-effort, read-only samples from known millidegree-Celsius drivers."""
samples = []
try:
chips = sorted(HWMON_ROOT.glob("hwmon*"))[:64]
except OSError:
return samples
for chip in chips:
try:
device = (chip / "name").read_text().strip()
if device not in ("coretemp", "k10temp", "nvme") and not device.startswith("pch_"):
continue
# hwmon numbers can change on reboot. The physical sysfs path
# and channel identify separate CPU sockets and NVMe devices.
identity = re.sub(r"/(?:hwmon/)?hwmon\d+$", "", str(chip.resolve()))
inputs = sorted(chip.glob("temp[0-9]*_input"))[:128]
except (OSError, ValueError):
continue
for source in inputs:
try:
channel = source.name.removesuffix("_input")
label_path = chip / (channel + "_label")
label = label_path.read_text().strip() if label_path.exists() else channel
if not label or len(label) > 120:
continue
fault = chip / (channel + "_fault")
enabled = chip / (channel + "_enable")
if (fault.exists() and int(fault.read_text()) != 0) or (enabled.exists() and int(enabled.read_text()) == 0):
continue
if device == "coretemp":
kind = "cpu-package" if label.startswith("Package id ") else "cpu-core" if label.startswith("Core ") else None
elif device == "k10temp":
# Tctl can contain a fan-control offset. Never present
# it as the physical die temperature or a fleet metric.
kind = "cpu-die" if label == "Tdie" or re.fullmatch(r"Tccd\d+", label) else None
else:
kind = "nvme" if device == "nvme" else "chipset"
if kind is None:
continue
celsius = int(source.read_text()) / 1000
if not -100 <= celsius <= 200:
continue
sensor_id = hashlib.sha256((identity + "/" + channel).encode()).hexdigest()[:24]
samples.append({"id": sensor_id, "device": device, "label": label,
"kind": kind, "celsius": round(celsius, 1)})
if len(samples) >= 128:
return samples
except (OSError, ValueError):
# Read permission alone does not imply data is available
# (e.g. ENODATA), and a device can disappear during a read.
continue
return samples
def instance_resources(self, observations=None): def instance_resources(self, observations=None):
"""One bounded Docker stats batch; unavailable samples are never zero load.""" """One bounded Docker stats batch; unavailable samples are never zero load."""
ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true", ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true",
@@ -1392,6 +1446,7 @@ class Agent:
def inventory(self): def inventory(self):
cpu_usage = self.cpu_usage_percent() cpu_usage = self.cpu_usage_percent()
temperatures = self.temperatures()
usage = shutil.disk_usage(self.root) usage = shutil.disk_usage(self.root)
memory = {} memory = {}
try: try:
@@ -1516,9 +1571,9 @@ class Agent:
if line.startswith("PRETTY_NAME="): if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"') os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory() update = self.update_inventory()
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) capabilities = ["host.temperatures", "library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
plutonium_cache = self.plutonium_cache_inventory() plutonium_cache = self.plutonium_cache_inventory()
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "temperatures": temperatures, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False): def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
health = self.inspect(instance_id, generation) health = self.inspect(instance_id, generation)
+6
View File
@@ -778,6 +778,12 @@ class AgentTests(unittest.TestCase):
else: else:
self.assertEqual(inventory["cpuUsagePercent"], usage) self.assertEqual(inventory["cpuUsagePercent"], usage)
def test_inventory_reports_optional_temperature_capability_and_empty_samples(self):
with patch.object(self.agent, "temperatures", return_value=[]):
inventory = self.agent.inventory()
self.assertEqual(inventory["temperatures"], [])
self.assertIn("host.temperatures", inventory["capabilities"])
def resource_container(self): def resource_container(self):
return {"Name": self.agent.container_name(INSTANCE, 1), return {"Name": self.agent.container_name(INSTANCE, 1),
"Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}}, "Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}},
+96
View File
@@ -0,0 +1,96 @@
import errno
import importlib.util
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location("ec_agent_temperatures", Path(__file__).resolve().parents[1] / "ec_agent.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
class TemperatureTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.hwmon = self.root / "class"
self.hwmon.mkdir()
self.patch = patch.object(module, "HWMON_ROOT", self.hwmon)
self.patch.start()
self.addCleanup(self.patch.stop)
self.agent = object.__new__(module.Agent)
def chip(self, number, device, readings):
physical = self.root / (device + str(number)) / "hwmon" / ("hwmon" + str(number))
physical.mkdir(parents=True)
(self.hwmon / physical.name).symlink_to(physical, target_is_directory=True)
(physical / "name").write_text(device)
for channel, label, value in readings:
(physical / (channel + "_input")).write_text(str(value))
if label is not None:
(physical / (channel + "_label")).write_text(label)
return physical
def test_ec2_sensors_and_units_without_guessing_unknown_devices(self):
self.chip(0, "nvme", [("temp1", "Composite", 50900), ("temp2", "Sensor 1", 50900)])
self.chip(1, "pch_cometlake", [("temp1", None, 62000)])
self.chip(2, "coretemp", [("temp1", "Package id 0", 65000), ("temp2", "Core 0", 63000)])
self.chip(3, "iwlwifi_1", [("temp1", None, "unavailable")])
samples = self.agent.temperatures()
self.assertEqual([s["celsius"] for s in samples], [50.9, 50.9, 62.0, 65.0, 63.0])
self.assertEqual([s["kind"] for s in samples], ["nvme", "nvme", "chipset", "cpu-package", "cpu-core"])
self.assertEqual(len({s["id"] for s in samples}), len(samples))
def test_fault_disabled_bad_value_and_enodata_do_not_lose_other_sensors(self):
chip = self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), value)
for i, value in enumerate([65000, 60000, "bad", 500000, 61000, 63000, 0], 1)])
(chip / "temp1_fault").write_text("1")
(chip / "temp2_enable").write_text("0")
original = Path.read_text
def read(path, *args, **kwargs):
if path.name == "temp5_input":
raise OSError(errno.ENODATA, "No data available")
return original(path, *args, **kwargs)
with patch.object(Path, "read_text", read):
self.assertEqual([s["celsius"] for s in self.agent.temperatures()], [63.0, 0.0])
def test_amd_keeps_physical_die_readings_and_excludes_control_offsets(self):
self.chip(0, "k10temp", [("temp1", "Tctl", 85000), ("temp2", "Tdie", 65000), ("temp3", "Tccd1", 64000)])
self.assertEqual([s["label"] for s in self.agent.temperatures()], ["Tdie", "Tccd1"])
def test_sensor_identity_survives_hwmon_renumbering_and_separates_drives(self):
physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
self.chip(2, "nvme", [("temp1", "Composite", 51000)])
before = {s["celsius"]: s["id"] for s in self.agent.temperatures()}
renamed = physical.with_name("hwmon9")
physical.rename(renamed)
(self.hwmon / "hwmon1").unlink()
(self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
self.assertEqual(before, {s["celsius"]: s["id"] for s in self.agent.temperatures()})
self.assertEqual(len(set(before.values())), 2)
def test_missing_hwmon_and_disappearing_devices_are_optional(self):
self.assertEqual(self.agent.temperatures(), [])
(self.hwmon / "hwmon0").symlink_to(self.root / "missing")
self.assertEqual(self.agent.temperatures(), [])
with patch.object(Path, "glob", side_effect=PermissionError()):
self.assertEqual(self.agent.temperatures(), [])
def test_direct_hwmon_child_keeps_identity_when_renumbered(self):
physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
direct = physical.parent.parent / "hwmon8"
physical.rename(direct)
(self.hwmon / "hwmon1").unlink()
(self.hwmon / "hwmon8").symlink_to(direct, target_is_directory=True)
before = self.agent.temperatures()[0]["id"]
renamed = direct.with_name("hwmon9")
direct.rename(renamed)
(self.hwmon / "hwmon8").unlink()
(self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
self.assertEqual(self.agent.temperatures()[0]["id"], before)
def test_inventory_size_is_bounded(self):
self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), 65000) for i in range(1, 200)])
self.assertEqual(len(self.agent.temperatures()), 128)
+34
View File
@@ -354,6 +354,40 @@ journalctl -u electric-cherry-update.service -n 100
## Placement and resource capacity ## Placement and resource capacity
### Host temperature
Agent **1.10.0** reports temperature sensors exposed by supported Linux hwmon
drivers without installing additional packages. **Hosts** shows CPU temperature
beneath CPU usage. A host's **Overview → Inventory** shows the selected reading
and expandable **Temperature sensors**, including per-core and storage details.
Stale readings are marked as last reported; absent or failed readings are
**Unavailable**, never zero or a readiness failure.
**Fleet → Capacity → Temperature history** uses the existing host selector and
Last hour / 24 hours / 7 days controls. **All hosts** displays a separate CPU line
for each host. Select one host to add its NVMe composite and chipset lines using
the legend. Hover, tap, or use the arrow keys on the chart to inspect a period.
Each host page's **Capacity** button opens the view filtered to that host.
The game filter affects launch estimates, not temperatures.
The CPU reading is the hottest reported package, falling back to a physical
AMD die/CCD and then a core reading. AMD Tctl cooling-control values are excluded
because they may include an offset. Supported device drivers are `coretemp`,
`k10temp`, `nvme` and `pch_*`; sensor availability depends on hardware and drivers.
Virtual machines and older agents may have no supported readings. Missing,
faulty and disabled sensors are skipped individually.
Temperature history starts after the agent upgrade. The controller stores one
peak per sensor per minute, limited to the CPU summary and up to sixteen NVMe
composite/chipset inputs per host. Longer chart periods show the maximum of
those peaks; a period without readings remains a gap. Brief gaps within an
aggregated period may still leave a peak for that period. **Host resource history
retention** applies to CPU, memory and temperature samples (seven days by default;
zero keeps them indefinitely). Series colors identify hosts/sensors and do not
indicate thermal safety. Temperature does not change placement or trigger alerts.
### CPU and memory capacity
### Host inventory and eligibility ### Host inventory and eligibility
The Inventory panel shows CPU usage with agent 1.4.2 and later. It measures The Inventory panel shows CPU usage with agent 1.4.2 and later. It measures
+8
View File
@@ -30,6 +30,14 @@ export interface CapacityHistory {
endAt: number; endAt: number;
bucketMs: number; bucketMs: number;
points: CapacityHistoryPoint[]; points: CapacityHistoryPoint[];
temperatures: TemperatureHistorySeries[];
}
export interface TemperatureHistorySeries {
id: string;
hostId: string;
label: string;
kind: 'cpu' | 'nvme' | 'chipset';
points: { at: number; celsius: number | null }[];
} }
export interface CapacityHost { export interface CapacityHost {
id: string; id: string;
+1 -1
View File
@@ -305,7 +305,7 @@ export const settingHelp: Partial<Record<keyof FleetSettings, string>> = {
'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.', 'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.',
recoveryKeepDays: recoveryKeepDays:
'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.', 'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.',
hostResourceKeepDays: 'Host CPU and memory history.', hostResourceKeepDays: 'Host CPU, memory and temperature history.',
instanceResourceKeepDays: instanceResourceKeepDays:
'Minute resource samples. Capacity learning continues to use its independent 24-hour window.', 'Minute resource samples. Capacity learning continues to use its independent 24-hour window.',
metricsKeepDays: 'Historical fleet occupancy and utilization samples.', metricsKeepDays: 'Historical fleet occupancy and utilization samples.',
+92
View File
@@ -0,0 +1,92 @@
export const temperatureKinds = ['cpu-package', 'cpu-die', 'cpu-core', 'nvme', 'chipset'] as const;
export type TemperatureKind = (typeof temperatureKinds)[number];
export interface HostTemperatureSensor {
id: string;
device: string;
label: string;
kind: TemperatureKind;
celsius: number;
}
/** Optional telemetry must never make an otherwise valid heartbeat fail. */
export function validateTemperatures(value: unknown): HostTemperatureSensor[] {
if (!Array.isArray(value)) return [];
const seen = new Set<string>();
return value.slice(0, 128).flatMap((sensor) => {
if (
!sensor ||
typeof sensor !== 'object' ||
Array.isArray(sensor) ||
typeof sensor.id !== 'string' ||
!/^[a-zA-Z0-9_-]{1,100}$/.test(sensor.id) ||
seen.has(sensor.id) ||
!['device', 'label'].every(
(key) =>
typeof sensor[key] === 'string' &&
sensor[key].length > 0 &&
sensor[key].length <= 120
) ||
!temperatureKinds.includes(sensor.kind) ||
typeof sensor.celsius !== 'number' ||
!Number.isFinite(sensor.celsius) ||
sensor.celsius < -100 ||
sensor.celsius > 200
)
return [];
seen.add(sensor.id);
return [
{
id: sensor.id,
device: sensor.device,
label: sensor.label,
kind: sensor.kind,
celsius: sensor.celsius
}
];
});
}
export function cpuTemperature(
sensors: HostTemperatureSensor[] = []
): HostTemperatureSensor | null {
// Prefer physical package/die readings. Cores are a fallback, never mixed
// into an average that might hide a hot package on a multi-socket host.
for (const kind of ['cpu-package', 'cpu-die', 'cpu-core'] as const) {
const candidates = sensors.filter((sensor) => sensor.kind === kind);
if (candidates.length)
return candidates.reduce((hot, sensor) =>
sensor.celsius > hot.celsius ? sensor : hot
);
}
return null;
}
export function temperatureLabel(sensor: HostTemperatureSensor): string {
if (sensor.kind === 'cpu-package')
return `CPU package ${sensor.label.replace(/^Package id /, '')}`;
if (sensor.kind === 'cpu-core') return `CPU ${sensor.label.toLowerCase()}`;
if (sensor.kind === 'cpu-die') return `CPU die · ${sensor.label}`;
if (sensor.kind === 'chipset') return 'Chipset';
return `NVMe · ${sensor.label}`;
}
export function formatTemperature(celsius: number | null | undefined): string {
return typeof celsius === 'number' && Number.isFinite(celsius)
? `${Number(celsius.toFixed(1))} °C`
: 'Unavailable';
}
/** One CPU series per host, plus bounded device series for the host drill-down. */
export function historyTemperatures(
sensors: HostTemperatureSensor[] = []
): HostTemperatureSensor[] {
const cpu = cpuTemperature(sensors);
const devices = sensors
.filter(
(sensor) =>
sensor.kind === 'chipset' ||
(sensor.kind === 'nvme' && sensor.label.toLowerCase() === 'composite')
)
.slice(0, 16);
return [...(cpu ? [{ ...cpu, id: 'cpu', label: 'CPU temperature' }] : []), ...devices];
}
+2
View File
@@ -1,4 +1,5 @@
import type { HostLibraryCache } from './game-files.js'; import type { HostLibraryCache } from './game-files.js';
import type { HostTemperatureSensor } from './host-temperatures.js';
export type HostMode = 'active' | 'draining' | 'disabled'; export type HostMode = 'active' | 'draining' | 'disabled';
export type HostReadiness = export type HostReadiness =
@@ -149,6 +150,7 @@ export interface HostInventory {
cpu?: string; cpu?: string;
cpuCount?: number; cpuCount?: number;
cpuUsagePercent?: number; cpuUsagePercent?: number;
temperatures?: HostTemperatureSensor[];
memoryTotal?: number; memoryTotal?: number;
memoryAvailable?: number; memoryAvailable?: number;
storageTotal?: number; storageTotal?: number;
+3 -1
View File
@@ -24,6 +24,7 @@ import { migrateLeaderboardReports } from './leaderboard-reports.js';
import { migratePlayerScoreTotal } from './player-score-total.js'; import { migratePlayerScoreTotal } from './player-score-total.js';
import { migrateBank } from './bank.js'; import { migrateBank } from './bank.js';
import { migrateBankCredentials } from './bank-credentials.js'; import { migrateBankCredentials } from './bank-credentials.js';
import { migrateTemperatures } from '../hosts/temperatures.js';
import type { Migration } from './migration-runner.js'; import type { Migration } from './migration-runner.js';
// Append only: never renumber entries or change an applied migration to add an // Append only: never renumber entries or change an applied migration to add an
@@ -118,7 +119,8 @@ export const migrations: readonly Migration[] = [
}, },
{ version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank }, { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank },
{ version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials }, { version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials },
{ version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions } { version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions },
{ version: 23, name: 'host temperature history', run: migrateTemperatures }
]; ];
export const databaseVersion = migrations.length; export const databaseVersion = migrations.length;
+15 -3
View File
@@ -1,5 +1,6 @@
import { db } from '../db/index.js'; import { db } from '../db/index.js';
import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js'; import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js';
import { readTemperatureHistory } from './temperatures.js';
const windows: Record<CapacityRange, { duration: number; bucketMs: number }> = { const windows: Record<CapacityRange, { duration: number; bucketMs: number }> = {
'1h': { duration: 3_600_000, bucketMs: 60_000 }, '1h': { duration: 3_600_000, bucketMs: 60_000 },
@@ -15,10 +16,14 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
const count = duration / bucketMs; const count = duration / bucketMs;
const hosts = handle const hosts = handle
.prepare( .prepare(
`SELECT id,created_at FROM ec_hosts `SELECT id,name,created_at FROM ec_hosts
WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id` WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`
) )
.all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[]; .all(...(query.hostId ? [query.hostId] : [])) as {
id: string;
name: string;
created_at: string;
}[];
// The primary key bounds each host scan to the selected interval. Aggregate // The primary key bounds each host scan to the selected interval. Aggregate
// before crossing the worker boundary; do not return raw heartbeat rows. // before crossing the worker boundary; do not return raw heartbeat rows.
const samples = handle.prepare(`SELECT const samples = handle.prepare(`SELECT
@@ -69,5 +74,12 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
point.memoryBytesAvailable = null; point.memoryBytesAvailable = null;
} }
} }
return { range: query.range, startAt, endAt: now, bucketMs, points }; return {
range: query.range,
startAt,
endAt: now,
bucketMs,
points,
temperatures: readTemperatureHistory(query, hosts, startAt, now, bucketMs)
};
} }
+5
View File
@@ -23,6 +23,8 @@ import { logResultMetadata } from './log-results.js';
import { parseHostInventory } from './request-context.js'; import { parseHostInventory } from './request-context.js';
import { fleetSettings } from '../settings.js'; import { fleetSettings } from '../settings.js';
import { recordInstanceResources, validateInstanceResources } from './calibration.js'; import { recordInstanceResources, validateInstanceResources } from './calibration.js';
import { validateTemperatures } from '../../host-temperatures.js';
import { recordTemperatures } from './temperatures.js';
import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js'; import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js'; import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js';
import type { import type {
@@ -622,6 +624,8 @@ export function validateInventory(value: unknown): HostInventory {
throw new Error('Host inventory must be an object.'); throw new Error('Host inventory must be an object.');
const inventory = value as HostInventory; const inventory = value as HostInventory;
if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.'); if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.');
if (inventory.temperatures !== undefined)
inventory.temperatures = validateTemperatures(inventory.temperatures);
for (const key of ['hostname', 'agentVersion', 'os', 'architecture', 'cpu', 'dockerVersion']) { for (const key of ['hostname', 'agentVersion', 'os', 'architecture', 'cpu', 'dockerVersion']) {
if ( if (
inventory[key] !== undefined && inventory[key] !== undefined &&
@@ -924,6 +928,7 @@ export function heartbeat(hostId: string, value: unknown): Host {
function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void { function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
recordInstanceResources(hostId, inventory, timestamp); recordInstanceResources(hostId, inventory, timestamp);
recordTemperatures(hostId, inventory, timestamp);
database() database()
.prepare( .prepare(
`INSERT OR REPLACE INTO ec_host_resource_samples `INSERT OR REPLACE INTO ec_host_resource_samples
+96
View File
@@ -0,0 +1,96 @@
import type Database from 'better-sqlite3';
import { db } from '../db/index.js';
import { historyTemperatures } from '../../host-temperatures.js';
import type { HostInventory } from '../../hosts.js';
import type { CapacityQuery, TemperatureHistorySeries } from '../../capacity.js';
export function migrateTemperatures(handle: Database.Database): void {
handle.exec(`CREATE TABLE IF NOT EXISTS ec_host_temperature_samples (
host_id TEXT NOT NULL REFERENCES ec_hosts(id), bucket TEXT NOT NULL,
sensor_id TEXT NOT NULL, kind TEXT NOT NULL, label TEXT NOT NULL,
celsius REAL NOT NULL CHECK(celsius BETWEEN -100 AND 200),
PRIMARY KEY(host_id,bucket,sensor_id)
);
CREATE INDEX IF NOT EXISTS ec_host_temperatures_retention ON ec_host_temperature_samples(bucket);`);
}
export function recordTemperatures(
hostId: string,
inventory: HostInventory,
timestamp: string
): void {
const sensors = historyTemperatures(inventory.temperatures);
if (!sensors.length) return;
const bucket = new Date(Math.floor(Date.parse(timestamp) / 60_000) * 60_000).toISOString();
const insert = db().prepare(`INSERT INTO ec_host_temperature_samples
(host_id,bucket,sensor_id,kind,label,celsius) VALUES(?,?,?,?,?,?)
ON CONFLICT(host_id,bucket,sensor_id) DO UPDATE SET
celsius=MAX(celsius,excluded.celsius),kind=excluded.kind,label=excluded.label`);
for (const sensor of sensors)
insert.run(hostId, bucket, sensor.id, sensor.kind, sensor.label, sensor.celsius);
}
/** Runs on the capacity report worker, returning at most 168 points per series. */
export function readTemperatureHistory(
query: CapacityQuery,
hosts: { id: string; name: string; created_at: string }[],
startAt: number,
endAt: number,
bucketMs: number
): TemperatureHistorySeries[] {
const count = Math.round((endAt - startAt) / bucketMs);
const statement = db().prepare(`SELECT sensor_id,kind,label,
MIN(?,MAX(0,CAST((CAST(strftime('%s',bucket) AS INTEGER)*1000-?) / ? AS INTEGER))) AS period,
MAX(celsius) AS celsius
FROM ec_host_temperature_samples WHERE host_id=? AND bucket>=? AND bucket<=?
${query.hostId ? '' : "AND sensor_id='cpu'"}
GROUP BY sensor_id,period ORDER BY sensor_id,period`);
const result: TemperatureHistorySeries[] = [];
for (const host of hosts) {
const bySensor = new Map<string, TemperatureHistorySeries>();
const add = (id: string, kind: TemperatureHistorySeries['kind'], label: string) => {
const series = {
id: `${host.id}:${id}`,
hostId: host.id,
kind,
label,
points: Array.from({ length: count }, (_, i) => ({
at: startAt + i * bucketMs,
celsius: null as number | null
}))
};
bySensor.set(id, series);
result.push(series);
return series;
};
add('cpu', 'cpu', query.hostId ? 'CPU temperature' : host.name);
const rows = statement.all(
count - 1,
startAt,
bucketMs,
host.id,
new Date(
Math.max(
Math.floor(Date.parse(host.created_at) / 60_000) * 60_000,
Math.floor(startAt / 60_000) * 60_000
)
).toISOString(),
new Date(endAt).toISOString()
) as { sensor_id: string; kind: string; label: string; period: number; celsius: number }[];
for (const row of rows) {
let series = bySensor.get(row.sensor_id);
if (!series) {
if (!['nvme', 'chipset'].includes(row.kind) || bySensor.size >= 17) continue;
const kind = row.kind as 'nvme' | 'chipset';
const number = [...bySensor.values()].filter((s) => s.kind === kind).length + 1;
series = add(
row.sensor_id,
kind,
`${kind === 'nvme' ? 'NVMe drive' : 'Chipset'} ${number}`
);
}
if (series.points[row.period]) series.points[row.period].celsius = row.celsius;
}
}
return result;
}
+8
View File
@@ -556,6 +556,14 @@ function samplePolicies(s = fleetSettings()) {
value: iso(s.hostResourceKeepDays), value: iso(s.hostResourceKeepDays),
enabled: s.hostResourceKeepDays > 0 enabled: s.hostResourceKeepDays > 0
}, },
{
category: 'host-temperatures',
label: 'Host temperature samples',
table: 'ec_host_temperature_samples',
condition: 'bucket < ?',
value: iso(s.hostResourceKeepDays),
enabled: s.hostResourceKeepDays > 0
},
{ {
category: 'instance-resources', category: 'instance-resources',
label: 'Instance resource samples', label: 'Instance resource samples',
+8 -1
View File
@@ -1,5 +1,6 @@
<script lang="ts"> <script lang="ts">
import type { Host } from '$lib/hosts.js'; import type { Host } from '$lib/hosts.js';
import { cpuTemperature, formatTemperature } from '$lib/host-temperatures.js';
import { contextHref } from '$lib/navigation-context.js'; import { contextHref } from '$lib/navigation-context.js';
import { import {
hostListStatus, hostListStatus,
@@ -68,7 +69,13 @@
? '—' ? '—'
: `${host.playerCount}${host.unreportedInstances ? '+' : ''}`}</td : `${host.playerCount}${host.unreportedInstances ? '+' : ''}`}</td
> >
<td class:stale>{hostCpuUsage(host.inventory.cpuUsagePercent)}</td> <td class:stale
>{hostCpuUsage(host.inventory.cpuUsagePercent)}<span class="secondary"
>CPU temp: {formatTemperature(
cpuTemperature(host.inventory.temperatures)?.celsius
)}</span
></td
>
<td class:stale <td class:stale
>{hostUsedSpace( >{hostUsedSpace(
host.inventory.memoryTotal, host.inventory.memoryTotal,
@@ -0,0 +1,79 @@
<script lang="ts">
import type { Host } from '$lib/hosts.js';
import { cpuTemperature, formatTemperature, temperatureLabel } from '$lib/host-temperatures.js';
import { hostReadingsStale } from '$ui/host-overview.js';
import { onMount } from 'svelte';
let { host }: { host: Host } = $props();
let now = $state(Date.now());
const stale = $derived(hostReadingsStale(host, now));
const sensors = $derived(host.inventory.temperatures ?? []);
const cpu = $derived(cpuTemperature(sensors));
onMount(() => {
const clock = setInterval(() => (now = Date.now()), 15_000);
return () => clearInterval(clock);
});
</script>
<div class:stale>
{formatTemperature(cpu?.celsius)}{#if stale && cpu}
(last reported){/if}
{#if cpu}<span class="source">{temperatureLabel(cpu)}</span>{/if}
{#if sensors.length}
<details>
<summary>Temperature sensors</summary>
<ul>
{#each sensors as sensor (sensor.id)}
<li>
<span
>{temperatureLabel(sensor)}<small
>{sensor.device} · {sensor.label} · {sensor.id.slice(-6)}</small
></span
><span>{formatTemperature(sensor.celsius)}</span>
</li>
{/each}
</ul>
{#if stale}<p>Last reported readings; waiting for a fresh host heartbeat.</p>{/if}
</details>
{:else}<span class="source">No supported temperature sensors reported.</span>{/if}
</div>
<style>
.source,
small {
display: block;
color: var(--text-secondary);
font-size: var(--text-xs);
margin-top: var(--space-1);
}
.stale {
color: var(--text-secondary);
}
details {
margin-top: var(--space-2);
}
summary {
cursor: pointer;
font-size: var(--text-sm);
}
summary:focus-visible {
outline: 2px solid var(--accent-primary);
outline-offset: 3px;
}
ul {
padding: 0;
list-style: none;
margin: var(--space-3) 0;
display: grid;
gap: var(--space-3);
}
li {
display: flex;
justify-content: space-between;
flex-wrap: wrap;
gap: var(--space-2);
font-variant-numeric: tabular-nums;
}
p {
font-size: var(--text-xs);
}
</style>
@@ -0,0 +1,456 @@
<script lang="ts">
import type { Snippet } from 'svelte';
import type { CapacityHistory, TemperatureHistorySeries } from '$lib/capacity.js';
import { formatTemperature } from '$lib/host-temperatures.js';
import { timeFormat } from '$ui/stores/time-zone.js';
let {
history,
scope,
controls
}: { history: CapacityHistory; scope: string; controls?: Snippet } = $props();
let visibility = $state<Record<string, boolean>>({});
let selectedAt = $state<number | null>(null);
const series = $derived(history.temperatures);
const visible = $derived(series.filter((s) => visibility[s.id] ?? s.kind === 'cpu'));
const values = $derived(
series.flatMap((s) => s.points.flatMap((p) => (p.celsius === null ? [] : [p.celsius])))
);
const minimum = $derived(
values.length
? Math.floor((values.reduce((a, b) => Math.min(a, b), Infinity) - 5) / 10) * 10
: 0
);
const maximum = $derived(
values.length
? Math.ceil((values.reduce((a, b) => Math.max(a, b), -Infinity) + 5) / 10) * 10
: 100
);
const times = $derived(history.points.map((p) => p.at));
const measured = $derived(visible.some((s) => s.points.some((p) => p.celsius !== null)));
const selectedIndex = $derived.by(() => {
if (selectedAt !== null)
return times.reduce(
(best, at, i) =>
Math.abs(at - selectedAt!) < Math.abs(times[best] - selectedAt!) ? i : best,
0
);
for (let i = times.length - 1; i >= 0; i--)
if (visible.some((s) => s.points[i]?.celsius != null)) return i;
return Math.max(0, times.length - 1);
});
const selectedTime = $derived(times[selectedIndex]);
const readout = $derived(
visible
.map((s) => `${s.label}: ${formatTemperature(s.points[selectedIndex]?.celsius)}`)
.join('. ')
);
const ticks = [0, 0.25, 0.5, 0.75, 1];
function color(s: TemperatureHistorySeries) {
return `var(--chart-series-${(series.indexOf(s) % 6) + 1})`;
}
function dash(s: TemperatureHistorySeries) {
return ['', '7 4', '2 4'][Math.floor(series.indexOf(s) / 6) % 3];
}
function x(at: number) {
return (
8 +
984 *
Math.max(
0,
Math.min(
1,
(at - history.startAt) / Math.max(1, history.endAt - history.startAt)
)
)
);
}
function y(celsius: number) {
return 230 - (220 * (celsius - minimum)) / (maximum - minimum);
}
function segments(s: TemperatureHistorySeries) {
const result: { x: number; y: number }[][] = [];
let segment: { x: number; y: number }[] = [];
for (const point of s.points) {
if (point.celsius === null) {
if (segment.length) result.push(segment);
segment = [];
} else segment.push({ x: x(point.at), y: y(point.celsius) });
}
if (segment.length) result.push(segment);
return result;
}
function inspect(event: PointerEvent) {
if (!times.length || (event.type === 'pointermove' && event.pointerType === 'touch'))
return;
const rect = (event.currentTarget as HTMLElement).getBoundingClientRect();
selectedAt =
history.startAt +
Math.max(
0,
Math.min(1, (((event.clientX - rect.left) / rect.width) * 1000 - 8) / 984)
) *
(history.endAt - history.startAt);
}
function move(event: KeyboardEvent) {
if (!times.length || !['ArrowLeft', 'ArrowRight', 'Home', 'End'].includes(event.key))
return;
event.preventDefault();
const next =
event.key === 'Home'
? 0
: event.key === 'End'
? times.length - 1
: Math.max(
0,
Math.min(
times.length - 1,
selectedIndex + (event.key === 'ArrowLeft' ? -1 : 1)
)
);
selectedAt = times[next];
}
</script>
<section class="panel" id="temperatures" aria-labelledby="temperature-history-title">
<div class="heading">
<div>
<h2 id="temperature-history-title">Temperature history</h2>
<p>{scope} · Peak temperature per period</p>
</div>
{@render controls?.()}
</div>
<div class="legend" role="group" aria-label="Temperature series">
{#each series as sensor (sensor.id)}
<button
type="button"
aria-pressed={visibility[sensor.id] ?? sensor.kind === 'cpu'}
onclick={() =>
(visibility[sensor.id] = !(visibility[sensor.id] ?? sensor.kind === 'cpu'))}
>
<svg viewBox="0 0 24 8" aria-hidden="true"
><line
x1="0"
x2="24"
y1="4"
y2="4"
stroke={color(sensor)}
stroke-dasharray={dash(sensor)}
/></svg
>
{sensor.label}
</button>
{/each}
</div>
<div class="axis-title">Temperature (°C)</div>
<div class="chart-grid">
<div class="y-axis" aria-hidden="true">
{#each ticks as tick}<span style:top={`${(1 - tick) * 100}%`}
>{minimum + (maximum - minimum) * tick}</span
>{/each}
</div>
<div
class="plot"
role="slider"
tabindex="0"
aria-label="Temperature history. Use left and right arrows to inspect a period."
aria-valuemin="0"
aria-valuemax={Math.max(0, times.length - 1)}
aria-valuenow={selectedIndex}
aria-valuetext={selectedTime === undefined
? 'No temperature history'
: `${$timeFormat.period(selectedTime)}. ${readout || 'No sensors selected'}`}
onpointermove={inspect}
onpointerdown={inspect}
onkeydown={move}
>
<svg viewBox="0 0 1000 240" preserveAspectRatio="none" aria-hidden="true">
{#each ticks as tick}<line
class="grid-line"
x1="0"
x2="1000"
y1={y(minimum + (maximum - minimum) * tick)}
y2={y(minimum + (maximum - minimum) * tick)}
/>{/each}
{#each visible as sensor (sensor.id)}
<g data-series={sensor.id}>
{#each segments(sensor) as segment}
<path
class="trend"
stroke={color(sensor)}
stroke-dasharray={dash(sensor)}
d={segment
.map((point, i) => `${i ? 'L' : 'M'}${point.x},${point.y}`)
.join(' ')}
/>
{#if segment.length === 1}<circle
cx={segment[0].x}
cy={segment[0].y}
r="3"
fill={color(sensor)}
/>{/if}
{/each}
{#if sensor.points[selectedIndex]?.celsius != null}<circle
class="selected-point"
cx={x(selectedTime)}
cy={y(sensor.points[selectedIndex].celsius!)}
r="4"
fill={color(sensor)}
/>{/if}
</g>
{/each}
{#if selectedTime !== undefined && measured}<line
class="cursor"
x1={x(selectedTime)}
x2={x(selectedTime)}
y1="5"
y2="235"
/>{/if}
</svg>
{#if !measured}<div class="empty-chart">
<strong
>{visible.length || !series.length
? 'No temperature readings in this period'
: 'Select a sensor to show history'}</strong
>
{#if visible.length || !series.length}<span
>Readings appear when an agent reports supported temperature sensors.</span
>{/if}
</div>{/if}
</div>
<div class="x-axis" aria-hidden="true">
<span>{$timeFormat.period(history.startAt)}</span><span
>{$timeFormat.period(history.endAt)}</span
>
</div>
</div>
{#if selectedTime !== undefined && visible.length}
<div class="readout" aria-label="Selected temperature period">
<span>{$timeFormat.period(selectedTime)}</span>
<dl>
{#each visible as sensor (sensor.id)}<div>
<dt>{sensor.label}</dt>
<dd>{formatTemperature(sensor.points[selectedIndex]?.celsius)}</dd>
</div>{/each}
</dl>
</div>
{/if}
<details>
<summary>About this chart</summary>
<p>
Each CPU line represents one host: its hottest reported package or physical die, falling
back to its hottest core. Select a host to compare its CPU, NVMe drives and chipset.
Per-minute peaks are retained; longer periods show their highest reading. Gaps mean no
measurement was received in that period. Temperatures include all workloads and are
unaffected by the game filter. Colors identify series; they do not indicate safe or
unsafe temperatures.
</p>
</details>
</section>
<style>
.panel {
background: var(--surface);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-lg);
padding: var(--space-5);
min-width: 0;
}
h2 {
margin: 0;
font-size: var(--text-lg);
}
.heading {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: var(--space-4);
flex-wrap: wrap;
}
.heading p {
color: var(--text-secondary);
font-size: var(--text-sm);
margin: var(--space-2) 0 0;
}
.legend {
display: flex;
flex-wrap: wrap;
gap: var(--space-2) var(--space-5);
margin: var(--space-4) 0;
}
.legend button {
display: flex;
align-items: center;
gap: var(--space-2);
background: none;
border: 0;
padding: var(--space-2) 0;
color: var(--text-primary);
font: inherit;
font-size: var(--text-sm);
cursor: pointer;
overflow-wrap: anywhere;
max-width: 100%;
}
.legend button[aria-pressed='false'] {
color: var(--text-secondary);
}
.legend button[aria-pressed='false'] svg {
opacity: 0.35;
}
.legend svg {
width: 24px;
height: 8px;
flex-shrink: 0;
}
.legend line {
stroke-width: 3;
}
button:focus-visible,
.plot:focus-visible,
summary:focus-visible {
outline: 2px solid var(--accent-primary);
outline-offset: 3px;
}
.axis-title,
.x-axis,
.y-axis {
color: var(--text-tertiary);
font-size: var(--text-xs);
}
.chart-grid {
display: grid;
grid-template-columns: 42px minmax(0, 1fr);
gap: 10px;
margin-top: 14px;
}
.y-axis {
position: relative;
height: 220px;
margin-top: 10px;
}
.y-axis span {
position: absolute;
right: 0;
transform: translateY(-50%);
}
.plot {
position: relative;
height: 240px;
cursor: crosshair;
touch-action: pan-y;
}
.plot > svg {
display: block;
width: 100%;
height: 100%;
overflow: visible;
}
.plot path,
.plot line {
vector-effect: non-scaling-stroke;
}
.grid-line {
stroke: var(--border-subtle);
stroke-width: 1;
}
.trend {
fill: none;
stroke-width: 2;
stroke-linejoin: round;
stroke-linecap: round;
}
.selected-point {
stroke: var(--surface);
stroke-width: 2;
}
.cursor {
stroke: var(--text-tertiary);
stroke-width: 1;
stroke-dasharray: 4 5;
}
.x-axis {
grid-column: 2;
display: flex;
justify-content: space-between;
gap: var(--space-4);
}
.x-axis span:last-child {
text-align: right;
}
.empty-chart {
position: absolute;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
flex-direction: column;
gap: var(--space-2);
padding: var(--space-4);
text-align: center;
pointer-events: none;
}
.empty-chart strong {
font-size: var(--text-sm);
font-weight: var(--weight-medium);
}
.empty-chart span {
font-size: var(--text-xs);
color: var(--text-secondary);
}
.readout {
display: flex;
flex-wrap: wrap;
gap: var(--space-4);
font-size: var(--text-xs);
margin: var(--space-4) 0;
}
.readout > span,
dt {
color: var(--text-secondary);
}
dl {
display: flex;
flex-wrap: wrap;
gap: var(--space-3) var(--space-5);
margin: 0;
}
dl > div {
display: flex;
flex-wrap: wrap;
min-width: 0;
gap: var(--space-2);
}
dt {
overflow-wrap: anywhere;
}
dd {
margin: 0;
font-variant-numeric: tabular-nums;
}
summary {
cursor: pointer;
font-size: var(--text-xs);
color: var(--text-secondary);
padding: var(--space-2) 0;
}
details p {
color: var(--text-secondary);
font-size: var(--text-xs);
line-height: 1.65;
}
@media (max-width: 640px) {
.panel {
padding: var(--space-4);
}
.chart-grid {
grid-template-columns: 32px minmax(0, 1fr);
gap: 8px;
}
}
@media (pointer: coarse) {
.legend button {
min-height: 44px;
}
}
</style>
+6
View File
@@ -78,6 +78,12 @@
/* Metrics: unknown uses a pattern as well as color. */ /* Metrics: unknown uses a pattern as well as color. */
--chart-active: var(--accent-primary); --chart-active: var(--accent-primary);
--chart-series-1: var(--accent-primary);
--chart-series-2: #b99af7;
--chart-series-3: #69c6ba;
--chart-series-4: #dfb76c;
--chart-series-5: #df91b6;
--chart-series-6: #bbc7db;
--chart-idle: #2b3442; --chart-idle: #2b3442;
--chart-unknown-base: #605b64; --chart-unknown-base: #605b64;
--chart-unknown-stripe: #a7a0ad; --chart-unknown-stripe: #a7a0ad;
+15 -10
View File
@@ -17,6 +17,7 @@
import Select from '$ui/components/primitives/Select.svelte'; import Select from '$ui/components/primitives/Select.svelte';
import CapacityHosts from '$ui/components/capacity/CapacityHosts.svelte'; import CapacityHosts from '$ui/components/capacity/CapacityHosts.svelte';
import CapacityChart from '$ui/components/capacity/CapacityChart.svelte'; import CapacityChart from '$ui/components/capacity/CapacityChart.svelte';
import TemperatureChart from '$ui/components/capacity/TemperatureChart.svelte';
import RebalancePanel from '$ui/components/capacity/RebalancePanel.svelte'; import RebalancePanel from '$ui/components/capacity/RebalancePanel.svelte';
import Freshness from '$ui/components/Freshness.svelte'; import Freshness from '$ui/components/Freshness.svelte';
import { plural } from '$ui/plural.js'; import { plural } from '$ui/plural.js';
@@ -319,16 +320,20 @@
: 'Waiting for host reports.'} Refreshes every 30 seconds while visible. : 'Waiting for host reports.'} Refreshes every 30 seconds while visible.
</p> </p>
</details> </details>
<CapacityChart history={report.history} scope={scopeName}> {#snippet historyRanges()}<nav class="ranges" aria-label="History range">
{#snippet controls()}<nav class="ranges" aria-label="History range"> {#each ranges as range}<a
{#each ranges as range}<a href={scopeUrl({ range: range.value })}
href={scopeUrl({ range: range.value })} aria-current={report.query.range === range.value ? 'page' : undefined}
aria-current={report.query.range === range.value ? 'page' : undefined} data-sveltekit-keepfocus
data-sveltekit-keepfocus data-sveltekit-noscroll>{range.label}</a
data-sveltekit-noscroll>{range.label}</a >{/each}
>{/each} </nav>{/snippet}
</nav>{/snippet} <CapacityChart history={report.history} scope={scopeName} controls={historyRanges} />
</CapacityChart> {#key report.query.hostId}<TemperatureChart
history={report.history}
scope={scopeName}
controls={historyRanges}
/>{/key}
</div> </div>
<style> <style>
+3
View File
@@ -28,6 +28,7 @@
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte'; import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
import HostPreparation from '$ui/components/HostPreparation.svelte'; import HostPreparation from '$ui/components/HostPreparation.svelte';
import CapacitySamples from '$ui/components/CapacitySamples.svelte'; import CapacitySamples from '$ui/components/CapacitySamples.svelte';
import HostTemperatures from '$ui/components/HostTemperatures.svelte';
import ServerCard from '$ui/components/ServerCard.svelte'; import ServerCard from '$ui/components/ServerCard.svelte';
import type { ServerObservation } from '$lib/server-observation.js'; import type { ServerObservation } from '$lib/server-observation.js';
import { syncServerTime } from '$ui/stores/servers.js'; import { syncServerTime } from '$ui/stores/servers.js';
@@ -300,6 +301,8 @@
: `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined} : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined}
(last reported){/if} (last reported){/if}
</dd> </dd>
<dt>CPU temperature</dt>
<dd><HostTemperatures {host} /></dd>
<dt>Memory available / total</dt> <dt>Memory available / total</dt>
<dd> <dd>
{bytes(host.inventory.memoryAvailable)} / {bytes( {bytes(host.inventory.memoryAvailable)} / {bytes(
+58 -3
View File
@@ -35,7 +35,7 @@ import urllib.parse
import urllib.request import urllib.request
import uuid import uuid
VERSION = "1.9.3" VERSION = "1.10.0"
LABEL = "io.electric-cherry" LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024 MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000 MANIFEST_MAX_ENTRIES = 10000
@@ -63,6 +63,7 @@ UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-ch
UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json") UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json") UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json") AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
HWMON_ROOT = Path("/sys/class/hwmon")
ROOT_MARKER = ".electric-cherry.json" ROOT_MARKER = ".electric-cherry.json"
@@ -1308,6 +1309,59 @@ class Agent:
return None return None
return round(100 * (total - deltas[3] - deltas[4]) / total, 1) return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
def temperatures(self):
"""Best-effort, read-only samples from known millidegree-Celsius drivers."""
samples = []
try:
chips = sorted(HWMON_ROOT.glob("hwmon*"))[:64]
except OSError:
return samples
for chip in chips:
try:
device = (chip / "name").read_text().strip()
if device not in ("coretemp", "k10temp", "nvme") and not device.startswith("pch_"):
continue
# hwmon numbers can change on reboot. The physical sysfs path
# and channel identify separate CPU sockets and NVMe devices.
identity = re.sub(r"/(?:hwmon/)?hwmon\d+$", "", str(chip.resolve()))
inputs = sorted(chip.glob("temp[0-9]*_input"))[:128]
except (OSError, ValueError):
continue
for source in inputs:
try:
channel = source.name.removesuffix("_input")
label_path = chip / (channel + "_label")
label = label_path.read_text().strip() if label_path.exists() else channel
if not label or len(label) > 120:
continue
fault = chip / (channel + "_fault")
enabled = chip / (channel + "_enable")
if (fault.exists() and int(fault.read_text()) != 0) or (enabled.exists() and int(enabled.read_text()) == 0):
continue
if device == "coretemp":
kind = "cpu-package" if label.startswith("Package id ") else "cpu-core" if label.startswith("Core ") else None
elif device == "k10temp":
# Tctl can contain a fan-control offset. Never present
# it as the physical die temperature or a fleet metric.
kind = "cpu-die" if label == "Tdie" or re.fullmatch(r"Tccd\d+", label) else None
else:
kind = "nvme" if device == "nvme" else "chipset"
if kind is None:
continue
celsius = int(source.read_text()) / 1000
if not -100 <= celsius <= 200:
continue
sensor_id = hashlib.sha256((identity + "/" + channel).encode()).hexdigest()[:24]
samples.append({"id": sensor_id, "device": device, "label": label,
"kind": kind, "celsius": round(celsius, 1)})
if len(samples) >= 128:
return samples
except (OSError, ValueError):
# Read permission alone does not imply data is available
# (e.g. ENODATA), and a device can disappear during a read.
continue
return samples
def instance_resources(self, observations=None): def instance_resources(self, observations=None):
"""One bounded Docker stats batch; unavailable samples are never zero load.""" """One bounded Docker stats batch; unavailable samples are never zero load."""
ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true", ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true",
@@ -1392,6 +1446,7 @@ class Agent:
def inventory(self): def inventory(self):
cpu_usage = self.cpu_usage_percent() cpu_usage = self.cpu_usage_percent()
temperatures = self.temperatures()
usage = shutil.disk_usage(self.root) usage = shutil.disk_usage(self.root)
memory = {} memory = {}
try: try:
@@ -1516,9 +1571,9 @@ class Agent:
if line.startswith("PRETTY_NAME="): if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"') os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory() update = self.update_inventory()
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) capabilities = ["host.temperatures", "library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
plutonium_cache = self.plutonium_cache_inventory() plutonium_cache = self.plutonium_cache_inventory()
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "temperatures": temperatures, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False): def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
health = self.inspect(instance_id, generation) health = self.inspect(instance_id, generation)
+2 -2
View File
@@ -1,8 +1,8 @@
{ {
"version": "1.9.3", "version": "1.10.0",
"protocolVersion": 1, "protocolVersion": 1,
"platform": "debian-amd64", "platform": "debian-amd64",
"sha256": "a0baaab0f0a60e848300e63f31eeefff6293d8b749b5ca9397c36e8e75ff4a93", "sha256": "1a110c92ae875ee7cd060ac7de82260b63d776d2a865e57356d629e0a99573c7",
"updaterProtocolVersion": 2, "updaterProtocolVersion": 2,
"updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb", "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb",
"installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231" "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231"
-8
View File
@@ -1,8 +0,0 @@
-----BEGIN PGP SIGNATURE-----
iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCas2mehsUgAAAAAAEAA5t
YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfpvyAD/THg0KsH9kDSCF/I2vdOV
h2BBe9r1JWk1F0mFy2njfvAA/R9mMAMdJRkFIxMNgkIl17nQrUq4TpXQsKSSujXZ
u5oK
=GspF
-----END PGP SIGNATURE-----
+110
View File
@@ -0,0 +1,110 @@
import { afterEach, expect, it } from 'vitest';
import { flushSync, mount, unmount } from 'svelte';
import TemperatureChart from '../../src/lib/ui/components/capacity/TemperatureChart.svelte';
import type { CapacityHistory } from '../../src/lib/capacity.js';
let component: ReturnType<typeof mount> | undefined;
afterEach(async () => {
if (component) await unmount(component);
component = undefined;
document.body.replaceChildren();
});
function history(): CapacityHistory {
return {
range: '1h',
startAt: 1000,
endAt: 4000,
bucketMs: 1000,
points: [1000, 2000, 3000].map((at) => ({
at,
cpuCoresAvailable: null,
memoryBytesAvailable: null,
reportingHosts: 0,
expectedHosts: 1
})),
temperatures: [
{
id: 'host:cpu',
hostId: 'host',
label: 'CPU temperature',
kind: 'cpu',
points: [65, null, 67].map((celsius, i) => ({ at: 1000 + i * 1000, celsius }))
},
{
id: 'host:drive',
hostId: 'host',
label: 'NVMe drive 1',
kind: 'nvme',
points: [49, 50.9, 52].map((celsius, i) => ({ at: 1000 + i * 1000, celsius }))
}
]
};
}
function show(report = history()) {
component = mount(TemperatureChart, {
target: document.body,
props: { history: report, scope: 'ec-2' }
});
flushSync();
}
it('defaults to CPU and toggles device lines and readouts together', () => {
show();
const buttons = document.querySelectorAll<HTMLButtonElement>('.legend button');
expect(buttons[0].getAttribute('aria-pressed')).toBe('true');
expect(buttons[1].getAttribute('aria-pressed')).toBe('false');
expect(document.querySelectorAll('[data-series]')).toHaveLength(1);
// A missing middle bucket splits the CPU line into two isolated segments.
expect(document.querySelectorAll('[data-series="host:cpu"] path')).toHaveLength(2);
buttons[1].click();
flushSync();
expect(document.querySelectorAll('[data-series]')).toHaveLength(2);
expect(document.querySelector('.readout')?.textContent).toContain('52 °C');
buttons[0].click();
buttons[1].click();
flushSync();
expect(document.querySelectorAll('[data-series]')).toHaveLength(0);
expect(document.body.textContent).toContain('Select a sensor to show history');
expect(document.querySelector('.readout')).toBeNull();
});
it('supports keyboard and pointer inspection without turning missing readings into zero', () => {
show();
const plot = document.querySelector<HTMLElement>('[role="slider"]')!;
expect(plot.getAttribute('aria-valuenow')).toBe('2');
plot.dispatchEvent(new KeyboardEvent('keydown', { key: 'Home', bubbles: true }));
flushSync();
expect(plot.getAttribute('aria-valuetext')).toContain('65 °C');
plot.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowRight', bubbles: true }));
flushSync();
expect(plot.getAttribute('aria-valuetext')).toContain('Unavailable');
expect(document.querySelector('.readout')?.textContent).not.toContain('0 °C');
Object.defineProperty(plot, 'getBoundingClientRect', {
value: () => ({ left: 0, width: 1000 })
});
plot.dispatchEvent(
new PointerEvent('pointerdown', { clientX: 8, pointerType: 'touch', bubbles: true })
);
flushSync();
expect(plot.getAttribute('aria-valuenow')).toBe('0');
expect(document.querySelector('.readout')?.textContent).toContain('65 °C');
});
it('shows unsupported hosts as unavailable', () => {
const report = history();
report.temperatures[0].points = report.temperatures[0].points.map((point) => ({
...point,
celsius: null
}));
show(report);
expect(document.body.textContent).toContain('No temperature readings in this period');
expect(document.body.textContent).toContain('Unavailable');
});
it('plots a genuine zero-degree reading', () => {
const report = history();
report.temperatures[0].points[2].celsius = 0;
show(report);
expect(document.querySelector('.readout')?.textContent).toContain('0 °C');
expect(document.querySelector('.empty-chart')).toBeNull();
});
+95
View File
@@ -34,6 +34,101 @@ async function sample(id: string, at: number, cpu = 25, memory = 4 * GiB, cores
} }
describe('capacity history', () => { describe('capacity history', () => {
it('keeps separate CPU series, device drill-down, real zero and missing temperature gaps', async () => {
await host('one');
await host('two');
await host('unsupported');
await host('removed');
const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
const cpu = (celsius: number) => ({
id: 'package',
kind: 'cpu-package' as const,
device: 'coretemp',
label: 'Package id 0',
celsius
});
const drive = {
id: 'drive',
kind: 'nvme' as const,
device: 'nvme',
label: 'Composite',
celsius: 50.9
};
recordTemperatures(
'one',
{ temperatures: [cpu(65), drive] },
new Date(epoch - 55_000).toISOString()
);
recordTemperatures(
'one',
{ temperatures: [cpu(75)] },
new Date(epoch - 30_000).toISOString()
);
recordTemperatures(
'one',
{ temperatures: [cpu(0)] },
new Date(epoch - 180_000).toISOString()
);
recordTemperatures(
'two',
{ temperatures: [cpu(60)] },
new Date(epoch - 40_000).toISOString()
);
recordTemperatures(
'removed',
{ temperatures: [cpu(99)] },
new Date(epoch - 40_000).toISOString()
);
const { db } = await import('../../src/lib/server/db/index.js');
db()
.prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?')
.run(new Date(epoch).toISOString(), 'removed');
const { readCapacityHistory } =
await import('../../src/lib/server/hosts/capacity-history.js');
const fleet = readCapacityHistory(query, epoch).temperatures;
expect(fleet.map((s) => s.label)).toEqual(['one', 'two', 'unsupported']);
expect(fleet[0].points.slice(-3).map((p) => p.celsius)).toEqual([0, null, 75]);
expect(fleet[1].points.at(-1)?.celsius).toBe(60);
expect(fleet[2].points.every((p) => p.celsius === null)).toBe(true);
const selected = readCapacityHistory(
{ ...query, hostId: 'one', game: 't5' },
epoch
).temperatures;
expect(selected.map((s) => s.kind)).toEqual(['cpu', 'nvme']);
expect(selected[1].points.at(-1)?.celsius).toBe(50.9);
expect(
readCapacityHistory({ ...query, hostId: 'one', game: 't6' }, epoch).temperatures
).toEqual(selected);
expect(readCapacityHistory({ ...query, hostId: 'missing' }, epoch).temperatures).toEqual(
[]
);
});
it('preserves temperature peaks for long ranges and includes a host enrolled during the current minute', async () => {
await host('new', epoch - 20_000);
const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
recordTemperatures(
'new',
{
temperatures: [
{
id: 'pkg',
device: 'coretemp',
kind: 'cpu-package',
label: 'Package id 0',
celsius: 65
}
]
},
new Date(epoch - 10_000).toISOString()
);
const { readCapacityHistory } =
await import('../../src/lib/server/hosts/capacity-history.js');
expect(readCapacityHistory(query, epoch).temperatures[0].points.at(-1)?.celsius).toBe(65);
const long = readCapacityHistory({ ...query, range: '7d' }, epoch).temperatures[0];
expect(long.points).toHaveLength(168);
expect(long.points.at(-1)?.celsius).toBe(65);
});
it('averages each host before summing fleet headroom and exposes missing observations as gaps', async () => { it('averages each host before summing fleet headroom and exposes missing observations as gaps', async () => {
await host('one'); await host('one');
await host('two'); await host('two');
+151
View File
@@ -0,0 +1,151 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import {
cpuTemperature,
historyTemperatures,
validateTemperatures,
type HostTemperatureSensor
} from '../../src/lib/host-temperatures.js';
const sensor = (changes: Partial<HostTemperatureSensor> = {}): HostTemperatureSensor => ({
id: 'package',
device: 'coretemp',
label: 'Package id 0',
kind: 'cpu-package',
celsius: 65,
...changes
});
describe('temperature readings', () => {
it('keeps valid sensors while dropping malformed, duplicate and out-of-range entries', () => {
const good = sensor();
const readings = [
good,
good,
null,
'bad',
...[NaN, Infinity, -Infinity, null, '65', true, -101, 201].map((celsius) => ({
...good,
id: 'invalid',
celsius
})),
{ ...good, id: 'x'.repeat(101) },
{ ...good, label: '' },
{ ...good, kind: 'cpu-control' },
sensor({ id: 'zero', celsius: 0 })
];
expect(validateTemperatures(readings)).toEqual([good, sensor({ id: 'zero', celsius: 0 })]);
expect(validateTemperatures({})).toEqual([]);
expect(validateTemperatures(undefined)).toEqual([]);
});
it('chooses the hottest package and uses die/core readings only as fallbacks', () => {
const packages = [sensor(), sensor({ id: 'socket2', label: 'Package id 1', celsius: 72 })];
const core = sensor({ id: 'core', label: 'Core 0', kind: 'cpu-core', celsius: 80 });
const die = sensor({ id: 'die', label: 'Tdie', kind: 'cpu-die', celsius: 64 });
expect(cpuTemperature([...packages, core])?.celsius).toBe(72);
expect(cpuTemperature([die, core])?.celsius).toBe(64);
expect(cpuTemperature([core])?.celsius).toBe(80);
expect(cpuTemperature()).toBeNull();
const drive = sensor({
id: 'drive',
kind: 'nvme',
device: 'nvme',
label: 'Composite',
celsius: 50.9
});
expect(
historyTemperatures([
...packages,
core,
drive,
{ ...drive, id: 'internal', label: 'Sensor 1' }
])
).toEqual([{ ...packages[1], id: 'cpu', label: 'CPU temperature' }, drive]);
});
});
describe('temperature persistence', () => {
let fixture: Fixture;
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
});
afterEach(async () => {
vi.restoreAllMocks();
await fixture.cleanup();
});
it('accepts older agents and faulty temperature payloads without losing host metrics', async () => {
const { createHost, heartbeat, validateInventory } =
await import('../../src/lib/server/hosts/index.js');
const host = createHost({ name: 'ec-2' });
expect(validateInventory({ cpuUsagePercent: 25 }).temperatures).toBeUndefined();
expect(
heartbeat(host.id, {
cpuUsagePercent: 25,
temperatures: [sensor(), { celsius: 'bad' }]
}).inventory
).toMatchObject({ cpuUsagePercent: 25, temperatures: [sensor()] });
expect(
heartbeat(host.id, { cpuUsagePercent: 30, temperatures: [] }).inventory.temperatures
).toEqual([]);
expect(
heartbeat(host.id, { cpuUsagePercent: 35, temperatures: 'bad' }).inventory.temperatures
).toEqual([]);
const { db } = await import('../../src/lib/server/db/index.js');
expect(
db()
.prepare('SELECT celsius FROM ec_host_temperature_samples WHERE host_id=?')
.all(host.id)
).toEqual([{ celsius: 65 }]);
});
it('retains one peak per minute and applies host history retention, including offline hosts', async () => {
const { createHost } = await import('../../src/lib/server/hosts/index.js');
const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { fleetSettings, saveFleetSettings } =
await import('../../src/lib/server/settings.js');
const { pruneSampleHistory } = await import('../../src/lib/server/retention.js');
const host = createHost({ name: 'ec-2' });
const minute = Math.floor((Date.now() - 3 * 86400_000) / 60_000) * 60_000;
for (const [i, celsius] of [65, 78, 67].entries())
recordTemperatures(
host.id,
{ temperatures: [sensor({ celsius })] },
new Date(minute + i * 15_000).toISOString()
);
expect(db().prepare('SELECT celsius FROM ec_host_temperature_samples').all()).toEqual([
{ celsius: 78 }
]);
saveFleetSettings({ ...fleetSettings(), hostResourceKeepDays: 0 });
pruneSampleHistory();
expect(db().prepare('SELECT COUNT(*) AS n FROM ec_host_temperature_samples').get()).toEqual(
{ n: 1 }
);
recordTemperatures(host.id, { temperatures: [sensor()] }, new Date().toISOString());
saveFleetSettings({ ...fleetSettings(), hostResourceKeepDays: 1 });
pruneSampleHistory();
expect(db().prepare('SELECT celsius FROM ec_host_temperature_samples').all()).toEqual([
{ celsius: 65 }
]);
});
it('upgrades an existing database without rewriting its resource samples', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { createHost } = await import('../../src/lib/server/hosts/index.js');
const host = createHost({ name: 'old' });
db()
.prepare(
'INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_usage_percent) VALUES(?,?,?)'
)
.run(host.id, '2026-10-13T12:00:00Z', 25);
db().exec('DROP TABLE ec_host_temperature_samples; PRAGMA user_version=22;');
closeDb();
expect(db().prepare('SELECT * FROM ec_host_temperature_samples').all()).toEqual([]);
expect(
db().prepare('SELECT cpu_usage_percent FROM ec_host_resource_samples').all()
).toEqual([{ cpu_usage_percent: 25 }]);
expect(db().pragma('foreign_key_check')).toEqual([]);
});
});
+5
View File
@@ -43,6 +43,7 @@ const host = (changes: Partial<CapacityHost> = {}): CapacityHost => ({
...changes ...changes
}); });
const history: CapacityHistory = { const history: CapacityHistory = {
temperatures: [],
range: '24h', range: '24h',
startAt: 1000, startAt: 1000,
endAt: 5000, endAt: 5000,
@@ -112,6 +113,10 @@ it('keeps current estimates and host comparison before history with scoped histo
expect(body).not.toContain('Manage hosts'); expect(body).not.toContain('Manage hosts');
expect(body).toContain('Updates every 30 seconds'); expect(body).toContain('Updates every 30 seconds');
expect(body).not.toContain('>Refresh<'); expect(body).not.toContain('>Refresh<');
expect(body.indexOf('id="temperature-history-title"')).toBeGreaterThan(
body.indexOf('id="capacity-history-title"')
);
expect(body.match(/aria-label="History range"/g)).toHaveLength(2);
}); });
it('keeps fleet-wide queue scope explicit and preserves unavailable estimates', () => { it('keeps fleet-wide queue scope explicit and preserves unavailable estimates', () => {
+35
View File
@@ -9,6 +9,7 @@ import {
stableHostOrder stableHostOrder
} from '../../src/lib/ui/host-overview.js'; } from '../../src/lib/ui/host-overview.js';
import HostList from '../../src/lib/ui/components/HostList.svelte'; import HostList from '../../src/lib/ui/components/HostList.svelte';
import HostTemperatures from '../../src/lib/ui/components/HostTemperatures.svelte';
const now = Date.parse('2026-10-06T12:00:00Z'); const now = Date.parse('2026-10-06T12:00:00Z');
function host(id = 'ec-1', changes: Partial<Host> = {}): Host { function host(id = 'ec-1', changes: Partial<Host> = {}): Host {
@@ -43,6 +44,40 @@ function host(id = 'ec-1', changes: Partial<Host> = {}): Host {
}; };
} }
it('shows CPU temperature with stale status and keeps sensor details expandable', () => {
const target = host('ec-2', {
connected: false,
inventory: {
cpuUsagePercent: 25,
temperatures: [
{
id: 'cpu',
kind: 'cpu-package',
device: 'coretemp',
label: 'Package id 0',
celsius: 65
}
]
}
});
const list = render(HostList, {
props: {
hosts: [{ ...target, serverCount: 0, playerCount: 0, unreportedInstances: 0 }],
now
}
}).body;
expect(list).toContain('CPU temp: 65 °C');
expect(list).toContain('Resource readings stale');
const detail = render(HostTemperatures, { props: { host: target } }).body;
expect(detail).toContain('(last reported)');
expect(detail).toContain('Temperature sensors');
expect(detail).toContain('CPU package 0');
expect(detail).not.toMatch(/<details[^>]*open/);
const missing = render(HostTemperatures, { props: { host: host() } }).body;
expect(missing).toContain('Unavailable');
expect(missing).not.toContain('0 °C');
});
it('does not show an old Ready state as available when the heartbeat becomes stale', () => { it('does not show an old Ready state as available when the heartbeat becomes stale', () => {
expect(hostListStatus(host(), now).label).toBe('Ready'); expect(hostListStatus(host(), now).label).toBe('Ready');
expect(hostReadingsStale(host(), now + 90_001)).toBe(true); expect(hostReadingsStale(host(), now + 90_001)).toBe(true);