Add host temperature telemetry and history

src/lib/server/db/migrations.ts

@@ -24,6 +24,7 @@ import { migrateLeaderboardReports } from './leaderboard-reports.js';
 import { migratePlayerScoreTotal } from './player-score-total.js';
 import { migrateBank } from './bank.js';
 import { migrateBankCredentials } from './bank-credentials.js';
+import { migrateTemperatures } from '../hosts/temperatures.js';
 import type { Migration } from './migration-runner.js';
 
 // Append only: never renumber entries or change an applied migration to add an
@@ -118,7 +119,8 @@ export const migrations: readonly Migration[] = [
     },
     { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank },
     { version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials },
-    { version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions }
+    { version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions },
+    { version: 23, name: 'host temperature history', run: migrateTemperatures }
 ];
 
 export const databaseVersion = migrations.length;

src/lib/server/hosts/capacity-history.ts

@@ -1,5 +1,6 @@
 import { db } from '../db/index.js';
 import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js';
+import { readTemperatureHistory } from './temperatures.js';
 
 const windows: Record<CapacityRange, { duration: number; bucketMs: number }> = {
     '1h': { duration: 3_600_000, bucketMs: 60_000 },
@@ -15,10 +16,14 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
     const count = duration / bucketMs;
     const hosts = handle
         .prepare(
-            `SELECT id,created_at FROM ec_hosts
+            `SELECT id,name,created_at FROM ec_hosts
         WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`
         )
-        .all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[];
+        .all(...(query.hostId ? [query.hostId] : [])) as {
+        id: string;
+        name: string;
+        created_at: string;
+    }[];
     // The primary key bounds each host scan to the selected interval. Aggregate
     // before crossing the worker boundary; do not return raw heartbeat rows.
     const samples = handle.prepare(`SELECT
@@ -69,5 +74,12 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
             point.memoryBytesAvailable = null;
         }
     }
-    return { range: query.range, startAt, endAt: now, bucketMs, points };
+    return {
+        range: query.range,
+        startAt,
+        endAt: now,
+        bucketMs,
+        points,
+        temperatures: readTemperatureHistory(query, hosts, startAt, now, bucketMs)
+    };
 }

src/lib/server/hosts/index.ts

@@ -23,6 +23,8 @@ import { logResultMetadata } from './log-results.js';
 import { parseHostInventory } from './request-context.js';
 import { fleetSettings } from '../settings.js';
 import { recordInstanceResources, validateInstanceResources } from './calibration.js';
+import { validateTemperatures } from '../../host-temperatures.js';
+import { recordTemperatures } from './temperatures.js';
 import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
 import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js';
 import type {
@@ -622,6 +624,8 @@ export function validateInventory(value: unknown): HostInventory {
         throw new Error('Host inventory must be an object.');
     const inventory = value as HostInventory;
     if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.');
+    if (inventory.temperatures !== undefined)
+        inventory.temperatures = validateTemperatures(inventory.temperatures);
     for (const key of ['hostname', 'agentVersion', 'os', 'architecture', 'cpu', 'dockerVersion']) {
         if (
             inventory[key] !== undefined &&
@@ -924,6 +928,7 @@ export function heartbeat(hostId: string, value: unknown): Host {
 
 function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
     recordInstanceResources(hostId, inventory, timestamp);
+    recordTemperatures(hostId, inventory, timestamp);
     database()
         .prepare(
             `INSERT OR REPLACE INTO ec_host_resource_samples

src/lib/server/hosts/temperatures.ts

@@ -0,0 +1,96 @@
+import type Database from 'better-sqlite3';
+import { db } from '../db/index.js';
+import { historyTemperatures } from '../../host-temperatures.js';
+import type { HostInventory } from '../../hosts.js';
+import type { CapacityQuery, TemperatureHistorySeries } from '../../capacity.js';
+
+export function migrateTemperatures(handle: Database.Database): void {
+    handle.exec(`CREATE TABLE IF NOT EXISTS ec_host_temperature_samples (
+        host_id TEXT NOT NULL REFERENCES ec_hosts(id), bucket TEXT NOT NULL,
+        sensor_id TEXT NOT NULL, kind TEXT NOT NULL, label TEXT NOT NULL,
+        celsius REAL NOT NULL CHECK(celsius BETWEEN -100 AND 200),
+        PRIMARY KEY(host_id,bucket,sensor_id)
+    );
+    CREATE INDEX IF NOT EXISTS ec_host_temperatures_retention ON ec_host_temperature_samples(bucket);`);
+}
+
+export function recordTemperatures(
+    hostId: string,
+    inventory: HostInventory,
+    timestamp: string
+): void {
+    const sensors = historyTemperatures(inventory.temperatures);
+    if (!sensors.length) return;
+    const bucket = new Date(Math.floor(Date.parse(timestamp) / 60_000) * 60_000).toISOString();
+    const insert = db().prepare(`INSERT INTO ec_host_temperature_samples
+        (host_id,bucket,sensor_id,kind,label,celsius) VALUES(?,?,?,?,?,?)
+        ON CONFLICT(host_id,bucket,sensor_id) DO UPDATE SET
+        celsius=MAX(celsius,excluded.celsius),kind=excluded.kind,label=excluded.label`);
+    for (const sensor of sensors)
+        insert.run(hostId, bucket, sensor.id, sensor.kind, sensor.label, sensor.celsius);
+}
+
+/** Runs on the capacity report worker, returning at most 168 points per series. */
+export function readTemperatureHistory(
+    query: CapacityQuery,
+    hosts: { id: string; name: string; created_at: string }[],
+    startAt: number,
+    endAt: number,
+    bucketMs: number
+): TemperatureHistorySeries[] {
+    const count = Math.round((endAt - startAt) / bucketMs);
+    const statement = db().prepare(`SELECT sensor_id,kind,label,
+        MIN(?,MAX(0,CAST((CAST(strftime('%s',bucket) AS INTEGER)*1000-?) / ? AS INTEGER))) AS period,
+        MAX(celsius) AS celsius
+        FROM ec_host_temperature_samples WHERE host_id=? AND bucket>=? AND bucket<=?
+        ${query.hostId ? '' : "AND sensor_id='cpu'"}
+        GROUP BY sensor_id,period ORDER BY sensor_id,period`);
+    const result: TemperatureHistorySeries[] = [];
+    for (const host of hosts) {
+        const bySensor = new Map<string, TemperatureHistorySeries>();
+        const add = (id: string, kind: TemperatureHistorySeries['kind'], label: string) => {
+            const series = {
+                id: `${host.id}:${id}`,
+                hostId: host.id,
+                kind,
+                label,
+                points: Array.from({ length: count }, (_, i) => ({
+                    at: startAt + i * bucketMs,
+                    celsius: null as number | null
+                }))
+            };
+            bySensor.set(id, series);
+            result.push(series);
+            return series;
+        };
+        add('cpu', 'cpu', query.hostId ? 'CPU temperature' : host.name);
+        const rows = statement.all(
+            count - 1,
+            startAt,
+            bucketMs,
+            host.id,
+            new Date(
+                Math.max(
+                    Math.floor(Date.parse(host.created_at) / 60_000) * 60_000,
+                    Math.floor(startAt / 60_000) * 60_000
+                )
+            ).toISOString(),
+            new Date(endAt).toISOString()
+        ) as { sensor_id: string; kind: string; label: string; period: number; celsius: number }[];
+        for (const row of rows) {
+            let series = bySensor.get(row.sensor_id);
+            if (!series) {
+                if (!['nvme', 'chipset'].includes(row.kind) || bySensor.size >= 17) continue;
+                const kind = row.kind as 'nvme' | 'chipset';
+                const number = [...bySensor.values()].filter((s) => s.kind === kind).length + 1;
+                series = add(
+                    row.sensor_id,
+                    kind,
+                    `${kind === 'nvme' ? 'NVMe drive' : 'Chipset'} ${number}`
+                );
+            }
+            if (series.points[row.period]) series.points[row.period].celsius = row.celsius;
+        }
+    }
+    return result;
+}

src/lib/server/retention.ts

@@ -556,6 +556,14 @@ function samplePolicies(s = fleetSettings()) {
             value: iso(s.hostResourceKeepDays),
             enabled: s.hostResourceKeepDays > 0
         },
+        {
+            category: 'host-temperatures',
+            label: 'Host temperature samples',
+            table: 'ec_host_temperature_samples',
+            condition: 'bucket < ?',
+            value: iso(s.hostResourceKeepDays),
+            enabled: s.hostResourceKeepDays > 0
+        },
         {
             category: 'instance-resources',
             label: 'Instance resource samples',