Sign agent 1.10.1 release (Refs #155)

agent/CHANGELOG.md

@@ -1,5 +1,41 @@
 # Agent changelog
 
+## 2026-10-14
+
+### 1.10.1 — Retain verified rebuild artifacts (EC #154)
+
+- Retain the complete verified replacement file set before stopping the current
+  container, including inline files and private bank configuration, within the
+  existing 64 MiB operation limit. Staged-file comparison, replacement validation
+  and writing reuse those bytes without further artifact downloads.
+- Keep prepared bytes local to the operation. Failed and held attempts discard
+  their files; later operations prepare a fresh set. Immutable specifications,
+  journal requests and container identity labels remain unchanged.
+- Preserve checksum, path, per-file size, player-safety, bank-settlement and
+  generation checks. Add regressions for preflight failures, lost connectivity
+  after preflight, file-set limits and operation isolation.
+- Package release artifacts for offline signing before publication. No protocol
+  change or controller migration is required.
+
+## 2026-10-13
+
+### 1.10.0 — Host temperature telemetry
+
+- Read supported Linux hwmon CPU package/core, AMD physical die/CCD, NVMe and
+  Intel PCH temperatures with the Python standard library. Sensor failures,
+  faults and disabled inputs are skipped independently; unsupported hosts report
+  an empty list. AMD Tctl control offsets are excluded from physical readings.
+- Include labeled, bounded sensors in heartbeats with the `host.temperatures`
+  capability. Physical device/channel identities survive hwmon renumbering.
+- The controller shows current CPU readings and expandable sensor details, and
+  adds temperature history to Capacity. Fleet history has one CPU line per host;
+  individual hosts can add NVMe composite and chipset lines. History retains
+  minute peaks using the host-resource retention setting; missing data stays
+  unavailable and no thermal alerts or placement rules are introduced.
+- Adds database migration 23. Existing agents remain compatible. Release
+  artifacts are packaged and the stale signature is removed. Offline signing
+  and host rollout are still required; neither was performed here.
+
 ## 2026-10-12
 
 ### 1.9.3 — Protect private bank configuration (EC #152)

agent/ec_agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.9.3"
+VERSION = "1.10.1"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -63,6 +63,7 @@ UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-ch
 UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
 UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
 AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
+HWMON_ROOT = Path("/sys/class/hwmon")
 ROOT_MARKER = ".electric-cherry.json"
 
 
@@ -1308,6 +1309,59 @@ class Agent:
                 return None
             return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
 
+    def temperatures(self):
+        """Best-effort, read-only samples from known millidegree-Celsius drivers."""
+        samples = []
+        try:
+            chips = sorted(HWMON_ROOT.glob("hwmon*"))[:64]
+        except OSError:
+            return samples
+        for chip in chips:
+            try:
+                device = (chip / "name").read_text().strip()
+                if device not in ("coretemp", "k10temp", "nvme") and not device.startswith("pch_"):
+                    continue
+                # hwmon numbers can change on reboot. The physical sysfs path
+                # and channel identify separate CPU sockets and NVMe devices.
+                identity = re.sub(r"/(?:hwmon/)?hwmon\d+$", "", str(chip.resolve()))
+                inputs = sorted(chip.glob("temp[0-9]*_input"))[:128]
+            except (OSError, ValueError):
+                continue
+            for source in inputs:
+                try:
+                    channel = source.name.removesuffix("_input")
+                    label_path = chip / (channel + "_label")
+                    label = label_path.read_text().strip() if label_path.exists() else channel
+                    if not label or len(label) > 120:
+                        continue
+                    fault = chip / (channel + "_fault")
+                    enabled = chip / (channel + "_enable")
+                    if (fault.exists() and int(fault.read_text()) != 0) or (enabled.exists() and int(enabled.read_text()) == 0):
+                        continue
+                    if device == "coretemp":
+                        kind = "cpu-package" if label.startswith("Package id ") else "cpu-core" if label.startswith("Core ") else None
+                    elif device == "k10temp":
+                        # Tctl can contain a fan-control offset. Never present
+                        # it as the physical die temperature or a fleet metric.
+                        kind = "cpu-die" if label == "Tdie" or re.fullmatch(r"Tccd\d+", label) else None
+                    else:
+                        kind = "nvme" if device == "nvme" else "chipset"
+                    if kind is None:
+                        continue
+                    celsius = int(source.read_text()) / 1000
+                    if not -100 <= celsius <= 200:
+                        continue
+                    sensor_id = hashlib.sha256((identity + "/" + channel).encode()).hexdigest()[:24]
+                    samples.append({"id": sensor_id, "device": device, "label": label,
+                                    "kind": kind, "celsius": round(celsius, 1)})
+                    if len(samples) >= 128:
+                        return samples
+                except (OSError, ValueError):
+                    # Read permission alone does not imply data is available
+                    # (e.g. ENODATA), and a device can disappear during a read.
+                    continue
+        return samples
+
     def instance_resources(self, observations=None):
         """One bounded Docker stats batch; unavailable samples are never zero load."""
         ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true",
@@ -1392,6 +1446,7 @@ class Agent:
 
     def inventory(self):
         cpu_usage = self.cpu_usage_percent()
+        temperatures = self.temperatures()
         usage = shutil.disk_usage(self.root)
         memory = {}
         try:
@@ -1516,9 +1571,9 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["host.temperatures", "library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         plutonium_cache = self.plutonium_cache_inventory()
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "temperatures": temperatures, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
         health = self.inspect(instance_id, generation)
@@ -1630,7 +1685,7 @@ class Agent:
         players = status_players(text) if command == "status" else None
         return {"ok": True, "text": text, "players": players}
 
-    def validate_spec(self, payload):
+    def validate_spec(self, payload, *, prepared_files=None):
         game = payload.get("game")
         if game not in ("t5", "t6"):
             raise AgentError("Only T5 and T6 are supported")
@@ -1673,16 +1728,22 @@ class Agent:
         files = self.initial_files(payload)
         if not isinstance(files, list) or len(files) > 10000:
             raise Rejected("Invalid initial file list")
+        if prepared_files is not None:
+            if set(prepared_files) != {item["path"] for item in files} or any(not isinstance(content, bytes) for content in prepared_files.values()):
+                raise Rejected("Prepared files do not match the replacement specification")
+        verified_files = {}
         total, paths = 0, set()
         for item in files:
             scoped(self.root / "agent" / "validation", item["path"])
             if item["path"] in paths:
                 raise Rejected("Duplicate initial file path")
             paths.add(item["path"])
-            total += len(self.decode_file(item))
+            content = self.decode_file(item, content=None if prepared_files is None else prepared_files[item["path"]])
+            total += len(content)
             if total > 64 * 1024 * 1024:
                 raise Rejected("Initial files exceed the 64 MiB operation limit")
-        return game, library, image, image_id, port, bind, env
+            verified_files[item["path"]] = content
+        return game, library, image, image_id, port, bind, env, verified_files
 
     @staticmethod
     def plutonium_manifest(data):
@@ -1901,7 +1962,7 @@ class Agent:
             managed_write(app, marker, raw, uid, gid)
             return True
 
-    def create(self, instance_id, generation, payload, *, before=None):
+    def create(self, instance_id, generation, payload, *, before=None, prepared_files=None):
         self.operation_phase = "checking-instance"
         self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
@@ -1916,7 +1977,7 @@ class Agent:
                 raise AgentError("Instance already exists with a different specification")
             return existing
         self.operation_phase = "validating-create-spec"
-        game, library, image, image_id, port, bind, env = self.validate_spec(payload)
+        game, library, image, image_id, port, bind, env, prepared_files = self.validate_spec(payload, prepared_files=prepared_files)
         app = directory / "app"
         self.operation_phase = "preparing-create-directory"
         app.mkdir(parents=True, exist_ok=True)
@@ -1924,7 +1985,7 @@ class Agent:
         self.operation_phase = "seeding-runtime"
         self.seed_plutonium(app, image)
         self.operation_phase = "writing-create-files"
-        self.write_initial_files(app, self.initial_files(payload))
+        self.write_initial_files(app, prepared_files)
         self.operation_phase = "persisting-create-metadata"
         atomic_json(directory / "spec.json", payload)
         env_path = directory / "environment"
@@ -1977,30 +2038,32 @@ class Agent:
         content = f'set tws_bank_token "{bank["token"]}"\n'.encode()
         return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
 
-    def write_initial_files(self, root, files):
-        if not isinstance(files, list) or len(files) > 10000:
-            raise AgentError("Invalid file list")
-        for item in files:
-            content = self.decode_file(item)
-            managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
+    def write_initial_files(self, root, prepared_files):
+        for path, content in prepared_files.items():
+            managed_write(root, path, content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
 
-    def decode_file(self, item):
+    def decode_file(self, item, *, content=None):
         if "contentSha256" in item:
             sha = item["contentSha256"]
             if not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha):
                 raise AgentError("Invalid file content hash")
-            url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha
-            with self.artifact_open("GET", url) as response:
-                content = response.read(MAX_FILE + 1)
+            if content is None:
+                url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha
+                with self.artifact_open("GET", url) as response:
+                    content = response.read(MAX_FILE + 1)
             if len(content) > MAX_FILE:
                 raise AgentError("File exceeds staging size limit")
             if digest(content) != sha or item.get("sha256", sha) != sha:
                 raise AgentError("Staged file checksum mismatch")
             return content
         try:
-            content = base64.b64decode(item["contentBase64"], validate=True)
+            decoded = base64.b64decode(item["contentBase64"], validate=True)
         except (ValueError, KeyError, TypeError):
             raise AgentError("Invalid encoded file content") from None
+        if content is None:
+            content = decoded
+        elif content != decoded:
+            raise Rejected("Prepared files do not match the replacement specification")
         if len(content) > MAX_FILE:
             raise AgentError("File exceeds staging size limit")
         if item.get("sha256") and item["sha256"] != digest(content):
@@ -2135,13 +2198,16 @@ class Agent:
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
                     raise AgentError("Rebuild must use a newer immutable generation")
                 spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
-                self.validate_spec(spec)
+                # Keep the complete verified set on this call's stack only. Never
+                # mutate the immutable spec or cache bytes on the shared agent:
+                # held, failed and later operations must prepare their own files.
+                *_, prepared_files = self.validate_spec(spec)
             if not override_safety:
                 self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
             if "autoscaling" in payload:
                 self.authorize_autoscaling_removal(instance_id, generation, payload)
             if kind == "instance.rebuild" and payload.get("stageId"):
-                before = self.rebuild_before(instance_id, generation, payload["stageId"], spec)
+                before = self.rebuild_before(instance_id, generation, payload["stageId"], prepared_files)
                 if not override_safety:
                     self.safe_to_disrupt(instance_id, generation)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})})
@@ -2153,7 +2219,7 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 self.settle_bank(instance_id, generation)
                 if kind == "instance.rebuild":
-                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None)
+                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None, prepared_files=prepared_files)
                     self.operation_phase = "removing-old-instance"
                     old = self.instance_dir(instance_id, generation)
                     if old.exists():
@@ -2254,21 +2320,21 @@ class Agent:
         atomic_json(marker, manifest)
         return {"stageId": stage_id, "files": len(planned), "state": "staged"}
 
-    def rebuild_before(self, instance_id, generation, stage_id, spec):
+    def rebuild_before(self, instance_id, generation, stage_id, prepared_files):
         """Validate requested replacement bytes and snapshot the old generation before teardown."""
         directory = self.instance_dir(instance_id, generation)
         marker = directory / "stages" / identifier(stage_id) / "manifest.json"
         saved = json.loads(marker.read_text())
         if saved["state"] != "staged":
             raise Rejected("Rebuild requires an unactivated file stage")
-        replacement = {item["path"]: item for item in self.initial_files(spec)}
         before = []
         for item in saved["files"]:
-            target = replacement.get(item["path"])
+            require_public_file(item["path"])
+            target = prepared_files.get(item["path"])
             if item["delete"]:
                 if target is not None:
                     raise Rejected("Rebuild includes a file requested for removal")
-            elif target is None or digest(self.decode_file(target)) != item["sha256"]:
+            elif target is None or digest(target) != item["sha256"]:
                 raise Rejected("Rebuild does not match staged file content")
             content = managed_read(directory / "app", item["path"])
             actual_sha = None if content is None else digest(content)

agent/tests/test_agent.py

@@ -778,6 +778,12 @@ class AgentTests(unittest.TestCase):
             else:
                 self.assertEqual(inventory["cpuUsagePercent"], usage)
 
+    def test_inventory_reports_optional_temperature_capability_and_empty_samples(self):
+        with patch.object(self.agent, "temperatures", return_value=[]):
+            inventory = self.agent.inventory()
+        self.assertEqual(inventory["temperatures"], [])
+        self.assertIn("host.temperatures", inventory["capabilities"])
+
     def resource_container(self):
         return {"Name": self.agent.container_name(INSTANCE, 1),
                 "Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}},
@@ -1660,7 +1666,7 @@ class AgentTests(unittest.TestCase):
         with self.assertRaisesRegex(AgentError, "Private bank"):
             self.agent.activate(INSTANCE, 1, "private", override_safety=True)
         with self.assertRaisesRegex(AgentError, "Private bank"):
-            self.agent.rebuild_before(INSTANCE, 1, "private", {"files": []})
+            self.agent.rebuild_before(INSTANCE, 1, "private", {})
         with self.assertRaisesRegex(AgentError, "Private bank"):
             self.agent.restore_files(app, [{"path": value, "contentBase64": None}])
         with self.assertRaisesRegex(AgentError, "Private bank"):
@@ -1928,6 +1934,164 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(json.loads(result["result"])["before"][0]["contentBase64"], base64.b64encode(b"original").decode())
         self.assertEqual(self.docker.calls, calls)
 
+    def artifact_rebuild(self, staged=False):
+        if staged:
+            operation = self.staged_rebuild()
+        else:
+            spec = self.create()
+            self.agent.rcon = lambda *_: {"ok": True, "players": 0}
+            operation = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2}, "rebuild")
+        spec = operation["payload"]["spec"]
+        contents = {"configs/server.cfg": b"replacement", "scripts/replacement.gsc": b"verified artifact bytes"}
+        spec["files"] = [{"path": path, "contentSha256": module.digest(content), "sha256": module.digest(content)} for path, content in contents.items()]
+        spec["files"].append({"path": "configs/empty.cfg", "contentBase64": ""})
+        spec["bank"] = {"configPath": "configs/electric_cherry_bank.cfg", "token": "ec_bank_" + "a" * 32}
+        return operation, contents
+
+    def assert_rebuild_uses_preflight_artifacts(self, staged):
+        operation, contents = self.artifact_rebuild(staged)
+        original_request = json.dumps(operation, sort_keys=True)
+        available = {module.digest(content): content for content in contents.values()}
+
+        def fetch(method, url):
+            # Connectivity disappears as soon as the complete preflight set has
+            # been read, before staged comparison or destructive replacement.
+            if not available:
+                raise module.urllib.error.URLError(TimeoutError())
+            self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+            return Response(available.pop(url.rsplit("/", 1)[-1]))
+
+        with patch.object(self.agent, "artifact_open", side_effect=fetch) as downloads:
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(downloads.call_count, len(contents))
+        self.assertFalse(available)
+        self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
+        self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
+        app = self.agent.instance_dir(INSTANCE, 2) / "app"
+        for path, content in contents.items():
+            self.assertEqual((app / path).read_bytes(), content)
+        self.assertEqual((app / "configs/empty.cfg").read_bytes(), b"")
+        self.assertEqual((app / "configs/electric_cherry_bank.cfg").read_text(), 'set tws_bank_token "ec_bank_' + "a" * 32 + '"\n')
+        self.assertEqual(json.dumps(operation, sort_keys=True), original_request)
+        self.assertEqual(json.loads(result["request"]), operation)
+        self.assertEqual(self.agent.spec(INSTANCE, 2), operation["payload"]["spec"])
+        labels = self.docker.inspect(self.agent.container_name(INSTANCE, 2))["Config"]["Labels"]
+        self.assertEqual(labels[module.LABEL + ".operation"], operation["id"])
+        self.assertEqual(labels[module.LABEL + ".spec-sha256"], module.digest(json.dumps(operation["payload"]["spec"], sort_keys=True, separators=(",", ":")).encode()))
+        receipt = json.loads(result["result"])
+        self.assertEqual(receipt["before"], [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"original").decode()}] if staged else [])
+        for content in contents.values():
+            self.assertNotIn(base64.b64encode(content).decode(), json.dumps(result))
+
+        # A later operation with the same artifacts must download its own set.
+        later = self.operation("instance.rebuild", {"spec": operation["payload"]["spec"], "nextGeneration": 3}, "later-rebuild", generation=2)
+        with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
+            self.assertEqual(self.agent.execute(later)["state"], "failed")
+        downloads.assert_called_once()
+        self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
+        self.assertFalse(self.agent.inspect(INSTANCE, 3)["exists"])
+
+    def test_unstaged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self):
+        self.assert_rebuild_uses_preflight_artifacts(staged=False)
+
+    def test_staged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self):
+        self.assert_rebuild_uses_preflight_artifacts(staged=True)
+
+    def test_rebuild_preflight_download_failure_preserves_original_and_discards_partial_set(self):
+        operation, contents = self.artifact_rebuild()
+        with patch.object(self.agent, "artifact_open", side_effect=[Response(contents["configs/server.cfg"]), module.urllib.error.URLError(TimeoutError())]) as downloads:
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "failed", result)
+        self.assertEqual(downloads.call_count, 2)
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(self.agent.instance_dir(INSTANCE, 2).exists())
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+        operation["id"] = "retry-rebuild"
+        with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
+            self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        self.assertTrue(downloads.call_args.args[1].endswith(module.digest(contents["configs/server.cfg"])))
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+
+    def test_held_rebuild_discards_prepared_artifacts_before_retry(self):
+        operation, contents = self.artifact_rebuild(staged=True)
+        with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}):
+            self.assertEqual(self.agent.execute(operation)["state"], "accepted")
+        with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
+            self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        downloads.assert_called_once()
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
+
+    def test_rebuild_preflight_rejects_corrupt_and_oversized_artifacts(self):
+        operation, contents = self.artifact_rebuild()
+        for index, content in enumerate((b"corrupt", b"x" * (module.MAX_FILE + 1))):
+            with self.subTest(index=index):
+                operation["id"] = f"invalid-artifact-{index}"
+                with patch.object(self.agent, "artifact_open", return_value=Response(content)):
+                    result = self.agent.execute(operation)
+                self.assertEqual(result["state"], "failed", result)
+                self.assertIn("checksum mismatch" if index == 0 else "size limit", result["error"])
+                self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+
+    def test_rebuild_preflight_bounds_complete_file_set_at_64_mib(self):
+        operation, _ = self.artifact_rebuild()
+        spec = operation["payload"]["spec"]
+        content = b"x" * module.MAX_FILE
+        spec["files"] = [{"path": f"scripts/{index}.gsc", "contentSha256": module.digest(content)} for index in range(2)]
+        # The generated private bank file also counts toward the operation limit.
+        with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "failed", result)
+        self.assertIn("64 MiB operation limit", result["error"])
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+        del spec["bank"]
+        operation["id"] = "at-limit"
+        with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}):
+            self.assertEqual(self.agent.execute(operation)["state"], "accepted")
+
+    def test_replacement_validation_rejects_incomplete_or_mismatched_prepared_bytes_without_downloads(self):
+        operation, contents = self.artifact_rebuild()
+        spec = operation["payload"]["spec"]
+        del spec["bank"]
+        prepared = {**contents, "configs/empty.cfg": b""}
+        invalid_sets = [
+            {path: content for path, content in prepared.items() if path != "configs/server.cfg"},
+            {**prepared, "configs/extra.cfg": b"unexpected"},
+            {**prepared, "configs/server.cfg": b"wrong artifact"},
+            {**prepared, "configs/empty.cfg": b"wrong inline bytes"},
+            {**prepared, "configs/server.cfg": None},
+        ]
+        with patch.object(self.agent, "artifact_open", side_effect=AssertionError("Prepared validation must not download")) as downloads:
+            for index, files in enumerate(invalid_sets):
+                with self.subTest(index=index), self.assertRaises(AgentError):
+                    self.agent.validate_spec(spec, prepared_files=files)
+        downloads.assert_not_called()
+
+    def test_rebuild_preflight_preserves_file_path_and_duplicate_checks(self):
+        operation, _ = self.artifact_rebuild()
+        spec = operation["payload"]["spec"]
+        for index, paths in enumerate((["../escape.cfg"], ["configs/electric_cherry_bank.cfg"], ["configs/same.cfg"] * 2)):
+            with self.subTest(paths=paths):
+                operation["id"] = f"invalid-paths-{index}"
+                spec["files"] = [{"path": path, "contentBase64": ""} for path in paths]
+                self.assertEqual(self.agent.execute(operation)["state"], "failed")
+                self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+
+    def test_staged_rebuild_rejects_verified_artifacts_that_differ_from_the_stage(self):
+        operation, contents = self.artifact_rebuild(staged=True)
+        contents["configs/server.cfg"] = b"different verified replacement"
+        operation["payload"]["spec"]["files"][0].update(contentSha256=module.digest(contents["configs/server.cfg"]), sha256=module.digest(contents["configs/server.cfg"]))
+        with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "failed", result)
+        self.assertIn("does not match staged file content", result["error"])
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+
     def test_rebuild_moves_generation_and_rejects_old_commands(self):
         payload = self.create()
         self.agent.rcon = lambda *_: {"ok": True, "players": 0}

agent/tests/test_temperatures.py

@@ -0,0 +1,96 @@
+import errno
+import importlib.util
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_agent_temperatures", Path(__file__).resolve().parents[1] / "ec_agent.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+class TemperatureTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.hwmon = self.root / "class"
+        self.hwmon.mkdir()
+        self.patch = patch.object(module, "HWMON_ROOT", self.hwmon)
+        self.patch.start()
+        self.addCleanup(self.patch.stop)
+        self.agent = object.__new__(module.Agent)
+
+    def chip(self, number, device, readings):
+        physical = self.root / (device + str(number)) / "hwmon" / ("hwmon" + str(number))
+        physical.mkdir(parents=True)
+        (self.hwmon / physical.name).symlink_to(physical, target_is_directory=True)
+        (physical / "name").write_text(device)
+        for channel, label, value in readings:
+            (physical / (channel + "_input")).write_text(str(value))
+            if label is not None:
+                (physical / (channel + "_label")).write_text(label)
+        return physical
+
+    def test_ec2_sensors_and_units_without_guessing_unknown_devices(self):
+        self.chip(0, "nvme", [("temp1", "Composite", 50900), ("temp2", "Sensor 1", 50900)])
+        self.chip(1, "pch_cometlake", [("temp1", None, 62000)])
+        self.chip(2, "coretemp", [("temp1", "Package id 0", 65000), ("temp2", "Core 0", 63000)])
+        self.chip(3, "iwlwifi_1", [("temp1", None, "unavailable")])
+        samples = self.agent.temperatures()
+        self.assertEqual([s["celsius"] for s in samples], [50.9, 50.9, 62.0, 65.0, 63.0])
+        self.assertEqual([s["kind"] for s in samples], ["nvme", "nvme", "chipset", "cpu-package", "cpu-core"])
+        self.assertEqual(len({s["id"] for s in samples}), len(samples))
+
+    def test_fault_disabled_bad_value_and_enodata_do_not_lose_other_sensors(self):
+        chip = self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), value)
+                         for i, value in enumerate([65000, 60000, "bad", 500000, 61000, 63000, 0], 1)])
+        (chip / "temp1_fault").write_text("1")
+        (chip / "temp2_enable").write_text("0")
+        original = Path.read_text
+        def read(path, *args, **kwargs):
+            if path.name == "temp5_input":
+                raise OSError(errno.ENODATA, "No data available")
+            return original(path, *args, **kwargs)
+        with patch.object(Path, "read_text", read):
+            self.assertEqual([s["celsius"] for s in self.agent.temperatures()], [63.0, 0.0])
+
+    def test_amd_keeps_physical_die_readings_and_excludes_control_offsets(self):
+        self.chip(0, "k10temp", [("temp1", "Tctl", 85000), ("temp2", "Tdie", 65000), ("temp3", "Tccd1", 64000)])
+        self.assertEqual([s["label"] for s in self.agent.temperatures()], ["Tdie", "Tccd1"])
+
+    def test_sensor_identity_survives_hwmon_renumbering_and_separates_drives(self):
+        physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
+        self.chip(2, "nvme", [("temp1", "Composite", 51000)])
+        before = {s["celsius"]: s["id"] for s in self.agent.temperatures()}
+        renamed = physical.with_name("hwmon9")
+        physical.rename(renamed)
+        (self.hwmon / "hwmon1").unlink()
+        (self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
+        self.assertEqual(before, {s["celsius"]: s["id"] for s in self.agent.temperatures()})
+        self.assertEqual(len(set(before.values())), 2)
+
+    def test_missing_hwmon_and_disappearing_devices_are_optional(self):
+        self.assertEqual(self.agent.temperatures(), [])
+        (self.hwmon / "hwmon0").symlink_to(self.root / "missing")
+        self.assertEqual(self.agent.temperatures(), [])
+        with patch.object(Path, "glob", side_effect=PermissionError()):
+            self.assertEqual(self.agent.temperatures(), [])
+
+    def test_direct_hwmon_child_keeps_identity_when_renumbered(self):
+        physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
+        direct = physical.parent.parent / "hwmon8"
+        physical.rename(direct)
+        (self.hwmon / "hwmon1").unlink()
+        (self.hwmon / "hwmon8").symlink_to(direct, target_is_directory=True)
+        before = self.agent.temperatures()[0]["id"]
+        renamed = direct.with_name("hwmon9")
+        direct.rename(renamed)
+        (self.hwmon / "hwmon8").unlink()
+        (self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
+        self.assertEqual(self.agent.temperatures()[0]["id"], before)
+
+    def test_inventory_size_is_bounded(self):
+        self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), 65000) for i in range(1, 200)])
+        self.assertEqual(len(self.agent.temperatures()), 128)

docs/guides/collections.md

@@ -304,6 +304,27 @@ before continuing eligible interrupted startups. Unknown command results retain
 their original command ID and occupied slots until the host confirms the outcome.
 Missing server registrations and later destruction attempts require inspection.
 
+### Recovering a completed rebuild generation
+
+A completed rebuild records the replacement generation before EC verifies health.
+If fresh host inventory disagrees, verification pauses with the controller and
+host generations in the operation's recovery message.
+
+For an existing mismatch, open the **completed rebuild's Provisioning detail**
+(even if that operation was later cancelled), then choose **Reconcile replacement
+generation**. EC requires a completed command for the original host and immutable
+instance ID, the command's requested replacement generation, and matching host
+inventory observed within the last 30 seconds. Wait for that host to reconnect
+and report fresh inventory if the action is unavailable. A server name alone is
+not sufficient evidence.
+
+Reconciliation preserves cancellation and submitted commands. Use **Resume health
+verification** when offered, or request a new lifecycle operation after resolving
+any remaining active operation. Fresh health checks, log reads, and new lifecycle
+commands use the replacement generation. EC requires fresh container and game
+evidence before reporting the server online. The agent continues to reject stale
+commands; an admin player-safety override does not bypass generation checks.
+
 ## API
 
 Collections retain the existing `/api/blueprints` endpoints and `blueprintId` /
@@ -351,6 +372,9 @@ there while preserving query parameters.
 - `GET /api/deployments/:id`: includes `recipe`, `blueprintName`, and
   `canResumeVerification` and `canRetryWithRebuild` for lifecycle operations.
 - `POST /api/deployments/:id/resume`: resume eligible creation health checks.
+- `POST /api/provisioning/:id/reconcile-generation`: reconcile a completed rebuild
+  using fresh inventory from its original host. Returns 409 when identity or
+  completion evidence cannot be validated; repeated application is safe.
 - `POST /api/provisioning/:id/retry-rebuild`: request a new rebuild for an eligible
   failed creation and link it from the original operation.
 

docs/guides/native-hosts.md

@@ -354,6 +354,40 @@ journalctl -u electric-cherry-update.service -n 100
 
 ## Placement and resource capacity
 
+### Host temperature
+
+Agent **1.10.0** reports temperature sensors exposed by supported Linux hwmon
+drivers without installing additional packages. **Hosts** shows CPU temperature
+beneath CPU usage. A host's **Overview → Inventory** shows the selected reading
+and expandable **Temperature sensors**, including per-core and storage details.
+Stale readings are marked as last reported; absent or failed readings are
+**Unavailable**, never zero or a readiness failure.
+
+**Fleet → Capacity → Temperature history** uses the existing host selector and
+Last hour / 24 hours / 7 days controls. **All hosts** displays a separate CPU line
+for each host. Select one host to add its NVMe composite and chipset lines using
+the legend. Hover, tap, or use the arrow keys on the chart to inspect a period.
+Each host page's **Capacity** button opens the view filtered to that host.
+The game filter affects launch estimates, not temperatures.
+
+The CPU reading is the hottest reported package, falling back to a physical
+AMD die/CCD and then a core reading. AMD Tctl cooling-control values are excluded
+because they may include an offset. Supported device drivers are `coretemp`,
+`k10temp`, `nvme` and `pch_*`; sensor availability depends on hardware and drivers.
+Virtual machines and older agents may have no supported readings. Missing,
+faulty and disabled sensors are skipped individually.
+
+Temperature history starts after the agent upgrade. The controller stores one
+peak per sensor per minute, limited to the CPU summary and up to sixteen NVMe
+composite/chipset inputs per host. Longer chart periods show the maximum of
+those peaks; a period without readings remains a gap. Brief gaps within an
+aggregated period may still leave a peak for that period. **Host resource history
+retention** applies to CPU, memory and temperature samples (seven days by default;
+zero keeps them indefinitely). Series colors identify hosts/sensors and do not
+indicate thermal safety. Temperature does not change placement or trigger alerts.
+
+### CPU and memory capacity
+
 ### Host inventory and eligibility
 
 The Inventory panel shows CPU usage with agent 1.4.2 and later. It measures
@@ -620,6 +654,13 @@ restart and create complete only after post-operation health evidence; creation
 and rebuild also verify the selected map and running Plutonium revision. The
 base-game library and runtime revision are independent records.
 
+Agent 1.10.1 prepares and verifies the complete replacement file set before
+stopping the existing container, within the 64 MiB operation limit. Staged-file
+comparison, replacement validation and writing reuse those bytes, so losing
+artifact connectivity after preflight does not prevent replacement. A preflight
+download failure leaves the original container running. Prepared bytes last only
+for that operation attempt; held or failed work must prepare its files again.
+
 ## Rebalance instances
 
 Open **Fleet → Capacity → Rebalance** to spread existing instances across active

src/lib/capacity.ts

@@ -30,6 +30,14 @@ export interface CapacityHistory {
     endAt: number;
     bucketMs: number;
     points: CapacityHistoryPoint[];
+    temperatures: TemperatureHistorySeries[];
+}
+export interface TemperatureHistorySeries {
+    id: string;
+    hostId: string;
+    label: string;
+    kind: 'cpu' | 'nvme' | 'chipset';
+    points: { at: number; celsius: number | null }[];
 }
 export interface CapacityHost {
     id: string;

src/lib/fleet-settings.ts

@@ -305,7 +305,7 @@ export const settingHelp: Partial<Record<keyof FleetSettings, string>> = {
         'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.',
     recoveryKeepDays:
         'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.',
-    hostResourceKeepDays: 'Host CPU and memory history.',
+    hostResourceKeepDays: 'Host CPU, memory and temperature history.',
     instanceResourceKeepDays:
         'Minute resource samples. Capacity learning continues to use its independent 24-hour window.',
     metricsKeepDays: 'Historical fleet occupancy and utilization samples.',

src/lib/host-temperatures.ts

@@ -0,0 +1,92 @@
+export const temperatureKinds = ['cpu-package', 'cpu-die', 'cpu-core', 'nvme', 'chipset'] as const;
+export type TemperatureKind = (typeof temperatureKinds)[number];
+export interface HostTemperatureSensor {
+    id: string;
+    device: string;
+    label: string;
+    kind: TemperatureKind;
+    celsius: number;
+}
+
+/** Optional telemetry must never make an otherwise valid heartbeat fail. */
+export function validateTemperatures(value: unknown): HostTemperatureSensor[] {
+    if (!Array.isArray(value)) return [];
+    const seen = new Set<string>();
+    return value.slice(0, 128).flatMap((sensor) => {
+        if (
+            !sensor ||
+            typeof sensor !== 'object' ||
+            Array.isArray(sensor) ||
+            typeof sensor.id !== 'string' ||
+            !/^[a-zA-Z0-9_-]{1,100}$/.test(sensor.id) ||
+            seen.has(sensor.id) ||
+            !['device', 'label'].every(
+                (key) =>
+                    typeof sensor[key] === 'string' &&
+                    sensor[key].length > 0 &&
+                    sensor[key].length <= 120
+            ) ||
+            !temperatureKinds.includes(sensor.kind) ||
+            typeof sensor.celsius !== 'number' ||
+            !Number.isFinite(sensor.celsius) ||
+            sensor.celsius < -100 ||
+            sensor.celsius > 200
+        )
+            return [];
+        seen.add(sensor.id);
+        return [
+            {
+                id: sensor.id,
+                device: sensor.device,
+                label: sensor.label,
+                kind: sensor.kind,
+                celsius: sensor.celsius
+            }
+        ];
+    });
+}
+
+export function cpuTemperature(
+    sensors: HostTemperatureSensor[] = []
+): HostTemperatureSensor | null {
+    // Prefer physical package/die readings. Cores are a fallback, never mixed
+    // into an average that might hide a hot package on a multi-socket host.
+    for (const kind of ['cpu-package', 'cpu-die', 'cpu-core'] as const) {
+        const candidates = sensors.filter((sensor) => sensor.kind === kind);
+        if (candidates.length)
+            return candidates.reduce((hot, sensor) =>
+                sensor.celsius > hot.celsius ? sensor : hot
+            );
+    }
+    return null;
+}
+
+export function temperatureLabel(sensor: HostTemperatureSensor): string {
+    if (sensor.kind === 'cpu-package')
+        return `CPU package ${sensor.label.replace(/^Package id /, '')}`;
+    if (sensor.kind === 'cpu-core') return `CPU ${sensor.label.toLowerCase()}`;
+    if (sensor.kind === 'cpu-die') return `CPU die · ${sensor.label}`;
+    if (sensor.kind === 'chipset') return 'Chipset';
+    return `NVMe · ${sensor.label}`;
+}
+
+export function formatTemperature(celsius: number | null | undefined): string {
+    return typeof celsius === 'number' && Number.isFinite(celsius)
+        ? `${Number(celsius.toFixed(1))} °C`
+        : 'Unavailable';
+}
+
+/** One CPU series per host, plus bounded device series for the host drill-down. */
+export function historyTemperatures(
+    sensors: HostTemperatureSensor[] = []
+): HostTemperatureSensor[] {
+    const cpu = cpuTemperature(sensors);
+    const devices = sensors
+        .filter(
+            (sensor) =>
+                sensor.kind === 'chipset' ||
+                (sensor.kind === 'nvme' && sensor.label.toLowerCase() === 'composite')
+        )
+        .slice(0, 16);
+    return [...(cpu ? [{ ...cpu, id: 'cpu', label: 'CPU temperature' }] : []), ...devices];
+}

src/lib/hosts.ts

@@ -1,4 +1,5 @@
 import type { HostLibraryCache } from './game-files.js';
+import type { HostTemperatureSensor } from './host-temperatures.js';
 
 export type HostMode = 'active' | 'draining' | 'disabled';
 export type HostReadiness =
@@ -149,6 +150,7 @@ export interface HostInventory {
     cpu?: string;
     cpuCount?: number;
     cpuUsagePercent?: number;
+    temperatures?: HostTemperatureSensor[];
     memoryTotal?: number;
     memoryAvailable?: number;
     storageTotal?: number;

src/lib/provisioning-detail.ts

@@ -117,6 +117,13 @@ export interface ProvisioningCapacity {
     }[];
 }
 export interface ProvisioningDetail {
+    generationRecovery?: {
+        expectedGeneration: number;
+        replacementGeneration: number;
+        observedGeneration: number | null;
+        available: boolean;
+        reason: string | null;
+    } | null;
     canOverrideSafety?: boolean;
     safetyOverrideUnavailableReason?: string | null;
     safetyOverride?: { actorId: string; actorName: string; confirmedAt: string } | null;

src/lib/server/db/migrations.ts

@@ -24,6 +24,7 @@ import { migrateLeaderboardReports } from './leaderboard-reports.js';
 import { migratePlayerScoreTotal } from './player-score-total.js';
 import { migrateBank } from './bank.js';
 import { migrateBankCredentials } from './bank-credentials.js';
+import { migrateTemperatures } from '../hosts/temperatures.js';
 import type { Migration } from './migration-runner.js';
 
 // Append only: never renumber entries or change an applied migration to add an
@@ -118,7 +119,22 @@ export const migrations: readonly Migration[] = [
     },
     { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank },
     { version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials },
-    { version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions }
+    { version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions },
+    { version: 23, name: 'host temperature history', run: migrateTemperatures },
+    {
+        version: 24,
+        name: 'rebuild generation evidence boundary',
+        run: (handle) => {
+            if (
+                !(handle.pragma('table_info(native_instances)') as { name: string }[]).some(
+                    (column) => column.name === 'generation_reconciled_at'
+                )
+            )
+                handle.exec(
+                    'ALTER TABLE native_instances ADD COLUMN generation_reconciled_at TEXT'
+                );
+        }
+    }
 ];
 
 export const databaseVersion = migrations.length;

src/lib/server/fleet/lifecycle-evidence.ts

@@ -7,9 +7,10 @@ export function latestGameSubmission(server: string): string | null {
             .prepare(
                 `SELECT MAX(at) AS at FROM (
         SELECT MAX(submitted_at) AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
-        UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`
+        UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?
+        UNION ALL SELECT generation_reconciled_at AS at FROM native_instances WHERE server=? AND retired_at IS NULL)`
             )
-            .get(server, server) as { at: string | null }
+            .get(server, server, server) as { at: string | null }
     ).at;
 }
 
@@ -21,7 +22,8 @@ export function latestGameSubmissions(): Map<string, string | null> {
         SELECT s.name AS server, (SELECT MAX(submitted_at) FROM server_deployments
             WHERE server=s.name AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')) AS at
         FROM servers s
-        UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`
+        UNION ALL SELECT server, requested_at AS at FROM restart_rollout
+        UNION ALL SELECT server, generation_reconciled_at AS at FROM native_instances WHERE retired_at IS NULL) GROUP BY server`
         )
         .all() as { server: string; at: string | null }[];
     return new Map(rows.map((row) => [row.server, row.at]));

src/lib/server/hosts/capacity-history.ts

@@ -1,5 +1,6 @@
 import { db } from '../db/index.js';
 import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js';
+import { readTemperatureHistory } from './temperatures.js';
 
 const windows: Record<CapacityRange, { duration: number; bucketMs: number }> = {
     '1h': { duration: 3_600_000, bucketMs: 60_000 },
@@ -15,10 +16,14 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
     const count = duration / bucketMs;
     const hosts = handle
         .prepare(
-            `SELECT id,created_at FROM ec_hosts
+            `SELECT id,name,created_at FROM ec_hosts
         WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`
         )
-        .all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[];
+        .all(...(query.hostId ? [query.hostId] : [])) as {
+        id: string;
+        name: string;
+        created_at: string;
+    }[];
     // The primary key bounds each host scan to the selected interval. Aggregate
     // before crossing the worker boundary; do not return raw heartbeat rows.
     const samples = handle.prepare(`SELECT
@@ -69,5 +74,12 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap
             point.memoryBytesAvailable = null;
         }
     }
-    return { range: query.range, startAt, endAt: now, bucketMs, points };
+    return {
+        range: query.range,
+        startAt,
+        endAt: now,
+        bucketMs,
+        points,
+        temperatures: readTemperatureHistory(query, hosts, startAt, now, bucketMs)
+    };
 }

src/lib/server/hosts/index.ts

@@ -23,6 +23,8 @@ import { logResultMetadata } from './log-results.js';
 import { parseHostInventory } from './request-context.js';
 import { fleetSettings } from '../settings.js';
 import { recordInstanceResources, validateInstanceResources } from './calibration.js';
+import { validateTemperatures } from '../../host-temperatures.js';
+import { recordTemperatures } from './temperatures.js';
 import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
 import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js';
 import type {
@@ -622,6 +624,8 @@ export function validateInventory(value: unknown): HostInventory {
         throw new Error('Host inventory must be an object.');
     const inventory = value as HostInventory;
     if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.');
+    if (inventory.temperatures !== undefined)
+        inventory.temperatures = validateTemperatures(inventory.temperatures);
     for (const key of ['hostname', 'agentVersion', 'os', 'architecture', 'cpu', 'dockerVersion']) {
         if (
             inventory[key] !== undefined &&
@@ -924,6 +928,7 @@ export function heartbeat(hostId: string, value: unknown): Host {
 
 function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
     recordInstanceResources(hostId, inventory, timestamp);
+    recordTemperatures(hostId, inventory, timestamp);
     database()
         .prepare(
             `INSERT OR REPLACE INTO ec_host_resource_samples

src/lib/server/hosts/temperatures.ts

@@ -0,0 +1,96 @@
+import type Database from 'better-sqlite3';
+import { db } from '../db/index.js';
+import { historyTemperatures } from '../../host-temperatures.js';
+import type { HostInventory } from '../../hosts.js';
+import type { CapacityQuery, TemperatureHistorySeries } from '../../capacity.js';
+
+export function migrateTemperatures(handle: Database.Database): void {
+    handle.exec(`CREATE TABLE IF NOT EXISTS ec_host_temperature_samples (
+        host_id TEXT NOT NULL REFERENCES ec_hosts(id), bucket TEXT NOT NULL,
+        sensor_id TEXT NOT NULL, kind TEXT NOT NULL, label TEXT NOT NULL,
+        celsius REAL NOT NULL CHECK(celsius BETWEEN -100 AND 200),
+        PRIMARY KEY(host_id,bucket,sensor_id)
+    );
+    CREATE INDEX IF NOT EXISTS ec_host_temperatures_retention ON ec_host_temperature_samples(bucket);`);
+}
+
+export function recordTemperatures(
+    hostId: string,
+    inventory: HostInventory,
+    timestamp: string
+): void {
+    const sensors = historyTemperatures(inventory.temperatures);
+    if (!sensors.length) return;
+    const bucket = new Date(Math.floor(Date.parse(timestamp) / 60_000) * 60_000).toISOString();
+    const insert = db().prepare(`INSERT INTO ec_host_temperature_samples
+        (host_id,bucket,sensor_id,kind,label,celsius) VALUES(?,?,?,?,?,?)
+        ON CONFLICT(host_id,bucket,sensor_id) DO UPDATE SET
+        celsius=MAX(celsius,excluded.celsius),kind=excluded.kind,label=excluded.label`);
+    for (const sensor of sensors)
+        insert.run(hostId, bucket, sensor.id, sensor.kind, sensor.label, sensor.celsius);
+}
+
+/** Runs on the capacity report worker, returning at most 168 points per series. */
+export function readTemperatureHistory(
+    query: CapacityQuery,
+    hosts: { id: string; name: string; created_at: string }[],
+    startAt: number,
+    endAt: number,
+    bucketMs: number
+): TemperatureHistorySeries[] {
+    const count = Math.round((endAt - startAt) / bucketMs);
+    const statement = db().prepare(`SELECT sensor_id,kind,label,
+        MIN(?,MAX(0,CAST((CAST(strftime('%s',bucket) AS INTEGER)*1000-?) / ? AS INTEGER))) AS period,
+        MAX(celsius) AS celsius
+        FROM ec_host_temperature_samples WHERE host_id=? AND bucket>=? AND bucket<=?
+        ${query.hostId ? '' : "AND sensor_id='cpu'"}
+        GROUP BY sensor_id,period ORDER BY sensor_id,period`);
+    const result: TemperatureHistorySeries[] = [];
+    for (const host of hosts) {
+        const bySensor = new Map<string, TemperatureHistorySeries>();
+        const add = (id: string, kind: TemperatureHistorySeries['kind'], label: string) => {
+            const series = {
+                id: `${host.id}:${id}`,
+                hostId: host.id,
+                kind,
+                label,
+                points: Array.from({ length: count }, (_, i) => ({
+                    at: startAt + i * bucketMs,
+                    celsius: null as number | null
+                }))
+            };
+            bySensor.set(id, series);
+            result.push(series);
+            return series;
+        };
+        add('cpu', 'cpu', query.hostId ? 'CPU temperature' : host.name);
+        const rows = statement.all(
+            count - 1,
+            startAt,
+            bucketMs,
+            host.id,
+            new Date(
+                Math.max(
+                    Math.floor(Date.parse(host.created_at) / 60_000) * 60_000,
+                    Math.floor(startAt / 60_000) * 60_000
+                )
+            ).toISOString(),
+            new Date(endAt).toISOString()
+        ) as { sensor_id: string; kind: string; label: string; period: number; celsius: number }[];
+        for (const row of rows) {
+            let series = bySensor.get(row.sensor_id);
+            if (!series) {
+                if (!['nvme', 'chipset'].includes(row.kind) || bySensor.size >= 17) continue;
+                const kind = row.kind as 'nvme' | 'chipset';
+                const number = [...bySensor.values()].filter((s) => s.kind === kind).length + 1;
+                series = add(
+                    row.sensor_id,
+                    kind,
+                    `${kind === 'nvme' ? 'NVMe drive' : 'Chipset'} ${number}`
+                );
+            }
+            if (series.points[row.period]) series.points[row.period].celsius = row.celsius;
+        }
+    }
+    return result;
+}

src/lib/server/provisioning/cancellation.ts

@@ -128,6 +128,9 @@ export function unsettledCancelledCommand(
         WHERE d.server=? AND d.server_deleted_at IS NULL AND d.state='cancelled'
         AND (o.state NOT IN ('completed','failed')
             OR (o.state='completed' AND o.id=n.operation_id AND n.files_json IS NOT NULL AND n.applied_at IS NULL)
+            OR (o.state='completed' AND o.id=n.operation_id AND o.kind='instance.rebuild'
+                AND EXISTS (SELECT 1 FROM native_instances i WHERE i.id=n.instance_id
+                    AND i.retired_at IS NULL AND i.generation<=o.generation))
             OR (o.state='failed' AND o.id=n.operation_id AND o.kind='instance.rebuild'
                 AND json_type(o.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL))
         AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id AND c.cleanup_id=?)

src/lib/server/provisioning/history.ts

@@ -14,6 +14,7 @@ import { fleetSettings } from '../settings.js';
 import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
 import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
 import { bindNativeOperation } from './native.js';
+import { rebuildGenerationRecovery } from './rebuild-receipt.js';
 import { workerActivity } from '../pipeline/activity.js';
 import { canCancelCreation } from './cancel-creation.js';
 import {
@@ -337,6 +338,7 @@ export function getDeployment(
             canResumeCleanup: canResumeCleanup(id),
             canResumeRebuild: canResumeRebuild(id),
             canRetryWithRebuild: canRetryWithRebuild(id),
+            generationRecovery: rebuildGenerationRecovery(id),
             canCancelRemoval: canCancelRemoval(id),
             collection: collection ?? null,
             progress: readProgress(row),

src/lib/server/provisioning/native.ts

@@ -70,7 +70,11 @@ import { recordProgress, type CheckInput } from './progress.js';
 import { trackActivity } from '../pipeline/activity.js';
 import { fleetSettings } from '../settings.js';
 import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
-import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
+import {
+    probeServerStatus,
+    probePlayerEvidence,
+    latestGameSubmission
+} from '../fleet/status-evidence.js';
 import { monitoringIdentity } from '../fleet/health-observations.js';
 import { handoffVerifiedHealth } from './health-handoff.js';
 import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
@@ -85,6 +89,12 @@ import { reconcileHostDrains } from './host-drain.js';
 import { hostMigration, isRebalanceMigration, processMigrationRemoval } from './migrations.js';
 import { safetyOverride, safetyOverrideCapability } from './safety-override.js';
 import { runtimeLaunchSql } from '../fleet/runtime-config.js';
+import {
+    reconcileCompletedRebuild,
+    assertVerificationGeneration,
+    GenerationRecoveryRequired,
+    observedGeneration
+} from './rebuild-receipt.js';
 
 export type NativeSelection = import('../../hosts.js').NativePlacement;
 type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
@@ -813,27 +823,11 @@ export function reconcileNativeProvisioningReceipt(id: string): void {
         finishCreationCancellation(job.id);
         return;
     }
-    if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) {
-        db()
-            .prepare('UPDATE native_instances SET generation=generation+1 WHERE id=?')
-            .run(instance.id);
-        // Replacements queued while the delivered rebuild was settling bound the
-        // previous generation. Only untouched, newer jobs may follow this receipt.
-        if (job.state === 'cancelled')
-            db()
-                .prepare(
-                    `UPDATE native_provisioning SET generation=?
-            WHERE instance_id=? AND generation=? AND operation_id IS NULL AND stage_operation_id IS NULL
-            AND deployment_id IN (SELECT newer.id FROM server_deployments newer
-                WHERE newer.rowid>(SELECT rowid FROM server_deployments WHERE id=?)
-                AND newer.server_deleted_at IS NULL AND newer.state IN ('queued','waiting'))`
-                )
-                .run(job.generation + 1, instance.id, job.generation, id);
-    } else if (
-        instance.generation !== job.generation &&
-        !(operation.kind === 'instance.rebuild' && instance.generation === job.generation + 1)
-    )
-        return;
+    if (operation.kind === 'instance.rebuild') {
+        // An older applied receipt must never undo a later successful replacement.
+        if (job.applied_at && instance.generation > operation.generation! + 1) return;
+        reconcileCompletedRebuild(id);
+    } else if (instance.generation !== job.generation) return;
     applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
     db()
         .prepare(
@@ -1032,10 +1026,12 @@ export async function processNativeProvisioning(
             `SELECT d.id FROM server_deployments d
         JOIN native_provisioning n ON n.deployment_id=d.id
         LEFT JOIN ec_host_operations o ON o.id=n.operation_id
+        LEFT JOIN native_instances i ON i.id=n.instance_id AND i.retired_at IS NULL
         LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
         LEFT JOIN ec_host_operations removal ON removal.id=m.remove_operation_id
         WHERE d.state='cancelled' AND d.server_deleted_at IS NULL AND (
-            (o.state='completed' AND (n.applied_at IS NULL OR d.action='destroy'))
+            (o.state='completed' AND (n.applied_at IS NULL OR d.action='destroy'
+                OR (o.kind='instance.rebuild' AND i.generation=o.generation)))
             OR (o.state='failed' AND o.kind='instance.rebuild' AND json_type(o.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL)
             OR (o.state='failed' AND EXISTS (SELECT 1 FROM staged_file_changes f JOIN provisioning_sources s
                 ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
@@ -1061,7 +1057,8 @@ export async function processNativeProvisioning(
             `SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.applied_at,n.recovery_step,q.selection
         FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id
         LEFT JOIN native_creation_intents q ON q.deployment_id=d.id
-        WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`
+        WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
+        AND COALESCE(CASE WHEN json_valid(d.observation_json) THEN json_extract(d.observation_json,'$.blocker.code') END,'')!='generation-recovery' ORDER BY d.started_at,d.rowid`
         )
         .all() as Job[];
     // One host's observation must not prevent another job being admitted.
@@ -1075,7 +1072,27 @@ export async function processNativeProvisioning(
             () => processJob(job, eligibility)
         )
             .catch((error) => {
-                if (stillActive(job)) hold(job, (error as Error).message);
+                if (!stillActive(job)) return;
+                if (
+                    error instanceof GenerationRecoveryRequired ||
+                    /generation has been superseded or removed/i.test((error as Error).message)
+                ) {
+                    const instance = nativeInstance(job.server);
+                    const message =
+                        error instanceof GenerationRecoveryRequired
+                            ? error.message
+                            : `Generation recovery required: controller expects ${instance?.generation ?? job.generation}, host reports ${instance ? (observedGeneration(instance) ?? 'unknown') : 'unknown'}. Reconcile the completed rebuild before resuming verification.`;
+                    db()
+                        .prepare(
+                            "UPDATE server_deployments SET state='interrupted',error=? WHERE id=?"
+                        )
+                        .run(message, job.id);
+                    deploymentStage(job.id, message, {
+                        blocker: { code: 'generation-recovery', message, automatic: false },
+                        checks: [],
+                        recovery: { state: 'held', operationId: job.id, message }
+                    });
+                } else hold(job, (error as Error).message);
             })
             .finally(() => {
                 activeJobs.delete(job.id);
@@ -1328,6 +1345,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         job.instance_id = instance.id;
         job.generation = instance.generation;
     }
+    // Reconcile before comparing the job snapshot or issuing any observations.
+    // Historical jobs can have a stale binding even though their command is correct.
+    if (job.operation_id) {
+        const receipt = getHostOperationStatus(job.operation_id);
+        if (receipt?.kind === 'instance.rebuild' && receipt.state === 'completed') {
+            instance = reconcileCompletedRebuild(job.id);
+            job.generation = receipt.generation!;
+        }
+    }
     const migration = hostMigration(job.id);
     const rebuilding =
         job.recovery_step === 'rebuild' ||
@@ -1336,7 +1362,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         !instance ||
         instance.id !== job.instance_id ||
         (instance.generation !== job.generation &&
-            !(rebuilding && instance.generation === job.generation + 1))
+            !(rebuilding && job.operation_id && instance.generation === job.generation + 1))
     ) {
         finishDeployment(job.id, {
             ok: false,
@@ -1493,12 +1519,6 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             })();
             return;
         }
-        if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) {
-            db()
-                .prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?')
-                .run(job.generation + 1, instance.id, job.generation);
-            instance = nativeInstance(job.server)!;
-        }
         applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
         db()
             .prepare(
@@ -1542,6 +1562,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             blocker: null,
             checks
         });
+        assertVerificationGeneration(instance, operation.updatedAt);
         const verificationIdentity = monitoringIdentity(job.server);
         const health = await nativeRequest<{ healthy: boolean; state: string }>(
             instance,
@@ -1601,7 +1622,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             return;
         }
         deploymentStage(job.id, 'Checking game status', { phase: 'verify-game', blocker: null });
-        const status = await probeServerStatus(job.server, operation.updatedAt);
+        const submission = latestGameSubmission(job.server);
+        const status = await probeServerStatus(
+            job.server,
+            submission && submission > operation.updatedAt ? submission : operation.updatedAt
+        );
         if (!stillActive(job)) return;
         check(
             {
@@ -2255,8 +2280,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                   )
                 : queueHostOperation(instance!.host_id, kind, payload, operationOptions);
             db()
-                .prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?')
-                .run(operation.id, job.id);
+                .prepare(
+                    'UPDATE native_provisioning SET operation_id=?,generation=? WHERE deployment_id=?'
+                )
+                .run(operation.id, instance!.generation, job.id);
             for (const file of files)
                 db()
                     .prepare(

src/lib/server/provisioning/rebuild-receipt.ts

@@ -0,0 +1,173 @@
+import { db } from '../db/index.js';
+import { nativeInstance, type NativeInstance } from '../backends/native.js';
+import { getHost, getHostOperationStatus } from '../hosts/index.js';
+import { recordProgress } from './progress.js';
+
+export class GenerationRecoveryRequired extends Error {}
+
+/** Inventory is evidence only when it belongs to this host and immutable instance. */
+export function observedGeneration(instance: NativeInstance, notBefore?: string): number | null {
+    const host = getHost(instance.host_id);
+    if (!host?.connected) return null;
+    const states = host.inventory.instanceStates?.filter((state) => {
+        const age = Date.now() - Date.parse(state.observedAt);
+        return (
+            state.instanceId === instance.id &&
+            age >= 0 &&
+            age <= 30_000 &&
+            (!notBefore || Date.parse(state.observedAt) >= Date.parse(notBefore))
+        );
+    });
+    return states?.length === 1 ? states[0].generation : null;
+}
+
+function receipt(id: string) {
+    const binding = db()
+        .prepare(
+            `SELECT d.server,n.instance_id,n.generation,n.operation_id
+        FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
+        WHERE d.id=? AND d.server_deleted_at IS NULL
+        AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id)`
+        )
+        .get(id) as
+        | { server: string; instance_id: string; generation: number; operation_id: string }
+        | undefined;
+    const instance = binding && nativeInstance(binding.server);
+    const operation = binding?.operation_id && getHostOperationStatus(binding.operation_id);
+    if (
+        !binding ||
+        !instance ||
+        !operation ||
+        operation.kind !== 'instance.rebuild' ||
+        operation.state !== 'completed' ||
+        operation.error ||
+        instance.id !== binding.instance_id ||
+        operation.instanceId !== instance.id ||
+        operation.hostId !== instance.host_id ||
+        !Number.isSafeInteger(operation.generation) ||
+        operation.generation! < 1
+    ) {
+        throw new GenerationRecoveryRequired(
+            'Recovery requires a completed rebuild for this host and immutable instance ID. Inspect the original host command.'
+        );
+    }
+    const requested = db()
+        .prepare(
+            `SELECT COALESCE(
+        json_extract(COALESCE(p.payload,o.payload),'$.nextGeneration'),
+        json_extract(COALESCE(p.payload,o.payload),'$.newGeneration')) AS generation
+        FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`
+        )
+        .get(operation.id) as { generation: number | null };
+    const generation = requested.generation;
+    const result = operation.result as { instanceId?: unknown; generation?: unknown } | null;
+    if (
+        !Number.isSafeInteger(generation) ||
+        generation !== operation.generation! + 1 ||
+        (result?.instanceId !== undefined && result.instanceId !== instance.id) ||
+        (result?.generation !== undefined && result.generation !== generation) ||
+        ![operation.generation, generation].includes(instance.generation)
+    ) {
+        throw new GenerationRecoveryRequired(
+            'The completed rebuild does not match the registered instance and requested replacement generation. Inspect its host command; no generation was adopted.'
+        );
+    }
+    return { binding, instance, operation, generation: generation! };
+}
+
+/** Read-only availability for the operator recovery action, including old cancelled jobs. */
+export function rebuildGenerationRecovery(id: string) {
+    try {
+        const { binding, instance, operation, generation } = receipt(id);
+        const observed = observedGeneration(instance, operation.updatedAt);
+        if (instance.generation === generation && binding.generation === operation.generation)
+            return null;
+        return {
+            expectedGeneration: instance.generation,
+            replacementGeneration: generation,
+            observedGeneration: observed,
+            available: observed === generation,
+            reason:
+                observed === generation
+                    ? null
+                    : `Waiting for fresh inventory from the original host confirming generation ${generation} for this instance.`
+        };
+    } catch {
+        return null;
+    }
+}
+
+/** The durable command, not a potentially stale job snapshot, owns the transition. */
+export function reconcileCompletedRebuild(id: string, requireInventory = false): NativeInstance {
+    return db().transaction(() => {
+        const { binding, instance, operation, generation } = receipt(id);
+        const observed = observedGeneration(instance, operation.updatedAt);
+        if (
+            (requireInventory && observed !== generation) ||
+            (observed !== null && observed !== generation)
+        ) {
+            throw new GenerationRecoveryRequired(
+                `Generation recovery required: controller expects ${instance.generation}, completed rebuild requests ${generation}, host reports ${observed ?? 'unknown'}. Refresh the host inventory and reconcile the completed rebuild.`
+            );
+        }
+        const changed = instance.generation !== generation;
+        const repaired = binding.generation !== operation.generation;
+        if (changed) {
+            const at = new Date().toISOString();
+            db()
+                .prepare(
+                    `UPDATE native_instances SET generation=?,generation_reconciled_at=?
+                WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL`
+                )
+                .run(generation, at, instance.id, instance.host_id, instance.generation);
+            db()
+                .prepare(
+                    `UPDATE servers SET container_state='unknown',container_checked_at=?,
+                last_checked_at=NULL,last_players=NULL,last_roster=NULL WHERE name=?`
+                )
+                .run(at, instance.server);
+        }
+        db()
+            .prepare(
+                'UPDATE native_provisioning SET generation=? WHERE deployment_id=? AND operation_id=?'
+            )
+            .run(operation.generation, id, operation.id);
+        // Untouched successors may follow a cancelled, delivered rebuild. Already
+        // submitted commands retain their immutable target and must never be retargeted.
+        db()
+            .prepare(
+                `UPDATE native_provisioning SET generation=? WHERE instance_id=? AND generation=?
+            AND operation_id IS NULL AND stage_operation_id IS NULL
+            AND deployment_id IN (SELECT newer.id FROM server_deployments newer
+                WHERE newer.rowid>(SELECT rowid FROM server_deployments WHERE id=?)
+                AND newer.submitted_at IS NULL AND newer.server_deleted_at IS NULL
+                AND newer.state IN ('queued','waiting'))`
+            )
+            .run(generation, instance.id, instance.generation, id);
+        if (changed || repaired)
+            recordProgress(id, {
+                checks: [],
+                event: {
+                    kind: 'generation-reconciled',
+                    message: `Recorded replacement generation ${generation}; fresh health verification is required.`,
+                    details: {
+                        operationId: operation.id,
+                        hostId: instance.host_id,
+                        instanceId: instance.id,
+                        previousGeneration: instance.generation,
+                        generation
+                    }
+                }
+            });
+        return nativeInstance(instance.server)!;
+    })();
+}
+
+export function assertVerificationGeneration(instance: NativeInstance, notBefore: string): void {
+    const observed = observedGeneration(instance, notBefore);
+    if (observed !== null && observed !== instance.generation) {
+        throw new GenerationRecoveryRequired(
+            `Generation recovery required: controller expects ${instance.generation}, host reports ${observed}. Reconcile the completed rebuild before resuming verification.`
+        );
+    }
+}

src/lib/server/retention.ts

@@ -31,7 +31,7 @@ const unsettledDeployment = `d.state NOT IN ${terminal}
             OR (a.id=n.operation_id AND a.state='completed' AND (
             (n.applied_at IS NULL AND COALESCE(json_array_length(n.files_json),0)>0)
             OR (d.action='destroy' AND i.retired_at IS NULL)
-            OR (a.kind='instance.rebuild' AND i.retired_at IS NULL AND i.generation=n.generation)))))
+            OR (a.kind='instance.rebuild' AND i.retired_at IS NULL AND i.generation<=a.generation)))))
     OR EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id
         WHERE m.deployment_id=d.id AND (a.state NOT IN ('completed','failed') OR (a.state='completed' AND m.removed_at IS NULL)))`;
 // Recovery replaces operation_id with its latest command. The original launch
@@ -556,6 +556,14 @@ function samplePolicies(s = fleetSettings()) {
             value: iso(s.hostResourceKeepDays),
             enabled: s.hostResourceKeepDays > 0
         },
+        {
+            category: 'host-temperatures',
+            label: 'Host temperature samples',
+            table: 'ec_host_temperature_samples',
+            condition: 'bucket < ?',
+            value: iso(s.hostResourceKeepDays),
+            enabled: s.hostResourceKeepDays > 0
+        },
         {
             category: 'instance-resources',
             label: 'Instance resource samples',

src/lib/ui/components/HostList.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
     import type { Host } from '$lib/hosts.js';
+    import { cpuTemperature, formatTemperature } from '$lib/host-temperatures.js';
     import { contextHref } from '$lib/navigation-context.js';
     import {
         hostListStatus,
@@ -68,7 +69,13 @@
                             ? '—'
                             : `${host.playerCount}${host.unreportedInstances ? '+' : ''}`}</td
                     >
-                    <td class:stale>{hostCpuUsage(host.inventory.cpuUsagePercent)}</td>
+                    <td class:stale
+                        >{hostCpuUsage(host.inventory.cpuUsagePercent)}<span class="secondary"
+                            >CPU temp: {formatTemperature(
+                                cpuTemperature(host.inventory.temperatures)?.celsius
+                            )}</span
+                        ></td
+                    >
                     <td class:stale
                         >{hostUsedSpace(
                             host.inventory.memoryTotal,

src/lib/ui/components/HostTemperatures.svelte

@@ -0,0 +1,79 @@
+<script lang="ts">
+    import type { Host } from '$lib/hosts.js';
+    import { cpuTemperature, formatTemperature, temperatureLabel } from '$lib/host-temperatures.js';
+    import { hostReadingsStale } from '$ui/host-overview.js';
+    import { onMount } from 'svelte';
+    let { host }: { host: Host } = $props();
+    let now = $state(Date.now());
+    const stale = $derived(hostReadingsStale(host, now));
+    const sensors = $derived(host.inventory.temperatures ?? []);
+    const cpu = $derived(cpuTemperature(sensors));
+    onMount(() => {
+        const clock = setInterval(() => (now = Date.now()), 15_000);
+        return () => clearInterval(clock);
+    });
+</script>
+
+<div class:stale>
+    {formatTemperature(cpu?.celsius)}{#if stale && cpu}
+        (last reported){/if}
+    {#if cpu}<span class="source">{temperatureLabel(cpu)}</span>{/if}
+    {#if sensors.length}
+        <details>
+            <summary>Temperature sensors</summary>
+            <ul>
+                {#each sensors as sensor (sensor.id)}
+                    <li>
+                        <span
+                            >{temperatureLabel(sensor)}<small
+                                >{sensor.device} · {sensor.label} · {sensor.id.slice(-6)}</small
+                            ></span
+                        ><span>{formatTemperature(sensor.celsius)}</span>
+                    </li>
+                {/each}
+            </ul>
+            {#if stale}<p>Last reported readings; waiting for a fresh host heartbeat.</p>{/if}
+        </details>
+    {:else}<span class="source">No supported temperature sensors reported.</span>{/if}
+</div>
+
+<style>
+    .source,
+    small {
+        display: block;
+        color: var(--text-secondary);
+        font-size: var(--text-xs);
+        margin-top: var(--space-1);
+    }
+    .stale {
+        color: var(--text-secondary);
+    }
+    details {
+        margin-top: var(--space-2);
+    }
+    summary {
+        cursor: pointer;
+        font-size: var(--text-sm);
+    }
+    summary:focus-visible {
+        outline: 2px solid var(--accent-primary);
+        outline-offset: 3px;
+    }
+    ul {
+        padding: 0;
+        list-style: none;
+        margin: var(--space-3) 0;
+        display: grid;
+        gap: var(--space-3);
+    }
+    li {
+        display: flex;
+        justify-content: space-between;
+        flex-wrap: wrap;
+        gap: var(--space-2);
+        font-variant-numeric: tabular-nums;
+    }
+    p {
+        font-size: var(--text-xs);
+    }
+</style>

src/lib/ui/components/capacity/TemperatureChart.svelte

@@ -0,0 +1,456 @@
+<script lang="ts">
+    import type { Snippet } from 'svelte';
+    import type { CapacityHistory, TemperatureHistorySeries } from '$lib/capacity.js';
+    import { formatTemperature } from '$lib/host-temperatures.js';
+    import { timeFormat } from '$ui/stores/time-zone.js';
+
+    let {
+        history,
+        scope,
+        controls
+    }: { history: CapacityHistory; scope: string; controls?: Snippet } = $props();
+    let visibility = $state<Record<string, boolean>>({});
+    let selectedAt = $state<number | null>(null);
+    const series = $derived(history.temperatures);
+    const visible = $derived(series.filter((s) => visibility[s.id] ?? s.kind === 'cpu'));
+    const values = $derived(
+        series.flatMap((s) => s.points.flatMap((p) => (p.celsius === null ? [] : [p.celsius])))
+    );
+    const minimum = $derived(
+        values.length
+            ? Math.floor((values.reduce((a, b) => Math.min(a, b), Infinity) - 5) / 10) * 10
+            : 0
+    );
+    const maximum = $derived(
+        values.length
+            ? Math.ceil((values.reduce((a, b) => Math.max(a, b), -Infinity) + 5) / 10) * 10
+            : 100
+    );
+    const times = $derived(history.points.map((p) => p.at));
+    const measured = $derived(visible.some((s) => s.points.some((p) => p.celsius !== null)));
+    const selectedIndex = $derived.by(() => {
+        if (selectedAt !== null)
+            return times.reduce(
+                (best, at, i) =>
+                    Math.abs(at - selectedAt!) < Math.abs(times[best] - selectedAt!) ? i : best,
+                0
+            );
+        for (let i = times.length - 1; i >= 0; i--)
+            if (visible.some((s) => s.points[i]?.celsius != null)) return i;
+        return Math.max(0, times.length - 1);
+    });
+    const selectedTime = $derived(times[selectedIndex]);
+    const readout = $derived(
+        visible
+            .map((s) => `${s.label}: ${formatTemperature(s.points[selectedIndex]?.celsius)}`)
+            .join('. ')
+    );
+    const ticks = [0, 0.25, 0.5, 0.75, 1];
+    function color(s: TemperatureHistorySeries) {
+        return `var(--chart-series-${(series.indexOf(s) % 6) + 1})`;
+    }
+    function dash(s: TemperatureHistorySeries) {
+        return ['', '7 4', '2 4'][Math.floor(series.indexOf(s) / 6) % 3];
+    }
+    function x(at: number) {
+        return (
+            8 +
+            984 *
+                Math.max(
+                    0,
+                    Math.min(
+                        1,
+                        (at - history.startAt) / Math.max(1, history.endAt - history.startAt)
+                    )
+                )
+        );
+    }
+    function y(celsius: number) {
+        return 230 - (220 * (celsius - minimum)) / (maximum - minimum);
+    }
+    function segments(s: TemperatureHistorySeries) {
+        const result: { x: number; y: number }[][] = [];
+        let segment: { x: number; y: number }[] = [];
+        for (const point of s.points) {
+            if (point.celsius === null) {
+                if (segment.length) result.push(segment);
+                segment = [];
+            } else segment.push({ x: x(point.at), y: y(point.celsius) });
+        }
+        if (segment.length) result.push(segment);
+        return result;
+    }
+    function inspect(event: PointerEvent) {
+        if (!times.length || (event.type === 'pointermove' && event.pointerType === 'touch'))
+            return;
+        const rect = (event.currentTarget as HTMLElement).getBoundingClientRect();
+        selectedAt =
+            history.startAt +
+            Math.max(
+                0,
+                Math.min(1, (((event.clientX - rect.left) / rect.width) * 1000 - 8) / 984)
+            ) *
+                (history.endAt - history.startAt);
+    }
+    function move(event: KeyboardEvent) {
+        if (!times.length || !['ArrowLeft', 'ArrowRight', 'Home', 'End'].includes(event.key))
+            return;
+        event.preventDefault();
+        const next =
+            event.key === 'Home'
+                ? 0
+                : event.key === 'End'
+                  ? times.length - 1
+                  : Math.max(
+                        0,
+                        Math.min(
+                            times.length - 1,
+                            selectedIndex + (event.key === 'ArrowLeft' ? -1 : 1)
+                        )
+                    );
+        selectedAt = times[next];
+    }
+</script>
+
+<section class="panel" id="temperatures" aria-labelledby="temperature-history-title">
+    <div class="heading">
+        <div>
+            <h2 id="temperature-history-title">Temperature history</h2>
+            <p>{scope} · Peak temperature per period</p>
+        </div>
+        {@render controls?.()}
+    </div>
+    <div class="legend" role="group" aria-label="Temperature series">
+        {#each series as sensor (sensor.id)}
+            <button
+                type="button"
+                aria-pressed={visibility[sensor.id] ?? sensor.kind === 'cpu'}
+                onclick={() =>
+                    (visibility[sensor.id] = !(visibility[sensor.id] ?? sensor.kind === 'cpu'))}
+            >
+                <svg viewBox="0 0 24 8" aria-hidden="true"
+                    ><line
+                        x1="0"
+                        x2="24"
+                        y1="4"
+                        y2="4"
+                        stroke={color(sensor)}
+                        stroke-dasharray={dash(sensor)}
+                    /></svg
+                >
+                {sensor.label}
+            </button>
+        {/each}
+    </div>
+    <div class="axis-title">Temperature (°C)</div>
+    <div class="chart-grid">
+        <div class="y-axis" aria-hidden="true">
+            {#each ticks as tick}<span style:top={`${(1 - tick) * 100}%`}
+                    >{minimum + (maximum - minimum) * tick}</span
+                >{/each}
+        </div>
+        <div
+            class="plot"
+            role="slider"
+            tabindex="0"
+            aria-label="Temperature history. Use left and right arrows to inspect a period."
+            aria-valuemin="0"
+            aria-valuemax={Math.max(0, times.length - 1)}
+            aria-valuenow={selectedIndex}
+            aria-valuetext={selectedTime === undefined
+                ? 'No temperature history'
+                : `${$timeFormat.period(selectedTime)}. ${readout || 'No sensors selected'}`}
+            onpointermove={inspect}
+            onpointerdown={inspect}
+            onkeydown={move}
+        >
+            <svg viewBox="0 0 1000 240" preserveAspectRatio="none" aria-hidden="true">
+                {#each ticks as tick}<line
+                        class="grid-line"
+                        x1="0"
+                        x2="1000"
+                        y1={y(minimum + (maximum - minimum) * tick)}
+                        y2={y(minimum + (maximum - minimum) * tick)}
+                    />{/each}
+                {#each visible as sensor (sensor.id)}
+                    <g data-series={sensor.id}>
+                        {#each segments(sensor) as segment}
+                            <path
+                                class="trend"
+                                stroke={color(sensor)}
+                                stroke-dasharray={dash(sensor)}
+                                d={segment
+                                    .map((point, i) => `${i ? 'L' : 'M'}${point.x},${point.y}`)
+                                    .join(' ')}
+                            />
+                            {#if segment.length === 1}<circle
+                                    cx={segment[0].x}
+                                    cy={segment[0].y}
+                                    r="3"
+                                    fill={color(sensor)}
+                                />{/if}
+                        {/each}
+                        {#if sensor.points[selectedIndex]?.celsius != null}<circle
+                                class="selected-point"
+                                cx={x(selectedTime)}
+                                cy={y(sensor.points[selectedIndex].celsius!)}
+                                r="4"
+                                fill={color(sensor)}
+                            />{/if}
+                    </g>
+                {/each}
+                {#if selectedTime !== undefined && measured}<line
+                        class="cursor"
+                        x1={x(selectedTime)}
+                        x2={x(selectedTime)}
+                        y1="5"
+                        y2="235"
+                    />{/if}
+            </svg>
+            {#if !measured}<div class="empty-chart">
+                    <strong
+                        >{visible.length || !series.length
+                            ? 'No temperature readings in this period'
+                            : 'Select a sensor to show history'}</strong
+                    >
+                    {#if visible.length || !series.length}<span
+                            >Readings appear when an agent reports supported temperature sensors.</span
+                        >{/if}
+                </div>{/if}
+        </div>
+        <div class="x-axis" aria-hidden="true">
+            <span>{$timeFormat.period(history.startAt)}</span><span
+                >{$timeFormat.period(history.endAt)}</span
+            >
+        </div>
+    </div>
+    {#if selectedTime !== undefined && visible.length}
+        <div class="readout" aria-label="Selected temperature period">
+            <span>{$timeFormat.period(selectedTime)}</span>
+            <dl>
+                {#each visible as sensor (sensor.id)}<div>
+                        <dt>{sensor.label}</dt>
+                        <dd>{formatTemperature(sensor.points[selectedIndex]?.celsius)}</dd>
+                    </div>{/each}
+            </dl>
+        </div>
+    {/if}
+    <details>
+        <summary>About this chart</summary>
+        <p>
+            Each CPU line represents one host: its hottest reported package or physical die, falling
+            back to its hottest core. Select a host to compare its CPU, NVMe drives and chipset.
+            Per-minute peaks are retained; longer periods show their highest reading. Gaps mean no
+            measurement was received in that period. Temperatures include all workloads and are
+            unaffected by the game filter. Colors identify series; they do not indicate safe or
+            unsafe temperatures.
+        </p>
+    </details>
+</section>
+
+<style>
+    .panel {
+        background: var(--surface);
+        border: 1px solid var(--border-subtle);
+        border-radius: var(--radius-lg);
+        padding: var(--space-5);
+        min-width: 0;
+    }
+    h2 {
+        margin: 0;
+        font-size: var(--text-lg);
+    }
+    .heading {
+        display: flex;
+        align-items: flex-start;
+        justify-content: space-between;
+        gap: var(--space-4);
+        flex-wrap: wrap;
+    }
+    .heading p {
+        color: var(--text-secondary);
+        font-size: var(--text-sm);
+        margin: var(--space-2) 0 0;
+    }
+    .legend {
+        display: flex;
+        flex-wrap: wrap;
+        gap: var(--space-2) var(--space-5);
+        margin: var(--space-4) 0;
+    }
+    .legend button {
+        display: flex;
+        align-items: center;
+        gap: var(--space-2);
+        background: none;
+        border: 0;
+        padding: var(--space-2) 0;
+        color: var(--text-primary);
+        font: inherit;
+        font-size: var(--text-sm);
+        cursor: pointer;
+        overflow-wrap: anywhere;
+        max-width: 100%;
+    }
+    .legend button[aria-pressed='false'] {
+        color: var(--text-secondary);
+    }
+    .legend button[aria-pressed='false'] svg {
+        opacity: 0.35;
+    }
+    .legend svg {
+        width: 24px;
+        height: 8px;
+        flex-shrink: 0;
+    }
+    .legend line {
+        stroke-width: 3;
+    }
+    button:focus-visible,
+    .plot:focus-visible,
+    summary:focus-visible {
+        outline: 2px solid var(--accent-primary);
+        outline-offset: 3px;
+    }
+    .axis-title,
+    .x-axis,
+    .y-axis {
+        color: var(--text-tertiary);
+        font-size: var(--text-xs);
+    }
+    .chart-grid {
+        display: grid;
+        grid-template-columns: 42px minmax(0, 1fr);
+        gap: 10px;
+        margin-top: 14px;
+    }
+    .y-axis {
+        position: relative;
+        height: 220px;
+        margin-top: 10px;
+    }
+    .y-axis span {
+        position: absolute;
+        right: 0;
+        transform: translateY(-50%);
+    }
+    .plot {
+        position: relative;
+        height: 240px;
+        cursor: crosshair;
+        touch-action: pan-y;
+    }
+    .plot > svg {
+        display: block;
+        width: 100%;
+        height: 100%;
+        overflow: visible;
+    }
+    .plot path,
+    .plot line {
+        vector-effect: non-scaling-stroke;
+    }
+    .grid-line {
+        stroke: var(--border-subtle);
+        stroke-width: 1;
+    }
+    .trend {
+        fill: none;
+        stroke-width: 2;
+        stroke-linejoin: round;
+        stroke-linecap: round;
+    }
+    .selected-point {
+        stroke: var(--surface);
+        stroke-width: 2;
+    }
+    .cursor {
+        stroke: var(--text-tertiary);
+        stroke-width: 1;
+        stroke-dasharray: 4 5;
+    }
+    .x-axis {
+        grid-column: 2;
+        display: flex;
+        justify-content: space-between;
+        gap: var(--space-4);
+    }
+    .x-axis span:last-child {
+        text-align: right;
+    }
+    .empty-chart {
+        position: absolute;
+        inset: 0;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        flex-direction: column;
+        gap: var(--space-2);
+        padding: var(--space-4);
+        text-align: center;
+        pointer-events: none;
+    }
+    .empty-chart strong {
+        font-size: var(--text-sm);
+        font-weight: var(--weight-medium);
+    }
+    .empty-chart span {
+        font-size: var(--text-xs);
+        color: var(--text-secondary);
+    }
+    .readout {
+        display: flex;
+        flex-wrap: wrap;
+        gap: var(--space-4);
+        font-size: var(--text-xs);
+        margin: var(--space-4) 0;
+    }
+    .readout > span,
+    dt {
+        color: var(--text-secondary);
+    }
+    dl {
+        display: flex;
+        flex-wrap: wrap;
+        gap: var(--space-3) var(--space-5);
+        margin: 0;
+    }
+    dl > div {
+        display: flex;
+        flex-wrap: wrap;
+        min-width: 0;
+        gap: var(--space-2);
+    }
+    dt {
+        overflow-wrap: anywhere;
+    }
+    dd {
+        margin: 0;
+        font-variant-numeric: tabular-nums;
+    }
+    summary {
+        cursor: pointer;
+        font-size: var(--text-xs);
+        color: var(--text-secondary);
+        padding: var(--space-2) 0;
+    }
+    details p {
+        color: var(--text-secondary);
+        font-size: var(--text-xs);
+        line-height: 1.65;
+    }
+    @media (max-width: 640px) {
+        .panel {
+            padding: var(--space-4);
+        }
+        .chart-grid {
+            grid-template-columns: 32px minmax(0, 1fr);
+            gap: 8px;
+        }
+    }
+    @media (pointer: coarse) {
+        .legend button {
+            min-height: 44px;
+        }
+    }
+</style>

src/lib/ui/provisioning-refresh.ts

@@ -1,5 +1,5 @@
 export type ProvisioningMutation =
-    'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override';
+    'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override' | 'reconcile-generation';
 
 export interface ProvisioningRefreshState<T> {
     id: string;

src/lib/ui/styles/tokens.css

@@ -78,6 +78,12 @@
 
     /* Metrics: unknown uses a pattern as well as color. */
     --chart-active: var(--accent-primary);
+    --chart-series-1: var(--accent-primary);
+    --chart-series-2: #b99af7;
+    --chart-series-3: #69c6ba;
+    --chart-series-4: #dfb76c;
+    --chart-series-5: #df91b6;
+    --chart-series-6: #bbc7db;
     --chart-idle: #2b3442;
     --chart-unknown-base: #605b64;
     --chart-unknown-stripe: #a7a0ad;

src/routes/api/provisioning/[id]/reconcile-generation/+server.ts

@@ -0,0 +1,26 @@
+import { json } from '@sveltejs/kit';
+import { reconcileCompletedRebuild } from '$lib/server/provisioning/rebuild-receipt.js';
+import { reconcileNativeProvisioningReceipt } from '$lib/server/provisioning/native.js';
+import { db } from '$lib/server/db/index.js';
+import { recordProgress } from '$lib/server/provisioning/progress.js';
+
+export function POST({ params }: { params: { id: string } }) {
+    try {
+        const instance = db().transaction(() => {
+            const instance = reconcileCompletedRebuild(params.id, true);
+            reconcileNativeProvisioningReceipt(params.id);
+            recordProgress(params.id, {
+                blocker: null,
+                checks: [],
+                event: {
+                    kind: 'generation-recovery',
+                    message: `Replacement generation ${instance.generation} reconciled with fresh host inventory. Resume health verification or request a new lifecycle operation.`
+                }
+            });
+            return instance;
+        })();
+        return json({ ok: true, generation: instance.generation });
+    } catch (error) {
+        return json({ error: (error as Error).message }, { status: 409 });
+    }
+}

src/routes/capacity/+page.svelte

@@ -17,6 +17,7 @@
     import Select from '$ui/components/primitives/Select.svelte';
     import CapacityHosts from '$ui/components/capacity/CapacityHosts.svelte';
     import CapacityChart from '$ui/components/capacity/CapacityChart.svelte';
+    import TemperatureChart from '$ui/components/capacity/TemperatureChart.svelte';
     import RebalancePanel from '$ui/components/capacity/RebalancePanel.svelte';
     import Freshness from '$ui/components/Freshness.svelte';
     import { plural } from '$ui/plural.js';
@@ -319,16 +320,20 @@
                 : 'Waiting for host reports.'} Refreshes every 30 seconds while visible.
         </p>
     </details>
-    <CapacityChart history={report.history} scope={scopeName}>
-        {#snippet controls()}<nav class="ranges" aria-label="History range">
-                {#each ranges as range}<a
-                        href={scopeUrl({ range: range.value })}
-                        aria-current={report.query.range === range.value ? 'page' : undefined}
-                        data-sveltekit-keepfocus
-                        data-sveltekit-noscroll>{range.label}</a
-                    >{/each}
-            </nav>{/snippet}
-    </CapacityChart>
+    {#snippet historyRanges()}<nav class="ranges" aria-label="History range">
+            {#each ranges as range}<a
+                    href={scopeUrl({ range: range.value })}
+                    aria-current={report.query.range === range.value ? 'page' : undefined}
+                    data-sveltekit-keepfocus
+                    data-sveltekit-noscroll>{range.label}</a
+                >{/each}
+        </nav>{/snippet}
+    <CapacityChart history={report.history} scope={scopeName} controls={historyRanges} />
+    {#key report.query.hostId}<TemperatureChart
+            history={report.history}
+            scope={scopeName}
+            controls={historyRanges}
+        />{/key}
 </div>
 
 <style>

src/routes/hosts/[id]/+page.svelte

@@ -28,6 +28,7 @@
     import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
     import HostPreparation from '$ui/components/HostPreparation.svelte';
     import CapacitySamples from '$ui/components/CapacitySamples.svelte';
+    import HostTemperatures from '$ui/components/HostTemperatures.svelte';
     import ServerCard from '$ui/components/ServerCard.svelte';
     import type { ServerObservation } from '$lib/server-observation.js';
     import { syncServerTime } from '$ui/stores/servers.js';
@@ -300,6 +301,8 @@
                                 : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined}
                                 (last reported){/if}
                         </dd>
+                        <dt>CPU temperature</dt>
+                        <dd><HostTemperatures {host} /></dd>
                         <dt>Memory available / total</dt>
                         <dd>
                             {bytes(host.inventory.memoryAvailable)} / {bytes(

src/routes/provisioning/[id]/+page.svelte

@@ -324,6 +324,27 @@
                         )}.
                     </p>
                 </div>{/if}
+            {#if detail.generationRecovery}<div class="warnings">
+                    <h3>Replacement generation recovery</h3>
+                    <p>
+                        Controller generation {detail.generationRecovery.expectedGeneration};
+                        completed rebuild generation {detail.generationRecovery
+                            .replacementGeneration}; host generation {detail.generationRecovery
+                            .observedGeneration ?? 'unknown'}.
+                    </p>
+                    {#if detail.generationRecovery.reason}<p>
+                            {detail.generationRecovery.reason}
+                        </p>{/if}
+                    {#if writable}<Button
+                            disabled={!!view.busy || !detail.generationRecovery.available}
+                            loading={view.busy === 'reconcile-generation'}
+                            onclick={() =>
+                                controller.act(
+                                    'reconcile-generation',
+                                    'Replacement generation reconciled. Fresh health verification is required; cancelled operations remain cancelled.'
+                                )}>Reconcile replacement generation</Button
+                        >{/if}
+                </div>{/if}
             {#if writable && (detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canRetryWithRebuild || detail.canCancel || detail.canCancelCreation || canOverride)}
                 <div class="recovery-actions">
                     {#if canOverride}<div>

static/agent/ec-agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.9.3"
+VERSION = "1.10.1"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -63,6 +63,7 @@ UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-ch
 UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
 UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
 AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
+HWMON_ROOT = Path("/sys/class/hwmon")
 ROOT_MARKER = ".electric-cherry.json"
 
 
@@ -1308,6 +1309,59 @@ class Agent:
                 return None
             return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
 
+    def temperatures(self):
+        """Best-effort, read-only samples from known millidegree-Celsius drivers."""
+        samples = []
+        try:
+            chips = sorted(HWMON_ROOT.glob("hwmon*"))[:64]
+        except OSError:
+            return samples
+        for chip in chips:
+            try:
+                device = (chip / "name").read_text().strip()
+                if device not in ("coretemp", "k10temp", "nvme") and not device.startswith("pch_"):
+                    continue
+                # hwmon numbers can change on reboot. The physical sysfs path
+                # and channel identify separate CPU sockets and NVMe devices.
+                identity = re.sub(r"/(?:hwmon/)?hwmon\d+$", "", str(chip.resolve()))
+                inputs = sorted(chip.glob("temp[0-9]*_input"))[:128]
+            except (OSError, ValueError):
+                continue
+            for source in inputs:
+                try:
+                    channel = source.name.removesuffix("_input")
+                    label_path = chip / (channel + "_label")
+                    label = label_path.read_text().strip() if label_path.exists() else channel
+                    if not label or len(label) > 120:
+                        continue
+                    fault = chip / (channel + "_fault")
+                    enabled = chip / (channel + "_enable")
+                    if (fault.exists() and int(fault.read_text()) != 0) or (enabled.exists() and int(enabled.read_text()) == 0):
+                        continue
+                    if device == "coretemp":
+                        kind = "cpu-package" if label.startswith("Package id ") else "cpu-core" if label.startswith("Core ") else None
+                    elif device == "k10temp":
+                        # Tctl can contain a fan-control offset. Never present
+                        # it as the physical die temperature or a fleet metric.
+                        kind = "cpu-die" if label == "Tdie" or re.fullmatch(r"Tccd\d+", label) else None
+                    else:
+                        kind = "nvme" if device == "nvme" else "chipset"
+                    if kind is None:
+                        continue
+                    celsius = int(source.read_text()) / 1000
+                    if not -100 <= celsius <= 200:
+                        continue
+                    sensor_id = hashlib.sha256((identity + "/" + channel).encode()).hexdigest()[:24]
+                    samples.append({"id": sensor_id, "device": device, "label": label,
+                                    "kind": kind, "celsius": round(celsius, 1)})
+                    if len(samples) >= 128:
+                        return samples
+                except (OSError, ValueError):
+                    # Read permission alone does not imply data is available
+                    # (e.g. ENODATA), and a device can disappear during a read.
+                    continue
+        return samples
+
     def instance_resources(self, observations=None):
         """One bounded Docker stats batch; unavailable samples are never zero load."""
         ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true",
@@ -1392,6 +1446,7 @@ class Agent:
 
     def inventory(self):
         cpu_usage = self.cpu_usage_percent()
+        temperatures = self.temperatures()
         usage = shutil.disk_usage(self.root)
         memory = {}
         try:
@@ -1516,9 +1571,9 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["host.temperatures", "library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         plutonium_cache = self.plutonium_cache_inventory()
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "temperatures": temperatures, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
         health = self.inspect(instance_id, generation)
@@ -1630,7 +1685,7 @@ class Agent:
         players = status_players(text) if command == "status" else None
         return {"ok": True, "text": text, "players": players}
 
-    def validate_spec(self, payload):
+    def validate_spec(self, payload, *, prepared_files=None):
         game = payload.get("game")
         if game not in ("t5", "t6"):
             raise AgentError("Only T5 and T6 are supported")
@@ -1673,16 +1728,22 @@ class Agent:
         files = self.initial_files(payload)
         if not isinstance(files, list) or len(files) > 10000:
             raise Rejected("Invalid initial file list")
+        if prepared_files is not None:
+            if set(prepared_files) != {item["path"] for item in files} or any(not isinstance(content, bytes) for content in prepared_files.values()):
+                raise Rejected("Prepared files do not match the replacement specification")
+        verified_files = {}
         total, paths = 0, set()
         for item in files:
             scoped(self.root / "agent" / "validation", item["path"])
             if item["path"] in paths:
                 raise Rejected("Duplicate initial file path")
             paths.add(item["path"])
-            total += len(self.decode_file(item))
+            content = self.decode_file(item, content=None if prepared_files is None else prepared_files[item["path"]])
+            total += len(content)
             if total > 64 * 1024 * 1024:
                 raise Rejected("Initial files exceed the 64 MiB operation limit")
-        return game, library, image, image_id, port, bind, env
+            verified_files[item["path"]] = content
+        return game, library, image, image_id, port, bind, env, verified_files
 
     @staticmethod
     def plutonium_manifest(data):
@@ -1901,7 +1962,7 @@ class Agent:
             managed_write(app, marker, raw, uid, gid)
             return True
 
-    def create(self, instance_id, generation, payload, *, before=None):
+    def create(self, instance_id, generation, payload, *, before=None, prepared_files=None):
         self.operation_phase = "checking-instance"
         self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
@@ -1916,7 +1977,7 @@ class Agent:
                 raise AgentError("Instance already exists with a different specification")
             return existing
         self.operation_phase = "validating-create-spec"
-        game, library, image, image_id, port, bind, env = self.validate_spec(payload)
+        game, library, image, image_id, port, bind, env, prepared_files = self.validate_spec(payload, prepared_files=prepared_files)
         app = directory / "app"
         self.operation_phase = "preparing-create-directory"
         app.mkdir(parents=True, exist_ok=True)
@@ -1924,7 +1985,7 @@ class Agent:
         self.operation_phase = "seeding-runtime"
         self.seed_plutonium(app, image)
         self.operation_phase = "writing-create-files"
-        self.write_initial_files(app, self.initial_files(payload))
+        self.write_initial_files(app, prepared_files)
         self.operation_phase = "persisting-create-metadata"
         atomic_json(directory / "spec.json", payload)
         env_path = directory / "environment"
@@ -1977,30 +2038,32 @@ class Agent:
         content = f'set tws_bank_token "{bank["token"]}"\n'.encode()
         return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
 
-    def write_initial_files(self, root, files):
-        if not isinstance(files, list) or len(files) > 10000:
-            raise AgentError("Invalid file list")
-        for item in files:
-            content = self.decode_file(item)
-            managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
+    def write_initial_files(self, root, prepared_files):
+        for path, content in prepared_files.items():
+            managed_write(root, path, content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
 
-    def decode_file(self, item):
+    def decode_file(self, item, *, content=None):
         if "contentSha256" in item:
             sha = item["contentSha256"]
             if not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha):
                 raise AgentError("Invalid file content hash")
-            url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha
-            with self.artifact_open("GET", url) as response:
-                content = response.read(MAX_FILE + 1)
+            if content is None:
+                url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha
+                with self.artifact_open("GET", url) as response:
+                    content = response.read(MAX_FILE + 1)
             if len(content) > MAX_FILE:
                 raise AgentError("File exceeds staging size limit")
             if digest(content) != sha or item.get("sha256", sha) != sha:
                 raise AgentError("Staged file checksum mismatch")
             return content
         try:
-            content = base64.b64decode(item["contentBase64"], validate=True)
+            decoded = base64.b64decode(item["contentBase64"], validate=True)
         except (ValueError, KeyError, TypeError):
             raise AgentError("Invalid encoded file content") from None
+        if content is None:
+            content = decoded
+        elif content != decoded:
+            raise Rejected("Prepared files do not match the replacement specification")
         if len(content) > MAX_FILE:
             raise AgentError("File exceeds staging size limit")
         if item.get("sha256") and item["sha256"] != digest(content):
@@ -2135,13 +2198,16 @@ class Agent:
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
                     raise AgentError("Rebuild must use a newer immutable generation")
                 spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
-                self.validate_spec(spec)
+                # Keep the complete verified set on this call's stack only. Never
+                # mutate the immutable spec or cache bytes on the shared agent:
+                # held, failed and later operations must prepare their own files.
+                *_, prepared_files = self.validate_spec(spec)
             if not override_safety:
                 self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
             if "autoscaling" in payload:
                 self.authorize_autoscaling_removal(instance_id, generation, payload)
             if kind == "instance.rebuild" and payload.get("stageId"):
-                before = self.rebuild_before(instance_id, generation, payload["stageId"], spec)
+                before = self.rebuild_before(instance_id, generation, payload["stageId"], prepared_files)
                 if not override_safety:
                     self.safe_to_disrupt(instance_id, generation)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})})
@@ -2153,7 +2219,7 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 self.settle_bank(instance_id, generation)
                 if kind == "instance.rebuild":
-                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None)
+                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None, prepared_files=prepared_files)
                     self.operation_phase = "removing-old-instance"
                     old = self.instance_dir(instance_id, generation)
                     if old.exists():
@@ -2254,21 +2320,21 @@ class Agent:
         atomic_json(marker, manifest)
         return {"stageId": stage_id, "files": len(planned), "state": "staged"}
 
-    def rebuild_before(self, instance_id, generation, stage_id, spec):
+    def rebuild_before(self, instance_id, generation, stage_id, prepared_files):
         """Validate requested replacement bytes and snapshot the old generation before teardown."""
         directory = self.instance_dir(instance_id, generation)
         marker = directory / "stages" / identifier(stage_id) / "manifest.json"
         saved = json.loads(marker.read_text())
         if saved["state"] != "staged":
             raise Rejected("Rebuild requires an unactivated file stage")
-        replacement = {item["path"]: item for item in self.initial_files(spec)}
         before = []
         for item in saved["files"]:
-            target = replacement.get(item["path"])
+            require_public_file(item["path"])
+            target = prepared_files.get(item["path"])
             if item["delete"]:
                 if target is not None:
                     raise Rejected("Rebuild includes a file requested for removal")
-            elif target is None or digest(self.decode_file(target)) != item["sha256"]:
+            elif target is None or digest(target) != item["sha256"]:
                 raise Rejected("Rebuild does not match staged file content")
             content = managed_read(directory / "app", item["path"])
             actual_sha = None if content is None else digest(content)

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.9.3",
+  "version": "1.10.1",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "a0baaab0f0a60e848300e63f31eeefff6293d8b749b5ca9397c36e8e75ff4a93",
+  "sha256": "61481547f99921a5ea2159f863f263d1af7b60114a9d73b00300d098f5abd5af",
   "updaterProtocolVersion": 2,
   "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb",
   "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231"

static/agent/manifest.json.sig

@@ -1,8 +1,8 @@
 -----BEGIN PGP SIGNATURE-----
 
-iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCas2mehsUgAAAAAAEAA5t
-YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfpvyAD/THg0KsH9kDSCF/I2vdOV
-h2BBe9r1JWk1F0mFy2njfvAA/R9mMAMdJRkFIxMNgkIl17nQrUq4TpXQsKSSujXZ
-u5oK
-=GspF
+iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCas+vvhsUgAAAAAAEAA5t
+YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfqXzAD8D4zQMptxYUl1befVmlc4
+pNq9N9P2o1eIx1jWDIQtmK0A/RNG50PZ97hjVkzvM565YKM9lQXiytj22vgpiGG8
+B/QH
+=NY2I
 -----END PGP SIGNATURE-----

tests/client/temperature-chart.test.ts

@@ -0,0 +1,110 @@
+import { afterEach, expect, it } from 'vitest';
+import { flushSync, mount, unmount } from 'svelte';
+import TemperatureChart from '../../src/lib/ui/components/capacity/TemperatureChart.svelte';
+import type { CapacityHistory } from '../../src/lib/capacity.js';
+
+let component: ReturnType<typeof mount> | undefined;
+afterEach(async () => {
+    if (component) await unmount(component);
+    component = undefined;
+    document.body.replaceChildren();
+});
+function history(): CapacityHistory {
+    return {
+        range: '1h',
+        startAt: 1000,
+        endAt: 4000,
+        bucketMs: 1000,
+        points: [1000, 2000, 3000].map((at) => ({
+            at,
+            cpuCoresAvailable: null,
+            memoryBytesAvailable: null,
+            reportingHosts: 0,
+            expectedHosts: 1
+        })),
+        temperatures: [
+            {
+                id: 'host:cpu',
+                hostId: 'host',
+                label: 'CPU temperature',
+                kind: 'cpu',
+                points: [65, null, 67].map((celsius, i) => ({ at: 1000 + i * 1000, celsius }))
+            },
+            {
+                id: 'host:drive',
+                hostId: 'host',
+                label: 'NVMe drive 1',
+                kind: 'nvme',
+                points: [49, 50.9, 52].map((celsius, i) => ({ at: 1000 + i * 1000, celsius }))
+            }
+        ]
+    };
+}
+function show(report = history()) {
+    component = mount(TemperatureChart, {
+        target: document.body,
+        props: { history: report, scope: 'ec-2' }
+    });
+    flushSync();
+}
+
+it('defaults to CPU and toggles device lines and readouts together', () => {
+    show();
+    const buttons = document.querySelectorAll<HTMLButtonElement>('.legend button');
+    expect(buttons[0].getAttribute('aria-pressed')).toBe('true');
+    expect(buttons[1].getAttribute('aria-pressed')).toBe('false');
+    expect(document.querySelectorAll('[data-series]')).toHaveLength(1);
+    // A missing middle bucket splits the CPU line into two isolated segments.
+    expect(document.querySelectorAll('[data-series="host:cpu"] path')).toHaveLength(2);
+    buttons[1].click();
+    flushSync();
+    expect(document.querySelectorAll('[data-series]')).toHaveLength(2);
+    expect(document.querySelector('.readout')?.textContent).toContain('52 °C');
+    buttons[0].click();
+    buttons[1].click();
+    flushSync();
+    expect(document.querySelectorAll('[data-series]')).toHaveLength(0);
+    expect(document.body.textContent).toContain('Select a sensor to show history');
+    expect(document.querySelector('.readout')).toBeNull();
+});
+
+it('supports keyboard and pointer inspection without turning missing readings into zero', () => {
+    show();
+    const plot = document.querySelector<HTMLElement>('[role="slider"]')!;
+    expect(plot.getAttribute('aria-valuenow')).toBe('2');
+    plot.dispatchEvent(new KeyboardEvent('keydown', { key: 'Home', bubbles: true }));
+    flushSync();
+    expect(plot.getAttribute('aria-valuetext')).toContain('65 °C');
+    plot.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowRight', bubbles: true }));
+    flushSync();
+    expect(plot.getAttribute('aria-valuetext')).toContain('Unavailable');
+    expect(document.querySelector('.readout')?.textContent).not.toContain('0 °C');
+    Object.defineProperty(plot, 'getBoundingClientRect', {
+        value: () => ({ left: 0, width: 1000 })
+    });
+    plot.dispatchEvent(
+        new PointerEvent('pointerdown', { clientX: 8, pointerType: 'touch', bubbles: true })
+    );
+    flushSync();
+    expect(plot.getAttribute('aria-valuenow')).toBe('0');
+    expect(document.querySelector('.readout')?.textContent).toContain('65 °C');
+});
+
+it('shows unsupported hosts as unavailable', () => {
+    const report = history();
+    report.temperatures[0].points = report.temperatures[0].points.map((point) => ({
+        ...point,
+        celsius: null
+    }));
+    show(report);
+    expect(document.body.textContent).toContain('No temperature readings in this period');
+    expect(document.body.textContent).toContain('Unavailable');
+});
+
+it('plots a genuine zero-degree reading', () => {
+    const report = history();
+    report.temperatures[0].points[2].celsius = 0;
+    show(report);
+    expect(document.querySelector('.readout')?.textContent).toContain('0 °C');
+    expect(document.querySelector('.empty-chart')).toBeNull();
+});

tests/unit/auth-access.test.ts

@@ -90,6 +90,7 @@ describe('management route permissions', () => {
         ['/api/rebalancing', 'POST'],
         ['/api/provisioning/[id]/cancel', 'POST'],
         ['/api/provisioning/[id]/resume', 'POST'],
+        ['/api/provisioning/[id]/reconcile-generation', 'POST'],
         ['/api/deployment-collections', 'POST'],
         ['/api/deployment-collections/preview', 'POST'],
         ['/api/deployment-collections/[id]/cancel', 'POST'],

tests/unit/capacity-history.test.ts

@@ -34,6 +34,101 @@ async function sample(id: string, at: number, cpu = 25, memory = 4 * GiB, cores
 }
 
 describe('capacity history', () => {
+    it('keeps separate CPU series, device drill-down, real zero and missing temperature gaps', async () => {
+        await host('one');
+        await host('two');
+        await host('unsupported');
+        await host('removed');
+        const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
+        const cpu = (celsius: number) => ({
+            id: 'package',
+            kind: 'cpu-package' as const,
+            device: 'coretemp',
+            label: 'Package id 0',
+            celsius
+        });
+        const drive = {
+            id: 'drive',
+            kind: 'nvme' as const,
+            device: 'nvme',
+            label: 'Composite',
+            celsius: 50.9
+        };
+        recordTemperatures(
+            'one',
+            { temperatures: [cpu(65), drive] },
+            new Date(epoch - 55_000).toISOString()
+        );
+        recordTemperatures(
+            'one',
+            { temperatures: [cpu(75)] },
+            new Date(epoch - 30_000).toISOString()
+        );
+        recordTemperatures(
+            'one',
+            { temperatures: [cpu(0)] },
+            new Date(epoch - 180_000).toISOString()
+        );
+        recordTemperatures(
+            'two',
+            { temperatures: [cpu(60)] },
+            new Date(epoch - 40_000).toISOString()
+        );
+        recordTemperatures(
+            'removed',
+            { temperatures: [cpu(99)] },
+            new Date(epoch - 40_000).toISOString()
+        );
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db()
+            .prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?')
+            .run(new Date(epoch).toISOString(), 'removed');
+        const { readCapacityHistory } =
+            await import('../../src/lib/server/hosts/capacity-history.js');
+        const fleet = readCapacityHistory(query, epoch).temperatures;
+        expect(fleet.map((s) => s.label)).toEqual(['one', 'two', 'unsupported']);
+        expect(fleet[0].points.slice(-3).map((p) => p.celsius)).toEqual([0, null, 75]);
+        expect(fleet[1].points.at(-1)?.celsius).toBe(60);
+        expect(fleet[2].points.every((p) => p.celsius === null)).toBe(true);
+        const selected = readCapacityHistory(
+            { ...query, hostId: 'one', game: 't5' },
+            epoch
+        ).temperatures;
+        expect(selected.map((s) => s.kind)).toEqual(['cpu', 'nvme']);
+        expect(selected[1].points.at(-1)?.celsius).toBe(50.9);
+        expect(
+            readCapacityHistory({ ...query, hostId: 'one', game: 't6' }, epoch).temperatures
+        ).toEqual(selected);
+        expect(readCapacityHistory({ ...query, hostId: 'missing' }, epoch).temperatures).toEqual(
+            []
+        );
+    });
+
+    it('preserves temperature peaks for long ranges and includes a host enrolled during the current minute', async () => {
+        await host('new', epoch - 20_000);
+        const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
+        recordTemperatures(
+            'new',
+            {
+                temperatures: [
+                    {
+                        id: 'pkg',
+                        device: 'coretemp',
+                        kind: 'cpu-package',
+                        label: 'Package id 0',
+                        celsius: 65
+                    }
+                ]
+            },
+            new Date(epoch - 10_000).toISOString()
+        );
+        const { readCapacityHistory } =
+            await import('../../src/lib/server/hosts/capacity-history.js');
+        expect(readCapacityHistory(query, epoch).temperatures[0].points.at(-1)?.celsius).toBe(65);
+        const long = readCapacityHistory({ ...query, range: '7d' }, epoch).temperatures[0];
+        expect(long.points).toHaveLength(168);
+        expect(long.points.at(-1)?.celsius).toBe(65);
+    });
     it('averages each host before summing fleet headroom and exposes missing observations as gaps', async () => {
         await host('one');
         await host('two');

tests/unit/database-migrations.test.ts

@@ -291,3 +291,18 @@ it('adds provisioning report invalidation at version 14 without invalidating for
     upgraded.exec('DELETE FROM server_deployments');
     expect(revision()).toBeGreaterThan(completed);
 });
+
+it('adds the generation evidence boundary to a version 23 database without changing instance identity', async () => {
+    const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+    const handle = db();
+    handle.exec(
+        "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s','host',2,'lib','image',1,'2026-10-14T00:00:00Z')"
+    );
+    handle.exec('ALTER TABLE native_instances DROP COLUMN generation_reconciled_at');
+    handle.pragma('user_version = 23');
+    closeDb();
+    expect(
+        db().prepare('SELECT id,generation,generation_reconciled_at FROM native_instances').get()
+    ).toEqual({ id: 'instance', generation: 2, generation_reconciled_at: null });
+    expect(db().pragma('foreign_key_check')).toEqual([]);
+});

tests/unit/host-temperatures.test.ts

@@ -0,0 +1,151 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import {
+    cpuTemperature,
+    historyTemperatures,
+    validateTemperatures,
+    type HostTemperatureSensor
+} from '../../src/lib/host-temperatures.js';
+
+const sensor = (changes: Partial<HostTemperatureSensor> = {}): HostTemperatureSensor => ({
+    id: 'package',
+    device: 'coretemp',
+    label: 'Package id 0',
+    kind: 'cpu-package',
+    celsius: 65,
+    ...changes
+});
+
+describe('temperature readings', () => {
+    it('keeps valid sensors while dropping malformed, duplicate and out-of-range entries', () => {
+        const good = sensor();
+        const readings = [
+            good,
+            good,
+            null,
+            'bad',
+            ...[NaN, Infinity, -Infinity, null, '65', true, -101, 201].map((celsius) => ({
+                ...good,
+                id: 'invalid',
+                celsius
+            })),
+            { ...good, id: 'x'.repeat(101) },
+            { ...good, label: '' },
+            { ...good, kind: 'cpu-control' },
+            sensor({ id: 'zero', celsius: 0 })
+        ];
+        expect(validateTemperatures(readings)).toEqual([good, sensor({ id: 'zero', celsius: 0 })]);
+        expect(validateTemperatures({})).toEqual([]);
+        expect(validateTemperatures(undefined)).toEqual([]);
+    });
+    it('chooses the hottest package and uses die/core readings only as fallbacks', () => {
+        const packages = [sensor(), sensor({ id: 'socket2', label: 'Package id 1', celsius: 72 })];
+        const core = sensor({ id: 'core', label: 'Core 0', kind: 'cpu-core', celsius: 80 });
+        const die = sensor({ id: 'die', label: 'Tdie', kind: 'cpu-die', celsius: 64 });
+        expect(cpuTemperature([...packages, core])?.celsius).toBe(72);
+        expect(cpuTemperature([die, core])?.celsius).toBe(64);
+        expect(cpuTemperature([core])?.celsius).toBe(80);
+        expect(cpuTemperature()).toBeNull();
+        const drive = sensor({
+            id: 'drive',
+            kind: 'nvme',
+            device: 'nvme',
+            label: 'Composite',
+            celsius: 50.9
+        });
+        expect(
+            historyTemperatures([
+                ...packages,
+                core,
+                drive,
+                { ...drive, id: 'internal', label: 'Sensor 1' }
+            ])
+        ).toEqual([{ ...packages[1], id: 'cpu', label: 'CPU temperature' }, drive]);
+    });
+});
+
+describe('temperature persistence', () => {
+    let fixture: Fixture;
+    beforeEach(async () => {
+        vi.resetModules();
+        fixture = await newFixture([]);
+    });
+    afterEach(async () => {
+        vi.restoreAllMocks();
+        await fixture.cleanup();
+    });
+
+    it('accepts older agents and faulty temperature payloads without losing host metrics', async () => {
+        const { createHost, heartbeat, validateInventory } =
+            await import('../../src/lib/server/hosts/index.js');
+        const host = createHost({ name: 'ec-2' });
+        expect(validateInventory({ cpuUsagePercent: 25 }).temperatures).toBeUndefined();
+        expect(
+            heartbeat(host.id, {
+                cpuUsagePercent: 25,
+                temperatures: [sensor(), { celsius: 'bad' }]
+            }).inventory
+        ).toMatchObject({ cpuUsagePercent: 25, temperatures: [sensor()] });
+        expect(
+            heartbeat(host.id, { cpuUsagePercent: 30, temperatures: [] }).inventory.temperatures
+        ).toEqual([]);
+        expect(
+            heartbeat(host.id, { cpuUsagePercent: 35, temperatures: 'bad' }).inventory.temperatures
+        ).toEqual([]);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect(
+            db()
+                .prepare('SELECT celsius FROM ec_host_temperature_samples WHERE host_id=?')
+                .all(host.id)
+        ).toEqual([{ celsius: 65 }]);
+    });
+
+    it('retains one peak per minute and applies host history retention, including offline hosts', async () => {
+        const { createHost } = await import('../../src/lib/server/hosts/index.js');
+        const { recordTemperatures } = await import('../../src/lib/server/hosts/temperatures.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { fleetSettings, saveFleetSettings } =
+            await import('../../src/lib/server/settings.js');
+        const { pruneSampleHistory } = await import('../../src/lib/server/retention.js');
+        const host = createHost({ name: 'ec-2' });
+        const minute = Math.floor((Date.now() - 3 * 86400_000) / 60_000) * 60_000;
+        for (const [i, celsius] of [65, 78, 67].entries())
+            recordTemperatures(
+                host.id,
+                { temperatures: [sensor({ celsius })] },
+                new Date(minute + i * 15_000).toISOString()
+            );
+        expect(db().prepare('SELECT celsius FROM ec_host_temperature_samples').all()).toEqual([
+            { celsius: 78 }
+        ]);
+        saveFleetSettings({ ...fleetSettings(), hostResourceKeepDays: 0 });
+        pruneSampleHistory();
+        expect(db().prepare('SELECT COUNT(*) AS n FROM ec_host_temperature_samples').get()).toEqual(
+            { n: 1 }
+        );
+        recordTemperatures(host.id, { temperatures: [sensor()] }, new Date().toISOString());
+        saveFleetSettings({ ...fleetSettings(), hostResourceKeepDays: 1 });
+        pruneSampleHistory();
+        expect(db().prepare('SELECT celsius FROM ec_host_temperature_samples').all()).toEqual([
+            { celsius: 65 }
+        ]);
+    });
+
+    it('upgrades an existing database without rewriting its resource samples', async () => {
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const { createHost } = await import('../../src/lib/server/hosts/index.js');
+        const host = createHost({ name: 'old' });
+        db()
+            .prepare(
+                'INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_usage_percent) VALUES(?,?,?)'
+            )
+            .run(host.id, '2026-10-13T12:00:00Z', 25);
+        db().exec('DROP TABLE ec_host_temperature_samples; PRAGMA user_version=22;');
+        closeDb();
+        expect(db().prepare('SELECT * FROM ec_host_temperature_samples').all()).toEqual([]);
+        expect(
+            db().prepare('SELECT cpu_usage_percent FROM ec_host_resource_samples').all()
+        ).toEqual([{ cpu_usage_percent: 25 }]);
+        expect(db().pragma('foreign_key_check')).toEqual([]);
+    });
+});

tests/unit/native-provisioning.test.ts

@@ -465,7 +465,16 @@ describe('provisioning health handoff', () => {
                 .run(old, old, old, job.name);
             vi.useFakeTimers({ toFake: ['Date'] });
             vi.setSystemTime(Date.now() + 1000);
-            const observedAt = new Date().toISOString();
+            if (action === 'rebuild') {
+                // A new generation now establishes its own durable evidence boundary.
+                // Observe game health strictly after that transition.
+                const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+                vi.mocked(nativeRequest).mockImplementationOnce(async () => {
+                    vi.setSystemTime(Date.now() + 1);
+                    return { healthy: true, state: 'running', exists: true } as never;
+                });
+            }
+            const observedAt = new Date(Date.now() + (action === 'rebuild' ? 1 : 0)).toISOString();
             const roster = [
                 { num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }
             ];
@@ -4288,3 +4297,276 @@ describe('game key pool launch lifecycle', () => {
         expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
     });
 });
+
+describe('completed rebuild generation receipts (#155)', () => {
+    async function rebuild() {
+        const job = await queue();
+        await advance();
+        await acknowledge(job.deploymentId);
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const requested = requestLifecycle(job.name, 'rebuild');
+        if (!requested.ok) throw new Error(requested.error);
+        await advance();
+        return {
+            job,
+            id: requested.deploymentId,
+            operation: (await operationFor(requested.deploymentId))!
+        };
+    }
+
+    async function inventoryGeneration(
+        instanceId: string,
+        generation: number,
+        observedAt = new Date().toISOString(),
+        targetHost = hostId
+    ) {
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        heartbeat(targetHost, {
+            ...inventory(),
+            instanceStates: [
+                { instanceId, generation, state: 'running', healthy: true, observedAt }
+            ]
+        });
+    }
+
+    it('repairs an older job binding from the actual command before verification, including duplicate receipts and restart', async () => {
+        const { job, id } = await rebuild();
+        await acknowledge(id);
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const next = requestLifecycle(job.name, 'rebuild');
+        if (!next.ok) throw new Error(next.error);
+        await advance();
+        const operation = (await operationFor(next.deploymentId))!;
+        expect(operation.generation).toBe(2);
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        // Historical job snapshots could stay on generation 1 while the submitted
+        // command correctly targeted 2 -> 3. The old completion path kept 2.
+        db()
+            .prepare('UPDATE native_provisioning SET generation=1 WHERE deployment_id=?')
+            .run(next.deploymentId);
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: { instanceId: operation.instanceId, generation: 3, healthy: true }
+        });
+        await inventoryGeneration(operation.instanceId!, 3);
+        closeDb();
+        const { nativeRequest, nativeInstance } =
+            await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        remote.state = 'starting';
+        await advance();
+        expect(nativeInstance(job.name)?.generation).toBe(3);
+        expect(nativeRequest).toHaveBeenCalledWith(
+            expect.objectContaining({ id: operation.instanceId, generation: 3 }),
+            'instance.inspect'
+        );
+        expect(
+            db()
+                .prepare(
+                    'SELECT generation,applied_at FROM native_provisioning WHERE deployment_id=?'
+                )
+                .get(next.deploymentId)
+        ).toEqual({ generation: 2, applied_at: expect.any(String) });
+        const { reconcileNativeProvisioningReceipt } =
+            await import('../../src/lib/server/provisioning/native.js');
+        const before = db()
+            .prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
+            .get(operation.instanceId);
+        reconcileNativeProvisioningReceipt(next.deploymentId);
+        closeDb();
+        reconcileNativeProvisioningReceipt(next.deploymentId);
+        expect(nativeInstance(job.name)?.generation).toBe(3);
+        expect(
+            db()
+                .prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
+                .get(operation.instanceId)
+        ).toEqual(before);
+        expect((await detail(next.deploymentId)).state).toBe('awaiting-health');
+        for (const observation of await observations(job.name))
+            expect(observation.health).not.toBe('online');
+        const { latestGameSubmission, latestGameSubmissions } =
+            await import('../../src/lib/server/fleet/lifecycle-evidence.js');
+        expect(latestGameSubmission(job.name)).toBe(
+            (before as { generation_reconciled_at: string }).generation_reconciled_at
+        );
+        expect(latestGameSubmissions().get(job.name)).toBe(latestGameSubmission(job.name));
+        remote.state = 'running';
+        await advance();
+        expect((await detail(next.deploymentId)).state).toBe('done');
+        const restart = requestLifecycle(job.name, 'restart');
+        if (!restart.ok) throw new Error(restart.error);
+        await advance();
+        expect(await operationFor(restart.deploymentId)).toMatchObject({ generation: 3 });
+    });
+
+    it.each(['before', 'after'] as const)(
+        'reconciles delayed completion reported %s cancellation across restart',
+        async (when) => {
+            const { job, id, operation } = await rebuild();
+            const { reportHostOperation, pendingHostOperations } =
+                await import('../../src/lib/server/hosts/index.js');
+            const { cancelProvisioningJob } =
+                await import('../../src/lib/server/provisioning/cancellation.js');
+            const { closeDb } = await import('../../src/lib/server/db/index.js');
+            pendingHostOperations(hostId);
+            if (when === 'before')
+                reportHostOperation(hostId, operation.id, { state: 'completed' });
+            expect(cancelProvisioningJob(id)).toBe(true);
+            closeDb();
+            if (when === 'after') reportHostOperation(hostId, operation.id, { state: 'completed' });
+            await advance();
+            await advance();
+            const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+            expect(nativeInstance(job.name)?.generation).toBe(2);
+            expect((await detail(id)).state).toBe('cancelled');
+        }
+    );
+
+    it('recovers an already-applied cancelled mismatch through the API and targets replacement logs, health and removal', async () => {
+        const { job, id, operation } = await rebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: { instanceId: operation.instanceId, generation: 2 }
+        });
+        const { cancelProvisioningJob } =
+            await import('../../src/lib/server/provisioning/cancellation.js');
+        expect(cancelProvisioningJob(id)).toBe(true);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db()
+            .prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?')
+            .run(operation.updatedAt, id);
+        await inventoryGeneration(operation.instanceId!, 2);
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        expect(getDeployment(id)?.generationRecovery).toMatchObject({
+            expectedGeneration: 1,
+            observedGeneration: 2,
+            available: true
+        });
+        const { POST } =
+            await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
+        expect(await POST({ params: { id } }).json()).toEqual({ ok: true, generation: 2 });
+        expect(POST({ params: { id } }).status).toBe(200);
+        expect((await detail(id)).state).toBe('cancelled');
+        expect(getDeployment(id)?.generationRecovery).toBeNull();
+        const backend = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(backend.nativeRequest).mockRestore();
+        const hosts = await import('../../src/lib/server/hosts/index.js');
+        vi.spyOn(hosts, 'waitHostOperationStatus').mockImplementation(async (id) =>
+            hosts.reportHostOperationStatus(hostId, id, {
+                state: 'completed',
+                result: { healthy: true, state: 'running' }
+            })
+        );
+        await backend.nativeRequest(backend.nativeInstance(job.name)!, 'logs.read', {
+            file: 'console.log'
+        });
+        expect(await backend.nativeHealth(job.name)).toMatchObject({ ok: true });
+        expect(
+            db()
+                .prepare(
+                    "SELECT generation FROM ec_host_operations WHERE kind IN ('logs.read','instance.inspect')"
+                )
+                .all()
+                .every((row) => (row as { generation: number }).generation === 2)
+        ).toBe(true);
+        const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
+        const removal = destroyServer(job.name);
+        if (!removal.ok || !removal.deploymentId) throw new Error('Removal not queued');
+        await advance();
+        expect(await operationFor(removal.deploymentId)).toMatchObject({
+            generation: 2,
+            kind: 'instance.remove'
+        });
+    });
+
+    it.each([
+        'host',
+        'instance',
+        'result-instance',
+        'result-generation',
+        'payload',
+        'stale-inventory',
+        'missing-inventory',
+        'inventory-instance'
+    ] as const)('rejects recovery evidence with %s mismatch', async (mismatch) => {
+        const { job, id, operation } = await rebuild();
+        const { reportHostOperation, createHost } =
+            await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: {
+                instanceId:
+                    mismatch === 'result-instance' ? 'another-instance' : operation.instanceId,
+                generation: mismatch === 'result-generation' ? 3 : 2
+            }
+        });
+        const { db } = await import('../../src/lib/server/db/index.js');
+        if (mismatch === 'host')
+            db()
+                .prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?')
+                .run(createHost({ name: 'Other host' }).id, operation.id);
+        if (mismatch === 'instance')
+            db()
+                .prepare('UPDATE ec_host_operations SET instance_id=? WHERE id=?')
+                .run('another-instance', operation.id);
+        if (mismatch === 'payload')
+            db()
+                .prepare(
+                    "UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?"
+                )
+                .run(operation.id);
+        if (mismatch !== 'missing-inventory')
+            await inventoryGeneration(
+                mismatch === 'inventory-instance' ? 'another-instance' : operation.instanceId!,
+                2,
+                mismatch === 'stale-inventory' ? '2000-01-01T00:00:00Z' : new Date().toISOString()
+            );
+        const { POST } =
+            await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
+        expect(POST({ params: { id } }).status).toBe(409);
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        expect(nativeInstance(job.name)?.generation).toBe(1);
+    });
+
+    it('pauses after an agent stale-generation rejection even when inventory is unavailable', async () => {
+        const { id, operation } = await rebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed' });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        vi.mocked(nativeRequest).mockRejectedValueOnce(
+            new Error('Instance generation has been superseded or removed')
+        );
+        await advance();
+        expect((await detail(id)).state).toBe('interrupted');
+        expect((await detail(id)).error).toContain('controller expects 2, host reports unknown');
+        await advance();
+        expect(nativeRequest).toHaveBeenCalledTimes(1);
+    });
+
+    it('stops verification retries on a confirmed generation mismatch and keeps expected/observed evidence', async () => {
+        const { id, operation } = await rebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed' });
+        await inventoryGeneration(operation.instanceId!, 3);
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        await advance();
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const before = getDeployment(id)!;
+        expect(before.deployment.state).toBe('interrupted');
+        expect(before.progress?.blocker).toMatchObject({
+            code: 'generation-recovery',
+            automatic: false,
+            message: expect.stringContaining(
+                'controller expects 1, completed rebuild requests 2, host reports 3'
+            )
+        });
+        await advance();
+        await advance();
+        expect(nativeRequest).not.toHaveBeenCalled();
+        expect(getDeployment(id)?.events).toEqual(before.events);
+    });
+});

tests/unit/provisioning-history-performance.test.ts

@@ -157,7 +157,7 @@ describe('provisioning reads with retained lifecycle history', () => {
         };
         expectIndexed(
             () => expect(latestGameSubmission('current')).toBe('2026-01-01'),
-            ['current', 'current']
+            ['current', 'current', 'current']
         );
         expectIndexed(() => expect(hasActiveProvisioning('current')).toBe(true), ['current', '']);
         expectIndexed(() => expect(housePartyTargets()).toEqual(['current']));

tests/unit/retention.test.ts

@@ -710,3 +710,35 @@ it.each(['cancelled', 'failed'])(
         ).toEqual({ history_expired_at: expect.any(String) });
     }
 );
+
+it('retains completed rebuild evidence when a stale job binding hid an unreconciled generation', async () => {
+    const a = await api();
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    const host = hosts.createHost({ name: 'Host' });
+    a.db
+        .prepare(
+            "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,2,'lib','image',1,?)"
+        )
+        .run(host.id, old);
+    a.db
+        .prepare(
+            "INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')"
+        )
+        .run(old, old);
+    const command = hosts.queueHostOperation(
+        host.id,
+        'instance.rebuild',
+        { nextGeneration: 3 },
+        { instanceId: 'instance', generation: 2 }
+    );
+    hosts.reportHostOperation(host.id, command.id, { state: 'completed' });
+    a.db
+        .prepare(
+            "INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,applied_at) VALUES('cancelled','instance',1,?,?)"
+        )
+        .run(command.id, old);
+    a.saveFleetSettings({ ...a.fleetSettings(), provisioningKeepDays: 1, historyKeepCount: 0 });
+    expect(a.pruneHistory('provisioning')).toBe(0);
+    a.db.prepare("UPDATE native_instances SET generation=3 WHERE id='instance'").run();
+    expect(a.pruneHistory('provisioning')).toBe(1);
+});

tests/unit/ui-capacity.test.ts

@@ -43,6 +43,7 @@ const host = (changes: Partial<CapacityHost> = {}): CapacityHost => ({
     ...changes
 });
 const history: CapacityHistory = {
+    temperatures: [],
     range: '24h',
     startAt: 1000,
     endAt: 5000,
@@ -112,6 +113,10 @@ it('keeps current estimates and host comparison before history with scoped histo
     expect(body).not.toContain('Manage hosts');
     expect(body).toContain('Updates every 30 seconds');
     expect(body).not.toContain('>Refresh<');
+    expect(body.indexOf('id="temperature-history-title"')).toBeGreaterThan(
+        body.indexOf('id="capacity-history-title"')
+    );
+    expect(body.match(/aria-label="History range"/g)).toHaveLength(2);
 });
 
 it('keeps fleet-wide queue scope explicit and preserves unavailable estimates', () => {

tests/unit/ui-host-overview.test.ts

@@ -9,6 +9,7 @@ import {
     stableHostOrder
 } from '../../src/lib/ui/host-overview.js';
 import HostList from '../../src/lib/ui/components/HostList.svelte';
+import HostTemperatures from '../../src/lib/ui/components/HostTemperatures.svelte';
 
 const now = Date.parse('2026-10-06T12:00:00Z');
 function host(id = 'ec-1', changes: Partial<Host> = {}): Host {
@@ -43,6 +44,40 @@ function host(id = 'ec-1', changes: Partial<Host> = {}): Host {
     };
 }
 
+it('shows CPU temperature with stale status and keeps sensor details expandable', () => {
+    const target = host('ec-2', {
+        connected: false,
+        inventory: {
+            cpuUsagePercent: 25,
+            temperatures: [
+                {
+                    id: 'cpu',
+                    kind: 'cpu-package',
+                    device: 'coretemp',
+                    label: 'Package id 0',
+                    celsius: 65
+                }
+            ]
+        }
+    });
+    const list = render(HostList, {
+        props: {
+            hosts: [{ ...target, serverCount: 0, playerCount: 0, unreportedInstances: 0 }],
+            now
+        }
+    }).body;
+    expect(list).toContain('CPU temp: 65 °C');
+    expect(list).toContain('Resource readings stale');
+    const detail = render(HostTemperatures, { props: { host: target } }).body;
+    expect(detail).toContain('(last reported)');
+    expect(detail).toContain('Temperature sensors');
+    expect(detail).toContain('CPU package 0');
+    expect(detail).not.toMatch(/<details[^>]*open/);
+    const missing = render(HostTemperatures, { props: { host: host() } }).body;
+    expect(missing).toContain('Unavailable');
+    expect(missing).not.toContain('0 °C');
+});
+
 it('does not show an old Ready state as available when the heartbeat becomes stale', () => {
     expect(hostListStatus(host(), now).label).toBe('Ready');
     expect(hostReadingsStale(host(), now + 90_001)).toBe(true);