Reuse verified artifacts during rebuilds (Refs #154)

agent/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Agent changelog
 
+## 2026-10-14
+
+### 1.10.1 — Retain verified rebuild artifacts (EC #154)
+
+- Retain the complete verified replacement file set before stopping the current
+  container, including inline files and private bank configuration, within the
+  existing 64 MiB operation limit. Staged-file comparison, replacement validation
+  and writing reuse those bytes without further artifact downloads.
+- Keep prepared bytes local to the operation. Failed and held attempts discard
+  their files; later operations prepare a fresh set. Immutable specifications,
+  journal requests and container identity labels remain unchanged.
+- Preserve checksum, path, per-file size, player-safety, bank-settlement and
+  generation checks. Add regressions for preflight failures, lost connectivity
+  after preflight, file-set limits and operation isolation.
+- Package release artifacts for offline signing before publication. No protocol
+  change or controller migration is required.
+
 ## 2026-10-13
 
 ### 1.10.0 — Host temperature telemetry

agent/ec_agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.10.0"
+VERSION = "1.10.1"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1685,7 +1685,7 @@ class Agent:
         players = status_players(text) if command == "status" else None
         return {"ok": True, "text": text, "players": players}
 
-    def validate_spec(self, payload):
+    def validate_spec(self, payload, *, prepared_files=None):
         game = payload.get("game")
         if game not in ("t5", "t6"):
             raise AgentError("Only T5 and T6 are supported")
@@ -1728,16 +1728,22 @@ class Agent:
         files = self.initial_files(payload)
         if not isinstance(files, list) or len(files) > 10000:
             raise Rejected("Invalid initial file list")
+        if prepared_files is not None:
+            if set(prepared_files) != {item["path"] for item in files} or any(not isinstance(content, bytes) for content in prepared_files.values()):
+                raise Rejected("Prepared files do not match the replacement specification")
+        verified_files = {}
         total, paths = 0, set()
         for item in files:
             scoped(self.root / "agent" / "validation", item["path"])
             if item["path"] in paths:
                 raise Rejected("Duplicate initial file path")
             paths.add(item["path"])
-            total += len(self.decode_file(item))
+            content = self.decode_file(item, content=None if prepared_files is None else prepared_files[item["path"]])
+            total += len(content)
             if total > 64 * 1024 * 1024:
                 raise Rejected("Initial files exceed the 64 MiB operation limit")
-        return game, library, image, image_id, port, bind, env
+            verified_files[item["path"]] = content
+        return game, library, image, image_id, port, bind, env, verified_files
 
     @staticmethod
     def plutonium_manifest(data):
@@ -1956,7 +1962,7 @@ class Agent:
             managed_write(app, marker, raw, uid, gid)
             return True
 
-    def create(self, instance_id, generation, payload, *, before=None):
+    def create(self, instance_id, generation, payload, *, before=None, prepared_files=None):
         self.operation_phase = "checking-instance"
         self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
@@ -1971,7 +1977,7 @@ class Agent:
                 raise AgentError("Instance already exists with a different specification")
             return existing
         self.operation_phase = "validating-create-spec"
-        game, library, image, image_id, port, bind, env = self.validate_spec(payload)
+        game, library, image, image_id, port, bind, env, prepared_files = self.validate_spec(payload, prepared_files=prepared_files)
         app = directory / "app"
         self.operation_phase = "preparing-create-directory"
         app.mkdir(parents=True, exist_ok=True)
@@ -1979,7 +1985,7 @@ class Agent:
         self.operation_phase = "seeding-runtime"
         self.seed_plutonium(app, image)
         self.operation_phase = "writing-create-files"
-        self.write_initial_files(app, self.initial_files(payload))
+        self.write_initial_files(app, prepared_files)
         self.operation_phase = "persisting-create-metadata"
         atomic_json(directory / "spec.json", payload)
         env_path = directory / "environment"
@@ -2032,18 +2038,16 @@ class Agent:
         content = f'set tws_bank_token "{bank["token"]}"\n'.encode()
         return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
 
-    def write_initial_files(self, root, files):
+    def write_initial_files(self, root, prepared_files):
-        if not isinstance(files, list) or len(files) > 10000:
+        for path, content in prepared_files.items():
-            raise AgentError("Invalid file list")
+            managed_write(root, path, content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
-        for item in files:
-            content = self.decode_file(item)
-            managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
 
-    def decode_file(self, item):
+    def decode_file(self, item, *, content=None):
         if "contentSha256" in item:
             sha = item["contentSha256"]
             if not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha):
                 raise AgentError("Invalid file content hash")
+            if content is None:
                 url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha
                 with self.artifact_open("GET", url) as response:
                     content = response.read(MAX_FILE + 1)
@@ -2053,9 +2057,13 @@ class Agent:
                 raise AgentError("Staged file checksum mismatch")
             return content
         try:
-            content = base64.b64decode(item["contentBase64"], validate=True)
+            decoded = base64.b64decode(item["contentBase64"], validate=True)
         except (ValueError, KeyError, TypeError):
             raise AgentError("Invalid encoded file content") from None
+        if content is None:
+            content = decoded
+        elif content != decoded:
+            raise Rejected("Prepared files do not match the replacement specification")
         if len(content) > MAX_FILE:
             raise AgentError("File exceeds staging size limit")
         if item.get("sha256") and item["sha256"] != digest(content):
@@ -2190,13 +2198,16 @@ class Agent:
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
                     raise AgentError("Rebuild must use a newer immutable generation")
                 spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
-                self.validate_spec(spec)
+                # Keep the complete verified set on this call's stack only. Never
+                # mutate the immutable spec or cache bytes on the shared agent:
+                # held, failed and later operations must prepare their own files.
+                *_, prepared_files = self.validate_spec(spec)
             if not override_safety:
                 self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
             if "autoscaling" in payload:
                 self.authorize_autoscaling_removal(instance_id, generation, payload)
             if kind == "instance.rebuild" and payload.get("stageId"):
-                before = self.rebuild_before(instance_id, generation, payload["stageId"], spec)
+                before = self.rebuild_before(instance_id, generation, payload["stageId"], prepared_files)
                 if not override_safety:
                     self.safe_to_disrupt(instance_id, generation)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})})
@@ -2208,7 +2219,7 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 self.settle_bank(instance_id, generation)
                 if kind == "instance.rebuild":
-                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None)
+                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None, prepared_files=prepared_files)
                     self.operation_phase = "removing-old-instance"
                     old = self.instance_dir(instance_id, generation)
                     if old.exists():
@@ -2309,21 +2320,21 @@ class Agent:
         atomic_json(marker, manifest)
         return {"stageId": stage_id, "files": len(planned), "state": "staged"}
 
-    def rebuild_before(self, instance_id, generation, stage_id, spec):
+    def rebuild_before(self, instance_id, generation, stage_id, prepared_files):
         """Validate requested replacement bytes and snapshot the old generation before teardown."""
         directory = self.instance_dir(instance_id, generation)
         marker = directory / "stages" / identifier(stage_id) / "manifest.json"
         saved = json.loads(marker.read_text())
         if saved["state"] != "staged":
             raise Rejected("Rebuild requires an unactivated file stage")
-        replacement = {item["path"]: item for item in self.initial_files(spec)}
         before = []
         for item in saved["files"]:
-            target = replacement.get(item["path"])
+            require_public_file(item["path"])
+            target = prepared_files.get(item["path"])
             if item["delete"]:
                 if target is not None:
                     raise Rejected("Rebuild includes a file requested for removal")
-            elif target is None or digest(self.decode_file(target)) != item["sha256"]:
+            elif target is None or digest(target) != item["sha256"]:
                 raise Rejected("Rebuild does not match staged file content")
             content = managed_read(directory / "app", item["path"])
             actual_sha = None if content is None else digest(content)

agent/tests/test_agent.py

@@ -1666,7 +1666,7 @@ class AgentTests(unittest.TestCase):
         with self.assertRaisesRegex(AgentError, "Private bank"):
             self.agent.activate(INSTANCE, 1, "private", override_safety=True)
         with self.assertRaisesRegex(AgentError, "Private bank"):
-            self.agent.rebuild_before(INSTANCE, 1, "private", {"files": []})
+            self.agent.rebuild_before(INSTANCE, 1, "private", {})
         with self.assertRaisesRegex(AgentError, "Private bank"):
             self.agent.restore_files(app, [{"path": value, "contentBase64": None}])
         with self.assertRaisesRegex(AgentError, "Private bank"):
@@ -1934,6 +1934,164 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(json.loads(result["result"])["before"][0]["contentBase64"], base64.b64encode(b"original").decode())
         self.assertEqual(self.docker.calls, calls)
 
+    def artifact_rebuild(self, staged=False):
+        if staged:
+            operation = self.staged_rebuild()
+        else:
+            spec = self.create()
+            self.agent.rcon = lambda *_: {"ok": True, "players": 0}
+            operation = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2}, "rebuild")
+        spec = operation["payload"]["spec"]
+        contents = {"configs/server.cfg": b"replacement", "scripts/replacement.gsc": b"verified artifact bytes"}
+        spec["files"] = [{"path": path, "contentSha256": module.digest(content), "sha256": module.digest(content)} for path, content in contents.items()]
+        spec["files"].append({"path": "configs/empty.cfg", "contentBase64": ""})
+        spec["bank"] = {"configPath": "configs/electric_cherry_bank.cfg", "token": "ec_bank_" + "a" * 32}
+        return operation, contents
+
+    def assert_rebuild_uses_preflight_artifacts(self, staged):
+        operation, contents = self.artifact_rebuild(staged)
+        original_request = json.dumps(operation, sort_keys=True)
+        available = {module.digest(content): content for content in contents.values()}
+
+        def fetch(method, url):
+            # Connectivity disappears as soon as the complete preflight set has
+            # been read, before staged comparison or destructive replacement.
+            if not available:
+                raise module.urllib.error.URLError(TimeoutError())
+            self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+            return Response(available.pop(url.rsplit("/", 1)[-1]))
+
+        with patch.object(self.agent, "artifact_open", side_effect=fetch) as downloads:
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(downloads.call_count, len(contents))
+        self.assertFalse(available)
+        self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
+        self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
+        app = self.agent.instance_dir(INSTANCE, 2) / "app"
+        for path, content in contents.items():
+            self.assertEqual((app / path).read_bytes(), content)
+        self.assertEqual((app / "configs/empty.cfg").read_bytes(), b"")
+        self.assertEqual((app / "configs/electric_cherry_bank.cfg").read_text(), 'set tws_bank_token "ec_bank_' + "a" * 32 + '"\n')
+        self.assertEqual(json.dumps(operation, sort_keys=True), original_request)
+        self.assertEqual(json.loads(result["request"]), operation)
+        self.assertEqual(self.agent.spec(INSTANCE, 2), operation["payload"]["spec"])
+        labels = self.docker.inspect(self.agent.container_name(INSTANCE, 2))["Config"]["Labels"]
+        self.assertEqual(labels[module.LABEL + ".operation"], operation["id"])
+        self.assertEqual(labels[module.LABEL + ".spec-sha256"], module.digest(json.dumps(operation["payload"]["spec"], sort_keys=True, separators=(",", ":")).encode()))
+        receipt = json.loads(result["result"])
+        self.assertEqual(receipt["before"], [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"original").decode()}] if staged else [])
+        for content in contents.values():
+            self.assertNotIn(base64.b64encode(content).decode(), json.dumps(result))
+
+        # A later operation with the same artifacts must download its own set.
+        later = self.operation("instance.rebuild", {"spec": operation["payload"]["spec"], "nextGeneration": 3}, "later-rebuild", generation=2)
+        with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
+            self.assertEqual(self.agent.execute(later)["state"], "failed")
+        downloads.assert_called_once()
+        self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
+        self.assertFalse(self.agent.inspect(INSTANCE, 3)["exists"])
+
+    def test_unstaged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self):
+        self.assert_rebuild_uses_preflight_artifacts(staged=False)
+
+    def test_staged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self):
+        self.assert_rebuild_uses_preflight_artifacts(staged=True)
+
+    def test_rebuild_preflight_download_failure_preserves_original_and_discards_partial_set(self):
+        operation, contents = self.artifact_rebuild()
+        with patch.object(self.agent, "artifact_open", side_effect=[Response(contents["configs/server.cfg"]), module.urllib.error.URLError(TimeoutError())]) as downloads:
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "failed", result)
+        self.assertEqual(downloads.call_count, 2)
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(self.agent.instance_dir(INSTANCE, 2).exists())
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+        operation["id"] = "retry-rebuild"
+        with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
+            self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        self.assertTrue(downloads.call_args.args[1].endswith(module.digest(contents["configs/server.cfg"])))
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+
+    def test_held_rebuild_discards_prepared_artifacts_before_retry(self):
+        operation, contents = self.artifact_rebuild(staged=True)
+        with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}):
+            self.assertEqual(self.agent.execute(operation)["state"], "accepted")
+        with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
+            self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        downloads.assert_called_once()
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
+
+    def test_rebuild_preflight_rejects_corrupt_and_oversized_artifacts(self):
+        operation, contents = self.artifact_rebuild()
+        for index, content in enumerate((b"corrupt", b"x" * (module.MAX_FILE + 1))):
+            with self.subTest(index=index):
+                operation["id"] = f"invalid-artifact-{index}"
+                with patch.object(self.agent, "artifact_open", return_value=Response(content)):
+                    result = self.agent.execute(operation)
+                self.assertEqual(result["state"], "failed", result)
+                self.assertIn("checksum mismatch" if index == 0 else "size limit", result["error"])
+                self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+
+    def test_rebuild_preflight_bounds_complete_file_set_at_64_mib(self):
+        operation, _ = self.artifact_rebuild()
+        spec = operation["payload"]["spec"]
+        content = b"x" * module.MAX_FILE
+        spec["files"] = [{"path": f"scripts/{index}.gsc", "contentSha256": module.digest(content)} for index in range(2)]
+        # The generated private bank file also counts toward the operation limit.
+        with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "failed", result)
+        self.assertIn("64 MiB operation limit", result["error"])
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+        del spec["bank"]
+        operation["id"] = "at-limit"
+        with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}):
+            self.assertEqual(self.agent.execute(operation)["state"], "accepted")
+
+    def test_replacement_validation_rejects_incomplete_or_mismatched_prepared_bytes_without_downloads(self):
+        operation, contents = self.artifact_rebuild()
+        spec = operation["payload"]["spec"]
+        del spec["bank"]
+        prepared = {**contents, "configs/empty.cfg": b""}
+        invalid_sets = [
+            {path: content for path, content in prepared.items() if path != "configs/server.cfg"},
+            {**prepared, "configs/extra.cfg": b"unexpected"},
+            {**prepared, "configs/server.cfg": b"wrong artifact"},
+            {**prepared, "configs/empty.cfg": b"wrong inline bytes"},
+            {**prepared, "configs/server.cfg": None},
+        ]
+        with patch.object(self.agent, "artifact_open", side_effect=AssertionError("Prepared validation must not download")) as downloads:
+            for index, files in enumerate(invalid_sets):
+                with self.subTest(index=index), self.assertRaises(AgentError):
+                    self.agent.validate_spec(spec, prepared_files=files)
+        downloads.assert_not_called()
+
+    def test_rebuild_preflight_preserves_file_path_and_duplicate_checks(self):
+        operation, _ = self.artifact_rebuild()
+        spec = operation["payload"]["spec"]
+        for index, paths in enumerate((["../escape.cfg"], ["configs/electric_cherry_bank.cfg"], ["configs/same.cfg"] * 2)):
+            with self.subTest(paths=paths):
+                operation["id"] = f"invalid-paths-{index}"
+                spec["files"] = [{"path": path, "contentBase64": ""} for path in paths]
+                self.assertEqual(self.agent.execute(operation)["state"], "failed")
+                self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+
+    def test_staged_rebuild_rejects_verified_artifacts_that_differ_from_the_stage(self):
+        operation, contents = self.artifact_rebuild(staged=True)
+        contents["configs/server.cfg"] = b"different verified replacement"
+        operation["payload"]["spec"]["files"][0].update(contentSha256=module.digest(contents["configs/server.cfg"]), sha256=module.digest(contents["configs/server.cfg"]))
+        with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "failed", result)
+        self.assertIn("does not match staged file content", result["error"])
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
+
     def test_rebuild_moves_generation_and_rejects_old_commands(self):
         payload = self.create()
         self.agent.rcon = lambda *_: {"ok": True, "players": 0}

docs/guides/native-hosts.md

@@ -654,6 +654,13 @@ restart and create complete only after post-operation health evidence; creation
 and rebuild also verify the selected map and running Plutonium revision. The
 base-game library and runtime revision are independent records.
 
+Agent 1.10.1 prepares and verifies the complete replacement file set before
+stopping the existing container, within the 64 MiB operation limit. Staged-file
+comparison, replacement validation and writing reuse those bytes, so losing
+artifact connectivity after preflight does not prevent replacement. A preflight
+download failure leaves the original container running. Prepared bytes last only
+for that operation attempt; held or failed work must prepare its files again.
+
 ## Rebalance instances
 
 Open **Fleet → Capacity → Rebalance** to spread existing instances across active

static/agent/ec-agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.10.0"
+VERSION = "1.10.1"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1685,7 +1685,7 @@ class Agent:
         players = status_players(text) if command == "status" else None
         return {"ok": True, "text": text, "players": players}
 
-    def validate_spec(self, payload):
+    def validate_spec(self, payload, *, prepared_files=None):
         game = payload.get("game")
         if game not in ("t5", "t6"):
             raise AgentError("Only T5 and T6 are supported")
@@ -1728,16 +1728,22 @@ class Agent:
         files = self.initial_files(payload)
         if not isinstance(files, list) or len(files) > 10000:
             raise Rejected("Invalid initial file list")
+        if prepared_files is not None:
+            if set(prepared_files) != {item["path"] for item in files} or any(not isinstance(content, bytes) for content in prepared_files.values()):
+                raise Rejected("Prepared files do not match the replacement specification")
+        verified_files = {}
         total, paths = 0, set()
         for item in files:
             scoped(self.root / "agent" / "validation", item["path"])
             if item["path"] in paths:
                 raise Rejected("Duplicate initial file path")
             paths.add(item["path"])
-            total += len(self.decode_file(item))
+            content = self.decode_file(item, content=None if prepared_files is None else prepared_files[item["path"]])
+            total += len(content)
             if total > 64 * 1024 * 1024:
                 raise Rejected("Initial files exceed the 64 MiB operation limit")
-        return game, library, image, image_id, port, bind, env
+            verified_files[item["path"]] = content
+        return game, library, image, image_id, port, bind, env, verified_files
 
     @staticmethod
     def plutonium_manifest(data):
@@ -1956,7 +1962,7 @@ class Agent:
             managed_write(app, marker, raw, uid, gid)
             return True
 
-    def create(self, instance_id, generation, payload, *, before=None):
+    def create(self, instance_id, generation, payload, *, before=None, prepared_files=None):
         self.operation_phase = "checking-instance"
         self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
@@ -1971,7 +1977,7 @@ class Agent:
                 raise AgentError("Instance already exists with a different specification")
             return existing
         self.operation_phase = "validating-create-spec"
-        game, library, image, image_id, port, bind, env = self.validate_spec(payload)
+        game, library, image, image_id, port, bind, env, prepared_files = self.validate_spec(payload, prepared_files=prepared_files)
         app = directory / "app"
         self.operation_phase = "preparing-create-directory"
         app.mkdir(parents=True, exist_ok=True)
@@ -1979,7 +1985,7 @@ class Agent:
         self.operation_phase = "seeding-runtime"
         self.seed_plutonium(app, image)
         self.operation_phase = "writing-create-files"
-        self.write_initial_files(app, self.initial_files(payload))
+        self.write_initial_files(app, prepared_files)
         self.operation_phase = "persisting-create-metadata"
         atomic_json(directory / "spec.json", payload)
         env_path = directory / "environment"
@@ -2032,18 +2038,16 @@ class Agent:
         content = f'set tws_bank_token "{bank["token"]}"\n'.encode()
         return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
 
-    def write_initial_files(self, root, files):
+    def write_initial_files(self, root, prepared_files):
-        if not isinstance(files, list) or len(files) > 10000:
+        for path, content in prepared_files.items():
-            raise AgentError("Invalid file list")
+            managed_write(root, path, content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
-        for item in files:
-            content = self.decode_file(item)
-            managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
 
-    def decode_file(self, item):
+    def decode_file(self, item, *, content=None):
         if "contentSha256" in item:
             sha = item["contentSha256"]
             if not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha):
                 raise AgentError("Invalid file content hash")
+            if content is None:
                 url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha
                 with self.artifact_open("GET", url) as response:
                     content = response.read(MAX_FILE + 1)
@@ -2053,9 +2057,13 @@ class Agent:
                 raise AgentError("Staged file checksum mismatch")
             return content
         try:
-            content = base64.b64decode(item["contentBase64"], validate=True)
+            decoded = base64.b64decode(item["contentBase64"], validate=True)
         except (ValueError, KeyError, TypeError):
             raise AgentError("Invalid encoded file content") from None
+        if content is None:
+            content = decoded
+        elif content != decoded:
+            raise Rejected("Prepared files do not match the replacement specification")
         if len(content) > MAX_FILE:
             raise AgentError("File exceeds staging size limit")
         if item.get("sha256") and item["sha256"] != digest(content):
@@ -2190,13 +2198,16 @@ class Agent:
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
                     raise AgentError("Rebuild must use a newer immutable generation")
                 spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
-                self.validate_spec(spec)
+                # Keep the complete verified set on this call's stack only. Never
+                # mutate the immutable spec or cache bytes on the shared agent:
+                # held, failed and later operations must prepare their own files.
+                *_, prepared_files = self.validate_spec(spec)
             if not override_safety:
                 self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
             if "autoscaling" in payload:
                 self.authorize_autoscaling_removal(instance_id, generation, payload)
             if kind == "instance.rebuild" and payload.get("stageId"):
-                before = self.rebuild_before(instance_id, generation, payload["stageId"], spec)
+                before = self.rebuild_before(instance_id, generation, payload["stageId"], prepared_files)
                 if not override_safety:
                     self.safe_to_disrupt(instance_id, generation)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})})
@@ -2208,7 +2219,7 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 self.settle_bank(instance_id, generation)
                 if kind == "instance.rebuild":
-                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None)
+                    result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None, prepared_files=prepared_files)
                     self.operation_phase = "removing-old-instance"
                     old = self.instance_dir(instance_id, generation)
                     if old.exists():
@@ -2309,21 +2320,21 @@ class Agent:
         atomic_json(marker, manifest)
         return {"stageId": stage_id, "files": len(planned), "state": "staged"}
 
-    def rebuild_before(self, instance_id, generation, stage_id, spec):
+    def rebuild_before(self, instance_id, generation, stage_id, prepared_files):
         """Validate requested replacement bytes and snapshot the old generation before teardown."""
         directory = self.instance_dir(instance_id, generation)
         marker = directory / "stages" / identifier(stage_id) / "manifest.json"
         saved = json.loads(marker.read_text())
         if saved["state"] != "staged":
             raise Rejected("Rebuild requires an unactivated file stage")
-        replacement = {item["path"]: item for item in self.initial_files(spec)}
         before = []
         for item in saved["files"]:
-            target = replacement.get(item["path"])
+            require_public_file(item["path"])
+            target = prepared_files.get(item["path"])
             if item["delete"]:
                 if target is not None:
                     raise Rejected("Rebuild includes a file requested for removal")
-            elif target is None or digest(self.decode_file(target)) != item["sha256"]:
+            elif target is None or digest(target) != item["sha256"]:
                 raise Rejected("Rebuild does not match staged file content")
             content = managed_read(directory / "app", item["path"])
             actual_sha = None if content is None else digest(content)

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.10.0",
+  "version": "1.10.1",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "1a110c92ae875ee7cd060ac7de82260b63d776d2a865e57356d629e0a99573c7",
+  "sha256": "61481547f99921a5ea2159f863f263d1af7b60114a9d73b00300d098f5abd5af",
   "updaterProtocolVersion": 2,
   "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb",
   "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231"