diff --git a/agent/CHANGELOG.md b/agent/CHANGELOG.md index b59cdc9..9f03af7 100644 --- a/agent/CHANGELOG.md +++ b/agent/CHANGELOG.md @@ -1,5 +1,22 @@ # Agent changelog +## 2026-10-14 + +### 1.10.1 — Retain verified rebuild artifacts (EC #154) + +- Retain the complete verified replacement file set before stopping the current + container, including inline files and private bank configuration, within the + existing 64 MiB operation limit. Staged-file comparison, replacement validation + and writing reuse those bytes without further artifact downloads. +- Keep prepared bytes local to the operation. Failed and held attempts discard + their files; later operations prepare a fresh set. Immutable specifications, + journal requests and container identity labels remain unchanged. +- Preserve checksum, path, per-file size, player-safety, bank-settlement and + generation checks. Add regressions for preflight failures, lost connectivity + after preflight, file-set limits and operation isolation. +- Package release artifacts for offline signing before publication. No protocol + change or controller migration is required. + ## 2026-10-13 ### 1.10.0 — Host temperature telemetry diff --git a/agent/ec_agent.py b/agent/ec_agent.py index a420f94..b3b50d8 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.10.0" +VERSION = "1.10.1" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1685,7 +1685,7 @@ class Agent: players = status_players(text) if command == "status" else None return {"ok": True, "text": text, "players": players} - def validate_spec(self, payload): + def validate_spec(self, payload, *, prepared_files=None): game = payload.get("game") if game not in ("t5", "t6"): raise AgentError("Only T5 and T6 are supported") @@ -1728,16 +1728,22 @@ class Agent: files = self.initial_files(payload) if not isinstance(files, list) or len(files) > 10000: raise Rejected("Invalid initial file list") + if prepared_files is not None: + if set(prepared_files) != {item["path"] for item in files} or any(not isinstance(content, bytes) for content in prepared_files.values()): + raise Rejected("Prepared files do not match the replacement specification") + verified_files = {} total, paths = 0, set() for item in files: scoped(self.root / "agent" / "validation", item["path"]) if item["path"] in paths: raise Rejected("Duplicate initial file path") paths.add(item["path"]) - total += len(self.decode_file(item)) + content = self.decode_file(item, content=None if prepared_files is None else prepared_files[item["path"]]) + total += len(content) if total > 64 * 1024 * 1024: raise Rejected("Initial files exceed the 64 MiB operation limit") - return game, library, image, image_id, port, bind, env + verified_files[item["path"]] = content + return game, library, image, image_id, port, bind, env, verified_files @staticmethod def plutonium_manifest(data): @@ -1956,7 +1962,7 @@ class Agent: managed_write(app, marker, raw, uid, gid) return True - def create(self, instance_id, generation, payload, *, before=None): + def create(self, instance_id, generation, payload, *, before=None, prepared_files=None): self.operation_phase = "checking-instance" self.check_creation_cancellation() directory = self.instance_dir(instance_id, generation) @@ -1971,7 +1977,7 @@ class Agent: raise AgentError("Instance already exists with a different specification") return existing self.operation_phase = "validating-create-spec" - game, library, image, image_id, port, bind, env = self.validate_spec(payload) + game, library, image, image_id, port, bind, env, prepared_files = self.validate_spec(payload, prepared_files=prepared_files) app = directory / "app" self.operation_phase = "preparing-create-directory" app.mkdir(parents=True, exist_ok=True) @@ -1979,7 +1985,7 @@ class Agent: self.operation_phase = "seeding-runtime" self.seed_plutonium(app, image) self.operation_phase = "writing-create-files" - self.write_initial_files(app, self.initial_files(payload)) + self.write_initial_files(app, prepared_files) self.operation_phase = "persisting-create-metadata" atomic_json(directory / "spec.json", payload) env_path = directory / "environment" @@ -2032,30 +2038,32 @@ class Agent: content = f'set tws_bank_token "{bank["token"]}"\n'.encode() return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}] - def write_initial_files(self, root, files): - if not isinstance(files, list) or len(files) > 10000: - raise AgentError("Invalid file list") - for item in files: - content = self.decode_file(item) - managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + def write_initial_files(self, root, prepared_files): + for path, content in prepared_files.items(): + managed_write(root, path, content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) - def decode_file(self, item): + def decode_file(self, item, *, content=None): if "contentSha256" in item: sha = item["contentSha256"] if not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha): raise AgentError("Invalid file content hash") - url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha - with self.artifact_open("GET", url) as response: - content = response.read(MAX_FILE + 1) + if content is None: + url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha + with self.artifact_open("GET", url) as response: + content = response.read(MAX_FILE + 1) if len(content) > MAX_FILE: raise AgentError("File exceeds staging size limit") if digest(content) != sha or item.get("sha256", sha) != sha: raise AgentError("Staged file checksum mismatch") return content try: - content = base64.b64decode(item["contentBase64"], validate=True) + decoded = base64.b64decode(item["contentBase64"], validate=True) except (ValueError, KeyError, TypeError): raise AgentError("Invalid encoded file content") from None + if content is None: + content = decoded + elif content != decoded: + raise Rejected("Prepared files do not match the replacement specification") if len(content) > MAX_FILE: raise AgentError("File exceeds staging size limit") if item.get("sha256") and item["sha256"] != digest(content): @@ -2190,13 +2198,16 @@ class Agent: if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation: raise AgentError("Rebuild must use a newer immutable generation") spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")} - self.validate_spec(spec) + # Keep the complete verified set on this call's stack only. Never + # mutate the immutable spec or cache bytes on the shared agent: + # held, failed and later operations must prepare their own files. + *_, prepared_files = self.validate_spec(spec) if not override_safety: self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery) if "autoscaling" in payload: self.authorize_autoscaling_removal(instance_id, generation, payload) if kind == "instance.rebuild" and payload.get("stageId"): - before = self.rebuild_before(instance_id, generation, payload["stageId"], spec) + before = self.rebuild_before(instance_id, generation, payload["stageId"], prepared_files) if not override_safety: self.safe_to_disrupt(instance_id, generation) self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})}) @@ -2208,7 +2219,7 @@ class Agent: self.docker.call("container", "rm", name) self.settle_bank(instance_id, generation) if kind == "instance.rebuild": - result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None) + result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None, prepared_files=prepared_files) self.operation_phase = "removing-old-instance" old = self.instance_dir(instance_id, generation) if old.exists(): @@ -2309,21 +2320,21 @@ class Agent: atomic_json(marker, manifest) return {"stageId": stage_id, "files": len(planned), "state": "staged"} - def rebuild_before(self, instance_id, generation, stage_id, spec): + def rebuild_before(self, instance_id, generation, stage_id, prepared_files): """Validate requested replacement bytes and snapshot the old generation before teardown.""" directory = self.instance_dir(instance_id, generation) marker = directory / "stages" / identifier(stage_id) / "manifest.json" saved = json.loads(marker.read_text()) if saved["state"] != "staged": raise Rejected("Rebuild requires an unactivated file stage") - replacement = {item["path"]: item for item in self.initial_files(spec)} before = [] for item in saved["files"]: - target = replacement.get(item["path"]) + require_public_file(item["path"]) + target = prepared_files.get(item["path"]) if item["delete"]: if target is not None: raise Rejected("Rebuild includes a file requested for removal") - elif target is None or digest(self.decode_file(target)) != item["sha256"]: + elif target is None or digest(target) != item["sha256"]: raise Rejected("Rebuild does not match staged file content") content = managed_read(directory / "app", item["path"]) actual_sha = None if content is None else digest(content) diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 8388315..8e7e9fb 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -1666,7 +1666,7 @@ class AgentTests(unittest.TestCase): with self.assertRaisesRegex(AgentError, "Private bank"): self.agent.activate(INSTANCE, 1, "private", override_safety=True) with self.assertRaisesRegex(AgentError, "Private bank"): - self.agent.rebuild_before(INSTANCE, 1, "private", {"files": []}) + self.agent.rebuild_before(INSTANCE, 1, "private", {}) with self.assertRaisesRegex(AgentError, "Private bank"): self.agent.restore_files(app, [{"path": value, "contentBase64": None}]) with self.assertRaisesRegex(AgentError, "Private bank"): @@ -1934,6 +1934,164 @@ class AgentTests(unittest.TestCase): self.assertEqual(json.loads(result["result"])["before"][0]["contentBase64"], base64.b64encode(b"original").decode()) self.assertEqual(self.docker.calls, calls) + def artifact_rebuild(self, staged=False): + if staged: + operation = self.staged_rebuild() + else: + spec = self.create() + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + operation = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2}, "rebuild") + spec = operation["payload"]["spec"] + contents = {"configs/server.cfg": b"replacement", "scripts/replacement.gsc": b"verified artifact bytes"} + spec["files"] = [{"path": path, "contentSha256": module.digest(content), "sha256": module.digest(content)} for path, content in contents.items()] + spec["files"].append({"path": "configs/empty.cfg", "contentBase64": ""}) + spec["bank"] = {"configPath": "configs/electric_cherry_bank.cfg", "token": "ec_bank_" + "a" * 32} + return operation, contents + + def assert_rebuild_uses_preflight_artifacts(self, staged): + operation, contents = self.artifact_rebuild(staged) + original_request = json.dumps(operation, sort_keys=True) + available = {module.digest(content): content for content in contents.values()} + + def fetch(method, url): + # Connectivity disappears as soon as the complete preflight set has + # been read, before staged comparison or destructive replacement. + if not available: + raise module.urllib.error.URLError(TimeoutError()) + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + return Response(available.pop(url.rsplit("/", 1)[-1])) + + with patch.object(self.agent, "artifact_open", side_effect=fetch) as downloads: + result = self.agent.execute(operation) + self.assertEqual(result["state"], "completed", result) + self.assertEqual(downloads.call_count, len(contents)) + self.assertFalse(available) + self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"]) + self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"]) + app = self.agent.instance_dir(INSTANCE, 2) / "app" + for path, content in contents.items(): + self.assertEqual((app / path).read_bytes(), content) + self.assertEqual((app / "configs/empty.cfg").read_bytes(), b"") + self.assertEqual((app / "configs/electric_cherry_bank.cfg").read_text(), 'set tws_bank_token "ec_bank_' + "a" * 32 + '"\n') + self.assertEqual(json.dumps(operation, sort_keys=True), original_request) + self.assertEqual(json.loads(result["request"]), operation) + self.assertEqual(self.agent.spec(INSTANCE, 2), operation["payload"]["spec"]) + labels = self.docker.inspect(self.agent.container_name(INSTANCE, 2))["Config"]["Labels"] + self.assertEqual(labels[module.LABEL + ".operation"], operation["id"]) + self.assertEqual(labels[module.LABEL + ".spec-sha256"], module.digest(json.dumps(operation["payload"]["spec"], sort_keys=True, separators=(",", ":")).encode())) + receipt = json.loads(result["result"]) + self.assertEqual(receipt["before"], [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"original").decode()}] if staged else []) + for content in contents.values(): + self.assertNotIn(base64.b64encode(content).decode(), json.dumps(result)) + + # A later operation with the same artifacts must download its own set. + later = self.operation("instance.rebuild", {"spec": operation["payload"]["spec"], "nextGeneration": 3}, "later-rebuild", generation=2) + with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads: + self.assertEqual(self.agent.execute(later)["state"], "failed") + downloads.assert_called_once() + self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"]) + self.assertFalse(self.agent.inspect(INSTANCE, 3)["exists"]) + + def test_unstaged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self): + self.assert_rebuild_uses_preflight_artifacts(staged=False) + + def test_staged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self): + self.assert_rebuild_uses_preflight_artifacts(staged=True) + + def test_rebuild_preflight_download_failure_preserves_original_and_discards_partial_set(self): + operation, contents = self.artifact_rebuild() + with patch.object(self.agent, "artifact_open", side_effect=[Response(contents["configs/server.cfg"]), module.urllib.error.URLError(TimeoutError())]) as downloads: + result = self.agent.execute(operation) + self.assertEqual(result["state"], "failed", result) + self.assertEqual(downloads.call_count, 2) + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + self.assertFalse(self.agent.instance_dir(INSTANCE, 2).exists()) + self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls)) + operation["id"] = "retry-rebuild" + with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads: + self.assertEqual(self.agent.execute(operation)["state"], "failed") + self.assertTrue(downloads.call_args.args[1].endswith(module.digest(contents["configs/server.cfg"]))) + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + + def test_held_rebuild_discards_prepared_artifacts_before_retry(self): + operation, contents = self.artifact_rebuild(staged=True) + with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}): + self.assertEqual(self.agent.execute(operation)["state"], "accepted") + with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads: + self.assertEqual(self.agent.execute(operation)["state"], "failed") + downloads.assert_called_once() + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"]) + + def test_rebuild_preflight_rejects_corrupt_and_oversized_artifacts(self): + operation, contents = self.artifact_rebuild() + for index, content in enumerate((b"corrupt", b"x" * (module.MAX_FILE + 1))): + with self.subTest(index=index): + operation["id"] = f"invalid-artifact-{index}" + with patch.object(self.agent, "artifact_open", return_value=Response(content)): + result = self.agent.execute(operation) + self.assertEqual(result["state"], "failed", result) + self.assertIn("checksum mismatch" if index == 0 else "size limit", result["error"]) + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls)) + + def test_rebuild_preflight_bounds_complete_file_set_at_64_mib(self): + operation, _ = self.artifact_rebuild() + spec = operation["payload"]["spec"] + content = b"x" * module.MAX_FILE + spec["files"] = [{"path": f"scripts/{index}.gsc", "contentSha256": module.digest(content)} for index in range(2)] + # The generated private bank file also counts toward the operation limit. + with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)): + result = self.agent.execute(operation) + self.assertEqual(result["state"], "failed", result) + self.assertIn("64 MiB operation limit", result["error"]) + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls)) + del spec["bank"] + operation["id"] = "at-limit" + with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}): + self.assertEqual(self.agent.execute(operation)["state"], "accepted") + + def test_replacement_validation_rejects_incomplete_or_mismatched_prepared_bytes_without_downloads(self): + operation, contents = self.artifact_rebuild() + spec = operation["payload"]["spec"] + del spec["bank"] + prepared = {**contents, "configs/empty.cfg": b""} + invalid_sets = [ + {path: content for path, content in prepared.items() if path != "configs/server.cfg"}, + {**prepared, "configs/extra.cfg": b"unexpected"}, + {**prepared, "configs/server.cfg": b"wrong artifact"}, + {**prepared, "configs/empty.cfg": b"wrong inline bytes"}, + {**prepared, "configs/server.cfg": None}, + ] + with patch.object(self.agent, "artifact_open", side_effect=AssertionError("Prepared validation must not download")) as downloads: + for index, files in enumerate(invalid_sets): + with self.subTest(index=index), self.assertRaises(AgentError): + self.agent.validate_spec(spec, prepared_files=files) + downloads.assert_not_called() + + def test_rebuild_preflight_preserves_file_path_and_duplicate_checks(self): + operation, _ = self.artifact_rebuild() + spec = operation["payload"]["spec"] + for index, paths in enumerate((["../escape.cfg"], ["configs/electric_cherry_bank.cfg"], ["configs/same.cfg"] * 2)): + with self.subTest(paths=paths): + operation["id"] = f"invalid-paths-{index}" + spec["files"] = [{"path": path, "contentBase64": ""} for path in paths] + self.assertEqual(self.agent.execute(operation)["state"], "failed") + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls)) + + def test_staged_rebuild_rejects_verified_artifacts_that_differ_from_the_stage(self): + operation, contents = self.artifact_rebuild(staged=True) + contents["configs/server.cfg"] = b"different verified replacement" + operation["payload"]["spec"]["files"][0].update(contentSha256=module.digest(contents["configs/server.cfg"]), sha256=module.digest(contents["configs/server.cfg"])) + with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]): + result = self.agent.execute(operation) + self.assertEqual(result["state"], "failed", result) + self.assertIn("does not match staged file content", result["error"]) + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls)) + def test_rebuild_moves_generation_and_rejects_old_commands(self): payload = self.create() self.agent.rcon = lambda *_: {"ok": True, "players": 0} diff --git a/docs/guides/native-hosts.md b/docs/guides/native-hosts.md index 2703fee..64a09e9 100644 --- a/docs/guides/native-hosts.md +++ b/docs/guides/native-hosts.md @@ -654,6 +654,13 @@ restart and create complete only after post-operation health evidence; creation and rebuild also verify the selected map and running Plutonium revision. The base-game library and runtime revision are independent records. +Agent 1.10.1 prepares and verifies the complete replacement file set before +stopping the existing container, within the 64 MiB operation limit. Staged-file +comparison, replacement validation and writing reuse those bytes, so losing +artifact connectivity after preflight does not prevent replacement. A preflight +download failure leaves the original container running. Prepared bytes last only +for that operation attempt; held or failed work must prepare its files again. + ## Rebalance instances Open **Fleet → Capacity → Rebalance** to spread existing instances across active diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index a420f94..b3b50d8 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.10.0" +VERSION = "1.10.1" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1685,7 +1685,7 @@ class Agent: players = status_players(text) if command == "status" else None return {"ok": True, "text": text, "players": players} - def validate_spec(self, payload): + def validate_spec(self, payload, *, prepared_files=None): game = payload.get("game") if game not in ("t5", "t6"): raise AgentError("Only T5 and T6 are supported") @@ -1728,16 +1728,22 @@ class Agent: files = self.initial_files(payload) if not isinstance(files, list) or len(files) > 10000: raise Rejected("Invalid initial file list") + if prepared_files is not None: + if set(prepared_files) != {item["path"] for item in files} or any(not isinstance(content, bytes) for content in prepared_files.values()): + raise Rejected("Prepared files do not match the replacement specification") + verified_files = {} total, paths = 0, set() for item in files: scoped(self.root / "agent" / "validation", item["path"]) if item["path"] in paths: raise Rejected("Duplicate initial file path") paths.add(item["path"]) - total += len(self.decode_file(item)) + content = self.decode_file(item, content=None if prepared_files is None else prepared_files[item["path"]]) + total += len(content) if total > 64 * 1024 * 1024: raise Rejected("Initial files exceed the 64 MiB operation limit") - return game, library, image, image_id, port, bind, env + verified_files[item["path"]] = content + return game, library, image, image_id, port, bind, env, verified_files @staticmethod def plutonium_manifest(data): @@ -1956,7 +1962,7 @@ class Agent: managed_write(app, marker, raw, uid, gid) return True - def create(self, instance_id, generation, payload, *, before=None): + def create(self, instance_id, generation, payload, *, before=None, prepared_files=None): self.operation_phase = "checking-instance" self.check_creation_cancellation() directory = self.instance_dir(instance_id, generation) @@ -1971,7 +1977,7 @@ class Agent: raise AgentError("Instance already exists with a different specification") return existing self.operation_phase = "validating-create-spec" - game, library, image, image_id, port, bind, env = self.validate_spec(payload) + game, library, image, image_id, port, bind, env, prepared_files = self.validate_spec(payload, prepared_files=prepared_files) app = directory / "app" self.operation_phase = "preparing-create-directory" app.mkdir(parents=True, exist_ok=True) @@ -1979,7 +1985,7 @@ class Agent: self.operation_phase = "seeding-runtime" self.seed_plutonium(app, image) self.operation_phase = "writing-create-files" - self.write_initial_files(app, self.initial_files(payload)) + self.write_initial_files(app, prepared_files) self.operation_phase = "persisting-create-metadata" atomic_json(directory / "spec.json", payload) env_path = directory / "environment" @@ -2032,30 +2038,32 @@ class Agent: content = f'set tws_bank_token "{bank["token"]}"\n'.encode() return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}] - def write_initial_files(self, root, files): - if not isinstance(files, list) or len(files) > 10000: - raise AgentError("Invalid file list") - for item in files: - content = self.decode_file(item) - managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + def write_initial_files(self, root, prepared_files): + for path, content in prepared_files.items(): + managed_write(root, path, content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) - def decode_file(self, item): + def decode_file(self, item, *, content=None): if "contentSha256" in item: sha = item["contentSha256"] if not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha): raise AgentError("Invalid file content hash") - url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha - with self.artifact_open("GET", url) as response: - content = response.read(MAX_FILE + 1) + if content is None: + url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha + with self.artifact_open("GET", url) as response: + content = response.read(MAX_FILE + 1) if len(content) > MAX_FILE: raise AgentError("File exceeds staging size limit") if digest(content) != sha or item.get("sha256", sha) != sha: raise AgentError("Staged file checksum mismatch") return content try: - content = base64.b64decode(item["contentBase64"], validate=True) + decoded = base64.b64decode(item["contentBase64"], validate=True) except (ValueError, KeyError, TypeError): raise AgentError("Invalid encoded file content") from None + if content is None: + content = decoded + elif content != decoded: + raise Rejected("Prepared files do not match the replacement specification") if len(content) > MAX_FILE: raise AgentError("File exceeds staging size limit") if item.get("sha256") and item["sha256"] != digest(content): @@ -2190,13 +2198,16 @@ class Agent: if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation: raise AgentError("Rebuild must use a newer immutable generation") spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")} - self.validate_spec(spec) + # Keep the complete verified set on this call's stack only. Never + # mutate the immutable spec or cache bytes on the shared agent: + # held, failed and later operations must prepare their own files. + *_, prepared_files = self.validate_spec(spec) if not override_safety: self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery) if "autoscaling" in payload: self.authorize_autoscaling_removal(instance_id, generation, payload) if kind == "instance.rebuild" and payload.get("stageId"): - before = self.rebuild_before(instance_id, generation, payload["stageId"], spec) + before = self.rebuild_before(instance_id, generation, payload["stageId"], prepared_files) if not override_safety: self.safe_to_disrupt(instance_id, generation) self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})}) @@ -2208,7 +2219,7 @@ class Agent: self.docker.call("container", "rm", name) self.settle_bank(instance_id, generation) if kind == "instance.rebuild": - result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None) + result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None, prepared_files=prepared_files) self.operation_phase = "removing-old-instance" old = self.instance_dir(instance_id, generation) if old.exists(): @@ -2309,21 +2320,21 @@ class Agent: atomic_json(marker, manifest) return {"stageId": stage_id, "files": len(planned), "state": "staged"} - def rebuild_before(self, instance_id, generation, stage_id, spec): + def rebuild_before(self, instance_id, generation, stage_id, prepared_files): """Validate requested replacement bytes and snapshot the old generation before teardown.""" directory = self.instance_dir(instance_id, generation) marker = directory / "stages" / identifier(stage_id) / "manifest.json" saved = json.loads(marker.read_text()) if saved["state"] != "staged": raise Rejected("Rebuild requires an unactivated file stage") - replacement = {item["path"]: item for item in self.initial_files(spec)} before = [] for item in saved["files"]: - target = replacement.get(item["path"]) + require_public_file(item["path"]) + target = prepared_files.get(item["path"]) if item["delete"]: if target is not None: raise Rejected("Rebuild includes a file requested for removal") - elif target is None or digest(self.decode_file(target)) != item["sha256"]: + elif target is None or digest(target) != item["sha256"]: raise Rejected("Rebuild does not match staged file content") content = managed_read(directory / "app", item["path"]) actual_sha = None if content is None else digest(content) diff --git a/static/agent/manifest.json b/static/agent/manifest.json index e334beb..21fe969 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.10.0", + "version": "1.10.1", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "1a110c92ae875ee7cd060ac7de82260b63d776d2a865e57356d629e0a99573c7", + "sha256": "61481547f99921a5ea2159f863f263d1af7b60114a9d73b00300d098f5abd5af", "updaterProtocolVersion": 2, "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb", "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231"