josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Reuse verified artifacts during rebuilds (Refs #154)

Browse Source
This commit is contained in:
josh
2026-10-14 11:56:24 -04:00
parent 171f657876
commit 61dbfb2c5f
6 changed files with 257 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/CHANGELOG.md
+17
View File
@@ -1,5 +1,22 @@
# Agent changelog
## 2026-10-14
### 1.10.1 — Retain verified rebuild artifacts (EC #154)
- Retain the complete verified replacement file set before stopping the current
container, including inline files and private bank configuration, within the
existing 64 MiB operation limit. Staged-file comparison, replacement validation
and writing reuse those bytes without further artifact downloads.
- Keep prepared bytes local to the operation. Failed and held attempts discard
their files; later operations prepare a fresh set. Immutable specifications,
journal requests and container identity labels remain unchanged.
- Preserve checksum, path, per-file size, player-safety, bank-settlement and
generation checks. Add regressions for preflight failures, lost connectivity
after preflight, file-set limits and operation isolation.
- Package release artifacts for offline signing before publication. No protocol
change or controller migration is required.
## 2026-10-13
### 1.10.0 — Host temperature telemetry
agent/ec_agent.py
+36 -25
View File
@@ -35,7 +35,7 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.10.0"
VERSION = "1.10.1"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000
@@ -1685,7 +1685,7 @@ class Agent:
players = status_players(text) if command == "status" else None
return {"ok": True, "text": text, "players": players}
def validate_spec(self, payload):
def validate_spec(self, payload, *, prepared_files=None):
game = payload.get("game")
if game not in ("t5", "t6"):
raise AgentError("Only T5 and T6 are supported")
@@ -1728,16 +1728,22 @@ class Agent:
files = self.initial_files(payload)
if not isinstance(files, list) or len(files) > 10000:
raise Rejected("Invalid initial file list")
if prepared_files is not None:
if set(prepared_files) != {item["path"] for item in files} or any(not isinstance(content, bytes) for content in prepared_files.values()):
raise Rejected("Prepared files do not match the replacement specification")
verified_files = {}
total, paths = 0, set()
for item in files:
scoped(self.root / "agent" / "validation", item["path"])
if item["path"] in paths:
raise Rejected("Duplicate initial file path")
paths.add(item["path"])
total += len(self.decode_file(item))
content = self.decode_file(item, content=None if prepared_files is None else prepared_files[item["path"]])
total += len(content)
if total > 64 * 1024 * 1024:
raise Rejected("Initial files exceed the 64 MiB operation limit")
return game, library, image, image_id, port, bind, env
verified_files[item["path"]] = content
return game, library, image, image_id, port, bind, env, verified_files
@staticmethod
def plutonium_manifest(data):
@@ -1956,7 +1962,7 @@ class Agent:
managed_write(app, marker, raw, uid, gid)
return True
def create(self, instance_id, generation, payload, *, before=None):
def create(self, instance_id, generation, payload, *, before=None, prepared_files=None):
self.operation_phase = "checking-instance"
self.check_creation_cancellation()
directory = self.instance_dir(instance_id, generation)
@@ -1971,7 +1977,7 @@ class Agent:
raise AgentError("Instance already exists with a different specification")
return existing
self.operation_phase = "validating-create-spec"
game, library, image, image_id, port, bind, env = self.validate_spec(payload)
game, library, image, image_id, port, bind, env, prepared_files = self.validate_spec(payload, prepared_files=prepared_files)
app = directory / "app"
self.operation_phase = "preparing-create-directory"
app.mkdir(parents=True, exist_ok=True)
@@ -1979,7 +1985,7 @@ class Agent:
self.operation_phase = "seeding-runtime"
self.seed_plutonium(app, image)
self.operation_phase = "writing-create-files"
self.write_initial_files(app, self.initial_files(payload))
self.write_initial_files(app, prepared_files)
self.operation_phase = "persisting-create-metadata"
atomic_json(directory / "spec.json", payload)
env_path = directory / "environment"
@@ -2032,30 +2038,32 @@ class Agent:
content = f'set tws_bank_token "{bank["token"]}"\n'.encode()
return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
def write_initial_files(self, root, files):
if not isinstance(files, list) or len(files) > 10000:
raise AgentError("Invalid file list")
for item in files:
content = self.decode_file(item)
managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
def write_initial_files(self, root, prepared_files):
for path, content in prepared_files.items():
managed_write(root, path, content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000)))
def decode_file(self, item):
def decode_file(self, item, *, content=None):
if "contentSha256" in item:
sha = item["contentSha256"]
if not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha):
raise AgentError("Invalid file content hash")
url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha
with self.artifact_open("GET", url) as response:
content = response.read(MAX_FILE + 1)
if content is None:
url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha
with self.artifact_open("GET", url) as response:
content = response.read(MAX_FILE + 1)
if len(content) > MAX_FILE:
raise AgentError("File exceeds staging size limit")
if digest(content) != sha or item.get("sha256", sha) != sha:
raise AgentError("Staged file checksum mismatch")
return content
try:
content = base64.b64decode(item["contentBase64"], validate=True)
decoded = base64.b64decode(item["contentBase64"], validate=True)
except (ValueError, KeyError, TypeError):
raise AgentError("Invalid encoded file content") from None
if content is None:
content = decoded
elif content != decoded:
raise Rejected("Prepared files do not match the replacement specification")
if len(content) > MAX_FILE:
raise AgentError("File exceeds staging size limit")
if item.get("sha256") and item["sha256"] != digest(content):
@@ -2190,13 +2198,16 @@ class Agent:
if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
raise AgentError("Rebuild must use a newer immutable generation")
spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
self.validate_spec(spec)
# Keep the complete verified set on this call's stack only. Never
# mutate the immutable spec or cache bytes on the shared agent:
# held, failed and later operations must prepare their own files.
*_, prepared_files = self.validate_spec(spec)
if not override_safety:
self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
if "autoscaling" in payload:
self.authorize_autoscaling_removal(instance_id, generation, payload)
if kind == "instance.rebuild" and payload.get("stageId"):
before = self.rebuild_before(instance_id, generation, payload["stageId"], spec)
before = self.rebuild_before(instance_id, generation, payload["stageId"], prepared_files)
if not override_safety:
self.safe_to_disrupt(instance_id, generation)
self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})})
@@ -2208,7 +2219,7 @@ class Agent:
self.docker.call("container", "rm", name)
self.settle_bank(instance_id, generation)
if kind == "instance.rebuild":
result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None)
result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None, prepared_files=prepared_files)
self.operation_phase = "removing-old-instance"
old = self.instance_dir(instance_id, generation)
if old.exists():
@@ -2309,21 +2320,21 @@ class Agent:
atomic_json(marker, manifest)
return {"stageId": stage_id, "files": len(planned), "state": "staged"}
def rebuild_before(self, instance_id, generation, stage_id, spec):
def rebuild_before(self, instance_id, generation, stage_id, prepared_files):
"""Validate requested replacement bytes and snapshot the old generation before teardown."""
directory = self.instance_dir(instance_id, generation)
marker = directory / "stages" / identifier(stage_id) / "manifest.json"
saved = json.loads(marker.read_text())
if saved["state"] != "staged":
raise Rejected("Rebuild requires an unactivated file stage")
replacement = {item["path"]: item for item in self.initial_files(spec)}
before = []
for item in saved["files"]:
target = replacement.get(item["path"])
require_public_file(item["path"])
target = prepared_files.get(item["path"])
if item["delete"]:
if target is not None:
raise Rejected("Rebuild includes a file requested for removal")
elif target is None or digest(self.decode_file(target)) != item["sha256"]:
elif target is None or digest(target) != item["sha256"]:
raise Rejected("Rebuild does not match staged file content")
content = managed_read(directory / "app", item["path"])
actual_sha = None if content is None else digest(content)
agent/tests/test_agent.py
+159 -1
View File
@@ -1666,7 +1666,7 @@ class AgentTests(unittest.TestCase):
with self.assertRaisesRegex(AgentError, "Private bank"):
self.agent.activate(INSTANCE, 1, "private", override_safety=True)
with self.assertRaisesRegex(AgentError, "Private bank"):
self.agent.rebuild_before(INSTANCE, 1, "private", {"files": []})
self.agent.rebuild_before(INSTANCE, 1, "private", {})
with self.assertRaisesRegex(AgentError, "Private bank"):
self.agent.restore_files(app, [{"path": value, "contentBase64": None}])
with self.assertRaisesRegex(AgentError, "Private bank"):
@@ -1934,6 +1934,164 @@ class AgentTests(unittest.TestCase):
self.assertEqual(json.loads(result["result"])["before"][0]["contentBase64"], base64.b64encode(b"original").decode())
self.assertEqual(self.docker.calls, calls)
def artifact_rebuild(self, staged=False):
if staged:
operation = self.staged_rebuild()
else:
spec = self.create()
self.agent.rcon = lambda *_: {"ok": True, "players": 0}
operation = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2}, "rebuild")
spec = operation["payload"]["spec"]
contents = {"configs/server.cfg": b"replacement", "scripts/replacement.gsc": b"verified artifact bytes"}
spec["files"] = [{"path": path, "contentSha256": module.digest(content), "sha256": module.digest(content)} for path, content in contents.items()]
spec["files"].append({"path": "configs/empty.cfg", "contentBase64": ""})
spec["bank"] = {"configPath": "configs/electric_cherry_bank.cfg", "token": "ec_bank_" + "a" * 32}
return operation, contents
def assert_rebuild_uses_preflight_artifacts(self, staged):
operation, contents = self.artifact_rebuild(staged)
original_request = json.dumps(operation, sort_keys=True)
available = {module.digest(content): content for content in contents.values()}
def fetch(method, url):
# Connectivity disappears as soon as the complete preflight set has
# been read, before staged comparison or destructive replacement.
if not available:
raise module.urllib.error.URLError(TimeoutError())
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
return Response(available.pop(url.rsplit("/", 1)[-1]))
with patch.object(self.agent, "artifact_open", side_effect=fetch) as downloads:
result = self.agent.execute(operation)
self.assertEqual(result["state"], "completed", result)
self.assertEqual(downloads.call_count, len(contents))
self.assertFalse(available)
self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
app = self.agent.instance_dir(INSTANCE, 2) / "app"
for path, content in contents.items():
self.assertEqual((app / path).read_bytes(), content)
self.assertEqual((app / "configs/empty.cfg").read_bytes(), b"")
self.assertEqual((app / "configs/electric_cherry_bank.cfg").read_text(), 'set tws_bank_token "ec_bank_' + "a" * 32 + '"\n')
self.assertEqual(json.dumps(operation, sort_keys=True), original_request)
self.assertEqual(json.loads(result["request"]), operation)
self.assertEqual(self.agent.spec(INSTANCE, 2), operation["payload"]["spec"])
labels = self.docker.inspect(self.agent.container_name(INSTANCE, 2))["Config"]["Labels"]
self.assertEqual(labels[module.LABEL + ".operation"], operation["id"])
self.assertEqual(labels[module.LABEL + ".spec-sha256"], module.digest(json.dumps(operation["payload"]["spec"], sort_keys=True, separators=(",", ":")).encode()))
receipt = json.loads(result["result"])
self.assertEqual(receipt["before"], [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"original").decode()}] if staged else [])
for content in contents.values():
self.assertNotIn(base64.b64encode(content).decode(), json.dumps(result))
# A later operation with the same artifacts must download its own set.
later = self.operation("instance.rebuild", {"spec": operation["payload"]["spec"], "nextGeneration": 3}, "later-rebuild", generation=2)
with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
self.assertEqual(self.agent.execute(later)["state"], "failed")
downloads.assert_called_once()
self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
self.assertFalse(self.agent.inspect(INSTANCE, 3)["exists"])
def test_unstaged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self):
self.assert_rebuild_uses_preflight_artifacts(staged=False)
def test_staged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self):
self.assert_rebuild_uses_preflight_artifacts(staged=True)
def test_rebuild_preflight_download_failure_preserves_original_and_discards_partial_set(self):
operation, contents = self.artifact_rebuild()
with patch.object(self.agent, "artifact_open", side_effect=[Response(contents["configs/server.cfg"]), module.urllib.error.URLError(TimeoutError())]) as downloads:
result = self.agent.execute(operation)
self.assertEqual(result["state"], "failed", result)
self.assertEqual(downloads.call_count, 2)
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
self.assertFalse(self.agent.instance_dir(INSTANCE, 2).exists())
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
operation["id"] = "retry-rebuild"
with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
self.assertEqual(self.agent.execute(operation)["state"], "failed")
self.assertTrue(downloads.call_args.args[1].endswith(module.digest(contents["configs/server.cfg"])))
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
def test_held_rebuild_discards_prepared_artifacts_before_retry(self):
operation, contents = self.artifact_rebuild(staged=True)
with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}):
self.assertEqual(self.agent.execute(operation)["state"], "accepted")
with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
self.assertEqual(self.agent.execute(operation)["state"], "failed")
downloads.assert_called_once()
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
def test_rebuild_preflight_rejects_corrupt_and_oversized_artifacts(self):
operation, contents = self.artifact_rebuild()
for index, content in enumerate((b"corrupt", b"x" * (module.MAX_FILE + 1))):
with self.subTest(index=index):
operation["id"] = f"invalid-artifact-{index}"
with patch.object(self.agent, "artifact_open", return_value=Response(content)):
result = self.agent.execute(operation)
self.assertEqual(result["state"], "failed", result)
self.assertIn("checksum mismatch" if index == 0 else "size limit", result["error"])
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
def test_rebuild_preflight_bounds_complete_file_set_at_64_mib(self):
operation, _ = self.artifact_rebuild()
spec = operation["payload"]["spec"]
content = b"x" * module.MAX_FILE
spec["files"] = [{"path": f"scripts/{index}.gsc", "contentSha256": module.digest(content)} for index in range(2)]
# The generated private bank file also counts toward the operation limit.
with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)):
result = self.agent.execute(operation)
self.assertEqual(result["state"], "failed", result)
self.assertIn("64 MiB operation limit", result["error"])
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
del spec["bank"]
operation["id"] = "at-limit"
with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}):
self.assertEqual(self.agent.execute(operation)["state"], "accepted")
def test_replacement_validation_rejects_incomplete_or_mismatched_prepared_bytes_without_downloads(self):
operation, contents = self.artifact_rebuild()
spec = operation["payload"]["spec"]
del spec["bank"]
prepared = {**contents, "configs/empty.cfg": b""}
invalid_sets = [
{path: content for path, content in prepared.items() if path != "configs/server.cfg"},
{**prepared, "configs/extra.cfg": b"unexpected"},
{**prepared, "configs/server.cfg": b"wrong artifact"},
{**prepared, "configs/empty.cfg": b"wrong inline bytes"},
{**prepared, "configs/server.cfg": None},
]
with patch.object(self.agent, "artifact_open", side_effect=AssertionError("Prepared validation must not download")) as downloads:
for index, files in enumerate(invalid_sets):
with self.subTest(index=index), self.assertRaises(AgentError):
self.agent.validate_spec(spec, prepared_files=files)
downloads.assert_not_called()
def test_rebuild_preflight_preserves_file_path_and_duplicate_checks(self):
operation, _ = self.artifact_rebuild()
spec = operation["payload"]["spec"]
for index, paths in enumerate((["../escape.cfg"], ["configs/electric_cherry_bank.cfg"], ["configs/same.cfg"] * 2)):
with self.subTest(paths=paths):
operation["id"] = f"invalid-paths-{index}"
spec["files"] = [{"path": path, "contentBase64": ""} for path in paths]
self.assertEqual(self.agent.execute(operation)["state"], "failed")
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
def test_staged_rebuild_rejects_verified_artifacts_that_differ_from_the_stage(self):
operation, contents = self.artifact_rebuild(staged=True)
contents["configs/server.cfg"] = b"different verified replacement"
operation["payload"]["spec"]["files"][0].update(contentSha256=module.digest(contents["configs/server.cfg"]), sha256=module.digest(contents["configs/server.cfg"]))
with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]):
result = self.agent.execute(operation)
self.assertEqual(result["state"], "failed", result)
self.assertIn("does not match staged file content", result["error"])
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
def test_rebuild_moves_generation_and_rejects_old_commands(self):
payload = self.create()
self.agent.rcon = lambda *_: {"ok": True, "players": 0}