josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild beta and campaign file activations

Browse Source
This commit is contained in:
josh
2026-10-12 22:52:58 -04:00
parent fa0baa8db5
commit 537b9841a7
22 changed files with 533 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/ec_agent.py
+45 -12
View File
@@ -35,7 +35,7 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.9.1"
VERSION = "1.9.2"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000
@@ -1504,7 +1504,7 @@ class Agent:
if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory()
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.rebuild-files", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
plutonium_cache = self.plutonium_cache_inventory()
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
@@ -1889,7 +1889,7 @@ class Agent:
managed_write(app, marker, raw, uid, gid)
return True
def create(self, instance_id, generation, payload):
def create(self, instance_id, generation, payload, *, before=None):
self.operation_phase = "checking-instance"
self.check_creation_cancellation()
directory = self.instance_dir(instance_id, generation)
@@ -1930,7 +1930,7 @@ class Agent:
args.append(image_id)
self.check_creation_cancellation()
self.operation_phase = "creating-container"
self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha})
self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha, **({"before": before} if before is not None else {})})
self.docker.call(*args, timeout=120)
self.check_creation_cancellation()
self.operation_phase = "starting-container"
@@ -2126,7 +2126,11 @@ class Agent:
self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
if "autoscaling" in payload:
self.authorize_autoscaling_removal(instance_id, generation, payload)
self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
if kind == "instance.rebuild" and payload.get("stageId"):
before = self.rebuild_before(instance_id, generation, payload["stageId"], spec)
if not override_safety:
self.safe_to_disrupt(instance_id, generation)
self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), **({"before": before} if kind == "instance.rebuild" and payload.get("stageId") else {})})
if payload.get("stageId") and kind == "instance.restart":
before = self.activate(instance_id, generation, payload["stageId"], override_safety=override_safety)
if kind in ("instance.remove", "instance.rebuild"):
@@ -2135,12 +2139,12 @@ class Agent:
self.docker.call("container", "rm", name)
self.settle_bank(instance_id, generation)
if kind == "instance.rebuild":
result = self.create(instance_id, next_generation, spec)
result = self.create(instance_id, next_generation, spec, before=before if payload.get("stageId") else None)
self.operation_phase = "removing-old-instance"
old = self.instance_dir(instance_id, generation)
if old.exists():
shutil.rmtree(old, ignore_errors=False)
return result
return {**result, "before": before}
directory = self.instance_dir(instance_id, generation)
if directory.exists():
shutil.rmtree(directory)
@@ -2235,6 +2239,29 @@ class Agent:
atomic_json(marker, manifest)
return {"stageId": stage_id, "files": len(planned), "state": "staged"}
def rebuild_before(self, instance_id, generation, stage_id, spec):
"""Validate requested replacement bytes and snapshot the old generation before teardown."""
directory = self.instance_dir(instance_id, generation)
marker = directory / "stages" / identifier(stage_id) / "manifest.json"
saved = json.loads(marker.read_text())
if saved["state"] != "staged":
raise Rejected("Rebuild requires an unactivated file stage")
replacement = {item["path"]: item for item in self.initial_files(spec)}
before = []
for item in saved["files"]:
target = replacement.get(item["path"])
if item["delete"]:
if target is not None:
raise Rejected("Rebuild includes a file requested for removal")
elif target is None or digest(self.decode_file(target)) != item["sha256"]:
raise Rejected("Rebuild does not match staged file content")
content = managed_read(directory / "app", item["path"])
actual_sha = None if content is None else digest(content)
if item.get("checkBefore") and actual_sha != item.get("expectedBeforeSha"):
raise Rejected("Target changed after campaign review")
before.append({"id": item.get("id"), "path": item["path"], "contentBase64": None if content is None else base64.b64encode(content).decode()})
return before
def activate(self, instance_id, generation, stage_id, *, override_safety=False):
directory = self.instance_dir(instance_id, generation)
stage = directory / "stages" / identifier(stage_id)
@@ -2966,11 +2993,17 @@ class Agent:
health = self.inspect(instance_id, observed_generation)
before = []
if payload.get("stageId"):
stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json"
saved = json.loads(stage.read_text())
if saved["state"] != "activated":
return row
before = saved["before"]
if kind == "instance.rebuild":
# The creating checkpoint retains snapshots after old storage is deleted.
if not isinstance(checkpoint.get("before"), list):
return row
before = checkpoint["before"]
else:
stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json"
saved = json.loads(stage.read_text())
if saved["state"] != "activated":
return row
before = saved["before"]
proven = False
if kind in ("instance.create", "instance.rebuild"):
proven = health["running"] and health.get("creationOperation") == operation["id"] and health.get("specSha256") == checkpoint.get("specSha256")
agent/tests/test_agent.py
+60
View File
@@ -1821,6 +1821,66 @@ class AgentTests(unittest.TestCase):
self.assertTrue(path.is_symlink())
request.assert_not_called()
def staged_rebuild(self, deleting=False):
spec = self.create()
self.agent.rcon = lambda *_: {"ok": True, "players": 0}
file = {"id": 9, "path": "configs/server.cfg", "contentBase64": None if deleting else base64.b64encode(b"replacement").decode(), "checkBefore": True, "expectedBeforeSha": module.digest(b"original")}
staged = self.agent.execute(self.operation("files.stage", {"stageId": "campaign", "files": [file]}, "stage"))
self.assertEqual(staged["state"], "completed")
spec["files"] = [] if deleting else [file]
return self.operation("instance.rebuild", {"spec": spec, "stageId": "campaign", "nextGeneration": 2}, "rebuild")
def test_file_rebuild_replaces_generation_and_retains_before_images(self):
operation = self.staged_rebuild()
result = self.agent.execute(operation)
self.assertEqual(result["state"], "completed", result)
receipt = json.loads(result["result"])
self.assertEqual(receipt["before"], [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"original").decode()}])
self.assertEqual(receipt["generation"], 2)
self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists())
self.assertEqual((self.agent.instance_dir(INSTANCE, 2) / "app/configs/server.cfg").read_bytes(), b"replacement")
self.assertIn("instance.rebuild-files", self.agent.inventory()["capabilities"])
calls = list(self.docker.calls)
self.assertEqual(self.agent.execute(operation)["state"], "completed")
self.assertEqual(self.docker.calls, calls)
def test_file_rebuild_removes_requested_files(self):
operation = self.staged_rebuild(deleting=True)
result = self.agent.execute(operation)
self.assertEqual(result["state"], "completed", result)
self.assertFalse((self.agent.instance_dir(INSTANCE, 2) / "app/configs/server.cfg").exists())
self.assertEqual(len(json.loads(result["result"])["before"]), 1)
def test_file_rebuild_rejects_changed_targets_and_mismatched_replacement(self):
operation = self.staged_rebuild()
target = self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg"
target.write_bytes(b"changed")
self.assertEqual(self.agent.execute(operation)["state"], "failed")
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
target.write_bytes(b"original")
operation["id"] = "mismatched"
operation["payload"]["spec"]["files"] = []
self.assertEqual(self.agent.execute(operation)["state"], "failed")
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
def test_file_rebuild_rechecks_players_after_snapshot_preparation(self):
operation = self.staged_rebuild()
with patch.object(self.agent, "rcon", side_effect=[{"ok": True, "players": 0}, {"ok": True, "players": 1}]):
self.assertEqual(self.agent.execute(operation)["state"], "accepted")
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
def test_file_rebuild_reconciles_snapshots_after_old_storage_was_removed(self):
operation = self.staged_rebuild()
self.assertEqual(self.agent.execute(operation)["state"], "completed")
self.agent.journal.update(operation["id"], "unknown", error="Lost completion")
calls = list(self.docker.calls)
result = self.agent.execute(operation)
self.assertEqual(result["state"], "completed", result)
self.assertEqual(json.loads(result["result"])["before"][0]["contentBase64"], base64.b64encode(b"original").decode())
self.assertEqual(self.docker.calls, calls)
def test_rebuild_moves_generation_and_rejects_old_commands(self):
payload = self.create()
self.agent.rcon = lambda *_: {"ok": True, "players": 0}