Recover unhealthy containers during create and rebuild
This commit is contained in:
+45
-21
@@ -33,7 +33,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.8.0"
|
||||
VERSION = "1.8.1"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
MANIFEST_MAX_ENTRIES = 10000
|
||||
@@ -1394,14 +1394,18 @@ class Agent:
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
update = self.update_inventory()
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
plutonium_cache = self.plutonium_cache_inventory()
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation):
|
||||
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
||||
health = self.inspect(instance_id, generation)
|
||||
if health["state"] in ("missing", "stopped", "exited", "created", "dead"):
|
||||
return
|
||||
# Provisioning recovery may treat explicitly unhealthy containers as
|
||||
# empty. Reinspect here so recovered containers still require occupancy.
|
||||
if allow_unhealthy and health["state"] == "unhealthy":
|
||||
return
|
||||
if not health["running"]:
|
||||
raise Held("Container state is unknown; disruption held")
|
||||
result = self.rcon(instance_id, generation, "status")
|
||||
@@ -1857,6 +1861,9 @@ class Agent:
|
||||
raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
|
||||
|
||||
def lifecycle(self, kind, instance_id, generation, payload):
|
||||
unhealthy_recovery = payload.get("unhealthyRecovery") is True
|
||||
if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload):
|
||||
raise AgentError("Unhealthy recovery only permits restart or rebuild without file activation or autoscaling")
|
||||
if kind == "instance.cancel-create":
|
||||
return self.cancel_creation(instance_id, generation, payload)
|
||||
if kind == "instance.create":
|
||||
@@ -1891,10 +1898,10 @@ class Agent:
|
||||
next_generation = payload.get("nextGeneration", payload.get("newGeneration"))
|
||||
if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
|
||||
raise AgentError("Rebuild must use a newer immutable generation")
|
||||
spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration")}
|
||||
spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
|
||||
self.validate_spec(spec)
|
||||
if not override_safety:
|
||||
self.safe_to_disrupt(instance_id, generation)
|
||||
self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
|
||||
if "autoscaling" in payload:
|
||||
self.authorize_autoscaling_removal(instance_id, generation, payload)
|
||||
self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
|
||||
@@ -1928,29 +1935,46 @@ class Agent:
|
||||
Repeating this cleanup after an ambiguous Docker result is safe.
|
||||
"""
|
||||
create_id = identifier(payload.get("createOperationId"))
|
||||
targets = [(generation, create_id)]
|
||||
if "recoveryRebuildOperationId" in payload or "recoveryGeneration" in payload:
|
||||
rebuild_id = identifier(payload.get("recoveryRebuildOperationId"))
|
||||
recovery_generation = payload.get("recoveryGeneration")
|
||||
if isinstance(recovery_generation, bool) or not isinstance(recovery_generation, int) or recovery_generation != generation + 1:
|
||||
raise Rejected("Cancellation recovery must target the next generation")
|
||||
saved = self.journal.db.execute("SELECT request FROM operations WHERE id=?", (rebuild_id,)).fetchone()
|
||||
if saved:
|
||||
rebuild = json.loads(saved["request"])
|
||||
rebuild_payload = rebuild.get("payload", {})
|
||||
if rebuild["kind"] != "instance.rebuild" or rebuild["instanceId"] != instance_id or rebuild["generation"] != generation or rebuild_payload.get("nextGeneration", rebuild_payload.get("newGeneration")) != recovery_generation or rebuild_payload.get("unhealthyRecovery") is not True:
|
||||
raise Rejected("Cancellation does not match the creation recovery rebuild")
|
||||
targets.append((recovery_generation, rebuild_id))
|
||||
marker = self.instance_dir(instance_id, generation).parent / "current.json"
|
||||
if marker.exists() and json.loads(marker.read_text())["generation"] > generation:
|
||||
if marker.exists() and json.loads(marker.read_text())["generation"] > targets[-1][0]:
|
||||
raise Rejected("Cancelled creation was superseded by a newer generation")
|
||||
saved = self.journal.db.execute("SELECT request FROM operations WHERE id=?", (create_id,)).fetchone()
|
||||
if saved:
|
||||
original = json.loads(saved["request"])
|
||||
if original["kind"] != "instance.create" or original["instanceId"] != instance_id or original["generation"] != generation:
|
||||
raise Rejected("Cancellation does not match the original creation")
|
||||
health = self.inspect(instance_id, generation)
|
||||
if health["exists"] and health.get("creationOperation") != create_id:
|
||||
raise Rejected("Container belongs to a different creation operation")
|
||||
health_by_generation = {}
|
||||
for target_generation, operation_id in targets:
|
||||
health = self.inspect(instance_id, target_generation)
|
||||
if health["exists"] and health.get("creationOperation") != operation_id:
|
||||
raise Rejected("Container belongs to a different creation operation")
|
||||
health_by_generation[target_generation] = health
|
||||
self.checkpoint({"phase": "cancelling-creation", "createOperationId": create_id})
|
||||
# Fence before deleting: a crash must never permit the old creation to run again.
|
||||
self.mark_removed(instance_id, generation)
|
||||
if health["exists"]:
|
||||
self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, generation))
|
||||
after = self.inspect(instance_id, generation)
|
||||
if after["exists"]:
|
||||
raise AgentError("Cancelled creation container still exists")
|
||||
directory = self.instance_dir(instance_id, generation)
|
||||
if directory.exists():
|
||||
shutil.rmtree(directory)
|
||||
return {**after, "cancelledCreation": create_id}
|
||||
# Fence both generations before deleting: delayed create/rebuild work
|
||||
# must never recreate either target after cancellation or a crash.
|
||||
self.mark_removed(instance_id, targets[-1][0])
|
||||
for target_generation, _ in targets:
|
||||
if health_by_generation[target_generation]["exists"]:
|
||||
self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation))
|
||||
if self.inspect(instance_id, target_generation)["exists"]:
|
||||
raise AgentError("Cancelled creation container still exists")
|
||||
directory = self.instance_dir(instance_id, target_generation)
|
||||
if directory.exists():
|
||||
shutil.rmtree(directory)
|
||||
return {**self.inspect(instance_id, generation), "cancelledCreation": create_id}
|
||||
|
||||
def stage(self, instance_id, generation, payload, operation_id):
|
||||
directory = self.instance_dir(instance_id, generation)
|
||||
@@ -2453,7 +2477,7 @@ class Agent:
|
||||
current_marker = self.instance_dir(instance_id, generation).parent / "current.json"
|
||||
if current_marker.exists():
|
||||
current = json.loads(current_marker.read_text())
|
||||
if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove", "instance.cancel-create"):
|
||||
if kind != "instance.cancel-create" and (current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove")):
|
||||
raise AgentError("Instance generation has been superseded or removed")
|
||||
if kind.startswith("instance."):
|
||||
return self.lifecycle(kind, instance_id, generation, payload)
|
||||
|
||||
Reference in New Issue
Block a user