Recover unhealthy containers during create and rebuild
This commit is contained in:
+45
-21
@@ -33,7 +33,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.8.0"
|
||||
VERSION = "1.8.1"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
MANIFEST_MAX_ENTRIES = 10000
|
||||
@@ -1394,14 +1394,18 @@ class Agent:
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
update = self.update_inventory()
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
plutonium_cache = self.plutonium_cache_inventory()
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation):
|
||||
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
||||
health = self.inspect(instance_id, generation)
|
||||
if health["state"] in ("missing", "stopped", "exited", "created", "dead"):
|
||||
return
|
||||
# Provisioning recovery may treat explicitly unhealthy containers as
|
||||
# empty. Reinspect here so recovered containers still require occupancy.
|
||||
if allow_unhealthy and health["state"] == "unhealthy":
|
||||
return
|
||||
if not health["running"]:
|
||||
raise Held("Container state is unknown; disruption held")
|
||||
result = self.rcon(instance_id, generation, "status")
|
||||
@@ -1857,6 +1861,9 @@ class Agent:
|
||||
raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
|
||||
|
||||
def lifecycle(self, kind, instance_id, generation, payload):
|
||||
unhealthy_recovery = payload.get("unhealthyRecovery") is True
|
||||
if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload):
|
||||
raise AgentError("Unhealthy recovery only permits restart or rebuild without file activation or autoscaling")
|
||||
if kind == "instance.cancel-create":
|
||||
return self.cancel_creation(instance_id, generation, payload)
|
||||
if kind == "instance.create":
|
||||
@@ -1891,10 +1898,10 @@ class Agent:
|
||||
next_generation = payload.get("nextGeneration", payload.get("newGeneration"))
|
||||
if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
|
||||
raise AgentError("Rebuild must use a newer immutable generation")
|
||||
spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration")}
|
||||
spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
|
||||
self.validate_spec(spec)
|
||||
if not override_safety:
|
||||
self.safe_to_disrupt(instance_id, generation)
|
||||
self.safe_to_disrupt(instance_id, generation, allow_unhealthy=unhealthy_recovery)
|
||||
if "autoscaling" in payload:
|
||||
self.authorize_autoscaling_removal(instance_id, generation, payload)
|
||||
self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
|
||||
@@ -1928,29 +1935,46 @@ class Agent:
|
||||
Repeating this cleanup after an ambiguous Docker result is safe.
|
||||
"""
|
||||
create_id = identifier(payload.get("createOperationId"))
|
||||
targets = [(generation, create_id)]
|
||||
if "recoveryRebuildOperationId" in payload or "recoveryGeneration" in payload:
|
||||
rebuild_id = identifier(payload.get("recoveryRebuildOperationId"))
|
||||
recovery_generation = payload.get("recoveryGeneration")
|
||||
if isinstance(recovery_generation, bool) or not isinstance(recovery_generation, int) or recovery_generation != generation + 1:
|
||||
raise Rejected("Cancellation recovery must target the next generation")
|
||||
saved = self.journal.db.execute("SELECT request FROM operations WHERE id=?", (rebuild_id,)).fetchone()
|
||||
if saved:
|
||||
rebuild = json.loads(saved["request"])
|
||||
rebuild_payload = rebuild.get("payload", {})
|
||||
if rebuild["kind"] != "instance.rebuild" or rebuild["instanceId"] != instance_id or rebuild["generation"] != generation or rebuild_payload.get("nextGeneration", rebuild_payload.get("newGeneration")) != recovery_generation or rebuild_payload.get("unhealthyRecovery") is not True:
|
||||
raise Rejected("Cancellation does not match the creation recovery rebuild")
|
||||
targets.append((recovery_generation, rebuild_id))
|
||||
marker = self.instance_dir(instance_id, generation).parent / "current.json"
|
||||
if marker.exists() and json.loads(marker.read_text())["generation"] > generation:
|
||||
if marker.exists() and json.loads(marker.read_text())["generation"] > targets[-1][0]:
|
||||
raise Rejected("Cancelled creation was superseded by a newer generation")
|
||||
saved = self.journal.db.execute("SELECT request FROM operations WHERE id=?", (create_id,)).fetchone()
|
||||
if saved:
|
||||
original = json.loads(saved["request"])
|
||||
if original["kind"] != "instance.create" or original["instanceId"] != instance_id or original["generation"] != generation:
|
||||
raise Rejected("Cancellation does not match the original creation")
|
||||
health = self.inspect(instance_id, generation)
|
||||
if health["exists"] and health.get("creationOperation") != create_id:
|
||||
raise Rejected("Container belongs to a different creation operation")
|
||||
health_by_generation = {}
|
||||
for target_generation, operation_id in targets:
|
||||
health = self.inspect(instance_id, target_generation)
|
||||
if health["exists"] and health.get("creationOperation") != operation_id:
|
||||
raise Rejected("Container belongs to a different creation operation")
|
||||
health_by_generation[target_generation] = health
|
||||
self.checkpoint({"phase": "cancelling-creation", "createOperationId": create_id})
|
||||
# Fence before deleting: a crash must never permit the old creation to run again.
|
||||
self.mark_removed(instance_id, generation)
|
||||
if health["exists"]:
|
||||
self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, generation))
|
||||
after = self.inspect(instance_id, generation)
|
||||
if after["exists"]:
|
||||
raise AgentError("Cancelled creation container still exists")
|
||||
directory = self.instance_dir(instance_id, generation)
|
||||
if directory.exists():
|
||||
shutil.rmtree(directory)
|
||||
return {**after, "cancelledCreation": create_id}
|
||||
# Fence both generations before deleting: delayed create/rebuild work
|
||||
# must never recreate either target after cancellation or a crash.
|
||||
self.mark_removed(instance_id, targets[-1][0])
|
||||
for target_generation, _ in targets:
|
||||
if health_by_generation[target_generation]["exists"]:
|
||||
self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation))
|
||||
if self.inspect(instance_id, target_generation)["exists"]:
|
||||
raise AgentError("Cancelled creation container still exists")
|
||||
directory = self.instance_dir(instance_id, target_generation)
|
||||
if directory.exists():
|
||||
shutil.rmtree(directory)
|
||||
return {**self.inspect(instance_id, generation), "cancelledCreation": create_id}
|
||||
|
||||
def stage(self, instance_id, generation, payload, operation_id):
|
||||
directory = self.instance_dir(instance_id, generation)
|
||||
@@ -2453,7 +2477,7 @@ class Agent:
|
||||
current_marker = self.instance_dir(instance_id, generation).parent / "current.json"
|
||||
if current_marker.exists():
|
||||
current = json.loads(current_marker.read_text())
|
||||
if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove", "instance.cancel-create"):
|
||||
if kind != "instance.cancel-create" and (current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove")):
|
||||
raise AgentError("Instance generation has been superseded or removed")
|
||||
if kind.startswith("instance."):
|
||||
return self.lifecycle(kind, instance_id, generation, payload)
|
||||
|
||||
@@ -1043,6 +1043,112 @@ class AgentTests(unittest.TestCase):
|
||||
self.assertEqual(self.agent.execute(self.operation("instance.stop"))["state"], "accepted")
|
||||
self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls))
|
||||
|
||||
def assert_unhealthy_recovery(self, kind):
|
||||
spec = self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
payload = {"unhealthyRecovery": True}
|
||||
if kind == "instance.rebuild":
|
||||
payload.update({**spec, "nextGeneration": 2})
|
||||
operation = self.operation(kind, payload)
|
||||
with patch.object(self.agent, "rcon", side_effect=AssertionError("Unhealthy recovery queried player status")):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
calls = list(self.docker.calls)
|
||||
self.assertEqual(self.agent.execute(operation)["state"], "completed")
|
||||
self.assertEqual(self.docker.calls, calls)
|
||||
target_generation = 2 if kind == "instance.rebuild" else 1
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, target_generation)["healthy"])
|
||||
self.assertNotIn("unhealthyRecovery", self.agent.spec(INSTANCE, target_generation))
|
||||
|
||||
def test_unhealthy_recovery_restarts_without_player_status(self):
|
||||
self.assert_unhealthy_recovery("instance.restart")
|
||||
self.assertIn("instance.unhealthy-recovery", self.agent.inventory()["capabilities"])
|
||||
|
||||
def test_unhealthy_recovery_rebuilds_without_player_status(self):
|
||||
self.assert_unhealthy_recovery("instance.rebuild")
|
||||
|
||||
def test_unhealthy_recovery_keeps_safety_for_other_health_states(self):
|
||||
spec = self.create()
|
||||
container = self.docker.containers[self.agent.container_name(INSTANCE, 1)]
|
||||
for kind in ("instance.restart", "instance.rebuild"):
|
||||
payload = {"unhealthyRecovery": True}
|
||||
if kind == "instance.rebuild":
|
||||
payload.update({"spec": spec, "nextGeneration": 2})
|
||||
for state in ("healthy", "starting", "restarting", "unknown"):
|
||||
with self.subTest(kind=kind, state=state):
|
||||
container["State"] = {"Status": "unknown" if state == "unknown" else "running",
|
||||
"Running": state != "unknown", "Health": {"Status": state}}
|
||||
with patch.object(self.agent, "rcon", return_value={"ok": True, "players": 2}) as rcon:
|
||||
result = self.agent.execute(self.operation(kind, payload, f"{kind.split('.')[-1]}-{state}"))
|
||||
self.assertEqual(result["state"], "accepted", result)
|
||||
self.assertEqual(rcon.call_count, 0 if state == "unknown" else 1)
|
||||
self.assertFalse(any(call[:2] in (("container", "restart"), ("container", "stop"), ("container", "rm")) for call in self.docker.calls))
|
||||
|
||||
def test_unhealthy_recovery_rechecks_health_before_disruption(self):
|
||||
spec = self.create()
|
||||
container = self.docker.containers[self.agent.container_name(INSTANCE, 1)]
|
||||
inspect = self.docker.inspect
|
||||
for kind in ("instance.restart", "instance.rebuild"):
|
||||
container["State"]["Health"] = {"Status": "unhealthy"}
|
||||
observations = []
|
||||
|
||||
def recovered(name):
|
||||
observations.append(name)
|
||||
if len(observations) == 2:
|
||||
container["State"]["Health"] = {"Status": "healthy"}
|
||||
return inspect(name)
|
||||
|
||||
payload = {"unhealthyRecovery": True}
|
||||
if kind == "instance.rebuild":
|
||||
payload.update({"spec": spec, "nextGeneration": 2})
|
||||
with patch.object(self.docker, "inspect", side_effect=recovered), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 2}) as rcon:
|
||||
result = self.agent.execute(self.operation(kind, payload, kind.split(".")[-1]))
|
||||
self.assertEqual(result["state"], "accepted", result)
|
||||
rcon.assert_called_once()
|
||||
self.assertFalse(any(call[:2] in (("container", "restart"), ("container", "stop"), ("container", "rm")) for call in self.docker.calls))
|
||||
|
||||
def test_unhealthy_recovery_requires_boolean_true(self):
|
||||
self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
for index, value in enumerate((False, None, "true", 1, {})):
|
||||
with self.subTest(value=value), patch.object(self.agent, "rcon", return_value={"ok": False, "players": None, "reason": "unreachable"}) as rcon:
|
||||
result = self.agent.execute(self.operation("instance.restart", {"unhealthyRecovery": value}, f"attempt-{index}"))
|
||||
self.assertEqual(result["state"], "accepted", result)
|
||||
rcon.assert_called_once()
|
||||
self.assertFalse(any(call[:2] == ("container", "restart") for call in self.docker.calls))
|
||||
|
||||
def test_unhealthy_recovery_rejects_other_lifecycle_actions_and_activation(self):
|
||||
self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
attempts = [(kind, {}) for kind in ("instance.stop", "instance.remove", "instance.start", "instance.create")]
|
||||
attempts.extend([("instance.restart", {"stageId": "campaign"}), ("instance.restart", {"autoscaling": {}})])
|
||||
calls = list(self.docker.calls)
|
||||
for index, (kind, payload) in enumerate(attempts):
|
||||
with self.subTest(kind=kind, payload=payload):
|
||||
result = self.agent.execute(self.operation(kind, {**payload, "unhealthyRecovery": True}, f"attempt-{index}"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("Unhealthy recovery only permits", result["error"])
|
||||
self.assertEqual(self.docker.calls, calls)
|
||||
with patch.object(self.agent, "rcon", return_value={"ok": True, "players": 2}) as rcon:
|
||||
result = self.agent.execute(self.operation("files.activate", {"stageId": "campaign", "unhealthyRecovery": True}))
|
||||
self.assertEqual(result["state"], "accepted", result)
|
||||
rcon.assert_called_once()
|
||||
|
||||
def test_unhealthy_recovery_preserves_ownership_and_generation_checks(self):
|
||||
spec = self.create()
|
||||
container = self.docker.containers[self.agent.container_name(INSTANCE, 1)]
|
||||
container["State"]["Health"] = {"Status": "unhealthy"}
|
||||
labels = container["Config"]["Labels"]
|
||||
container["Config"]["Labels"] = {}
|
||||
result = self.agent.execute(self.operation("instance.restart", {"unhealthyRecovery": True}, "unowned"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
container["Config"]["Labels"] = labels
|
||||
result = self.agent.execute(self.operation("instance.rebuild", {"unhealthyRecovery": True, "spec": spec, "nextGeneration": 1}, "same-generation"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
result = self.agent.execute(self.operation("instance.restart", {"unhealthyRecovery": True}, "stale", generation=0))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertFalse(any(call[:2] in (("container", "restart"), ("container", "stop"), ("container", "rm")) for call in self.docker.calls))
|
||||
|
||||
def assert_override_disruption(self, kind):
|
||||
spec = self.create()
|
||||
payload = {"safetyOverride": True}
|
||||
@@ -1241,6 +1347,88 @@ class AgentTests(unittest.TestCase):
|
||||
self.assertEqual(result["state"], "failed")
|
||||
self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
|
||||
|
||||
def test_cancel_creation_removes_completed_recovery_rebuild(self):
|
||||
spec = self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
rebuild = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2, "unhealthyRecovery": True}, "rebuild")
|
||||
self.assertEqual(self.agent.execute(rebuild)["state"], "completed")
|
||||
cancel = self.operation("instance.cancel-create", {"createOperationId": "create", "recoveryRebuildOperationId": "rebuild", "recoveryGeneration": 2}, "cancel")
|
||||
with patch.object(self.agent, "rcon", side_effect=AssertionError("Cancellation queried player status")):
|
||||
result = self.agent.execute(cancel)
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
self.assertFalse(json.loads(result["result"])["exists"])
|
||||
for generation in (1, 2):
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, generation)["exists"])
|
||||
self.assertFalse(self.agent.instance_dir(INSTANCE, generation).exists())
|
||||
self.assertEqual(json.loads((self.agent.instance_dir(INSTANCE, 1).parent / "current.json").read_text()), {"generation": 2, "removed": True})
|
||||
calls = list(self.docker.calls)
|
||||
self.assertEqual(self.agent.execute(cancel)["state"], "completed")
|
||||
self.assertEqual(self.docker.calls, calls)
|
||||
|
||||
def test_cancel_creation_fences_undelivered_recovery_rebuild(self):
|
||||
spec = self.create()
|
||||
cancel = self.operation("instance.cancel-create", {"createOperationId": "create", "recoveryRebuildOperationId": "late-rebuild", "recoveryGeneration": 2}, "cancel")
|
||||
self.assertEqual(self.agent.execute(cancel)["state"], "completed")
|
||||
rebuild = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2, "unhealthyRecovery": True}, "late-rebuild")
|
||||
result = self.agent.execute(rebuild)
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
|
||||
self.assertEqual(sum(call[:2] == ("container", "create") for call in self.docker.calls), 1)
|
||||
|
||||
def test_cancel_creation_reconciles_recovery_rebuild_removal(self):
|
||||
spec = self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
self.docker.lose_reply_after = "start"
|
||||
rebuild = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2, "unhealthyRecovery": True}, "rebuild")
|
||||
self.assertEqual(self.agent.execute(rebuild)["state"], "unknown")
|
||||
self.docker.lose_reply_after = "rm"
|
||||
cancel = self.operation("instance.cancel-create", {"createOperationId": "create", "recoveryRebuildOperationId": "rebuild", "recoveryGeneration": 2}, "cancel")
|
||||
self.assertEqual(self.agent.execute(cancel)["state"], "unknown")
|
||||
self.agent.journal.close()
|
||||
self.agent = Agent(self.config, self.docker)
|
||||
self.assertEqual(self.agent.execute(cancel)["state"], "completed")
|
||||
for generation in (1, 2):
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, generation)["exists"])
|
||||
self.assertFalse(self.agent.instance_dir(INSTANCE, generation).exists())
|
||||
|
||||
def test_cancel_creation_rejects_mismatched_recovery_rebuild(self):
|
||||
spec = self.create()
|
||||
for index, (kind, instance_id, generation, changes) in enumerate((
|
||||
("instance.restart", INSTANCE, 1, {}),
|
||||
("instance.rebuild", "another-instance", 1, {}),
|
||||
("instance.rebuild", INSTANCE, 2, {}),
|
||||
("instance.rebuild", INSTANCE, 1, {"nextGeneration": 3}),
|
||||
("instance.rebuild", INSTANCE, 1, {"unhealthyRecovery": False}),
|
||||
)):
|
||||
rebuild = self.operation(kind, {"spec": spec, "nextGeneration": 2, "unhealthyRecovery": True, **changes}, f"rebuild-{index}", generation=generation)
|
||||
rebuild["instanceId"] = instance_id
|
||||
self.agent.journal.accept(rebuild)
|
||||
cancel = self.operation("instance.cancel-create", {"createOperationId": "create", "recoveryRebuildOperationId": rebuild["id"], "recoveryGeneration": 2}, f"cancel-{index}")
|
||||
result = self.agent.execute(cancel)
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("does not match", result["error"])
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["exists"])
|
||||
self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
|
||||
|
||||
def test_cancel_creation_rejects_unrelated_recovery_container_or_newer_generation(self):
|
||||
self.create()
|
||||
original = self.docker.containers[self.agent.container_name(INSTANCE, 1)]
|
||||
unrelated = json.loads(json.dumps(original))
|
||||
unrelated["Config"]["Labels"].update({module.LABEL + ".generation": "2", module.LABEL + ".operation": "unrelated"})
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 2)] = unrelated
|
||||
payload = {"createOperationId": "create", "recoveryRebuildOperationId": "rebuild", "recoveryGeneration": 2}
|
||||
result = self.agent.execute(self.operation("instance.cancel-create", payload, "unrelated-container"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("different creation", result["error"])
|
||||
marker = self.agent.instance_dir(INSTANCE, 1).parent / "current.json"
|
||||
module.atomic_json(marker, {"generation": 3})
|
||||
result = self.agent.execute(self.operation("instance.cancel-create", payload, "newer-generation"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("superseded", result["error"])
|
||||
self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
|
||||
self.assertEqual(json.loads(marker.read_text()), {"generation": 3})
|
||||
|
||||
def test_unowned_container_never_mutated(self):
|
||||
self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["Config"]["Labels"] = {}
|
||||
|
||||
Reference in New Issue
Block a user