Recover unhealthy containers during create and rebuild
This commit is contained in:
@@ -1043,6 +1043,112 @@ class AgentTests(unittest.TestCase):
|
||||
self.assertEqual(self.agent.execute(self.operation("instance.stop"))["state"], "accepted")
|
||||
self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls))
|
||||
|
||||
def assert_unhealthy_recovery(self, kind):
|
||||
spec = self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
payload = {"unhealthyRecovery": True}
|
||||
if kind == "instance.rebuild":
|
||||
payload.update({**spec, "nextGeneration": 2})
|
||||
operation = self.operation(kind, payload)
|
||||
with patch.object(self.agent, "rcon", side_effect=AssertionError("Unhealthy recovery queried player status")):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
calls = list(self.docker.calls)
|
||||
self.assertEqual(self.agent.execute(operation)["state"], "completed")
|
||||
self.assertEqual(self.docker.calls, calls)
|
||||
target_generation = 2 if kind == "instance.rebuild" else 1
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, target_generation)["healthy"])
|
||||
self.assertNotIn("unhealthyRecovery", self.agent.spec(INSTANCE, target_generation))
|
||||
|
||||
def test_unhealthy_recovery_restarts_without_player_status(self):
|
||||
self.assert_unhealthy_recovery("instance.restart")
|
||||
self.assertIn("instance.unhealthy-recovery", self.agent.inventory()["capabilities"])
|
||||
|
||||
def test_unhealthy_recovery_rebuilds_without_player_status(self):
|
||||
self.assert_unhealthy_recovery("instance.rebuild")
|
||||
|
||||
def test_unhealthy_recovery_keeps_safety_for_other_health_states(self):
|
||||
spec = self.create()
|
||||
container = self.docker.containers[self.agent.container_name(INSTANCE, 1)]
|
||||
for kind in ("instance.restart", "instance.rebuild"):
|
||||
payload = {"unhealthyRecovery": True}
|
||||
if kind == "instance.rebuild":
|
||||
payload.update({"spec": spec, "nextGeneration": 2})
|
||||
for state in ("healthy", "starting", "restarting", "unknown"):
|
||||
with self.subTest(kind=kind, state=state):
|
||||
container["State"] = {"Status": "unknown" if state == "unknown" else "running",
|
||||
"Running": state != "unknown", "Health": {"Status": state}}
|
||||
with patch.object(self.agent, "rcon", return_value={"ok": True, "players": 2}) as rcon:
|
||||
result = self.agent.execute(self.operation(kind, payload, f"{kind.split('.')[-1]}-{state}"))
|
||||
self.assertEqual(result["state"], "accepted", result)
|
||||
self.assertEqual(rcon.call_count, 0 if state == "unknown" else 1)
|
||||
self.assertFalse(any(call[:2] in (("container", "restart"), ("container", "stop"), ("container", "rm")) for call in self.docker.calls))
|
||||
|
||||
def test_unhealthy_recovery_rechecks_health_before_disruption(self):
|
||||
spec = self.create()
|
||||
container = self.docker.containers[self.agent.container_name(INSTANCE, 1)]
|
||||
inspect = self.docker.inspect
|
||||
for kind in ("instance.restart", "instance.rebuild"):
|
||||
container["State"]["Health"] = {"Status": "unhealthy"}
|
||||
observations = []
|
||||
|
||||
def recovered(name):
|
||||
observations.append(name)
|
||||
if len(observations) == 2:
|
||||
container["State"]["Health"] = {"Status": "healthy"}
|
||||
return inspect(name)
|
||||
|
||||
payload = {"unhealthyRecovery": True}
|
||||
if kind == "instance.rebuild":
|
||||
payload.update({"spec": spec, "nextGeneration": 2})
|
||||
with patch.object(self.docker, "inspect", side_effect=recovered), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 2}) as rcon:
|
||||
result = self.agent.execute(self.operation(kind, payload, kind.split(".")[-1]))
|
||||
self.assertEqual(result["state"], "accepted", result)
|
||||
rcon.assert_called_once()
|
||||
self.assertFalse(any(call[:2] in (("container", "restart"), ("container", "stop"), ("container", "rm")) for call in self.docker.calls))
|
||||
|
||||
def test_unhealthy_recovery_requires_boolean_true(self):
|
||||
self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
for index, value in enumerate((False, None, "true", 1, {})):
|
||||
with self.subTest(value=value), patch.object(self.agent, "rcon", return_value={"ok": False, "players": None, "reason": "unreachable"}) as rcon:
|
||||
result = self.agent.execute(self.operation("instance.restart", {"unhealthyRecovery": value}, f"attempt-{index}"))
|
||||
self.assertEqual(result["state"], "accepted", result)
|
||||
rcon.assert_called_once()
|
||||
self.assertFalse(any(call[:2] == ("container", "restart") for call in self.docker.calls))
|
||||
|
||||
def test_unhealthy_recovery_rejects_other_lifecycle_actions_and_activation(self):
|
||||
self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
attempts = [(kind, {}) for kind in ("instance.stop", "instance.remove", "instance.start", "instance.create")]
|
||||
attempts.extend([("instance.restart", {"stageId": "campaign"}), ("instance.restart", {"autoscaling": {}})])
|
||||
calls = list(self.docker.calls)
|
||||
for index, (kind, payload) in enumerate(attempts):
|
||||
with self.subTest(kind=kind, payload=payload):
|
||||
result = self.agent.execute(self.operation(kind, {**payload, "unhealthyRecovery": True}, f"attempt-{index}"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("Unhealthy recovery only permits", result["error"])
|
||||
self.assertEqual(self.docker.calls, calls)
|
||||
with patch.object(self.agent, "rcon", return_value={"ok": True, "players": 2}) as rcon:
|
||||
result = self.agent.execute(self.operation("files.activate", {"stageId": "campaign", "unhealthyRecovery": True}))
|
||||
self.assertEqual(result["state"], "accepted", result)
|
||||
rcon.assert_called_once()
|
||||
|
||||
def test_unhealthy_recovery_preserves_ownership_and_generation_checks(self):
|
||||
spec = self.create()
|
||||
container = self.docker.containers[self.agent.container_name(INSTANCE, 1)]
|
||||
container["State"]["Health"] = {"Status": "unhealthy"}
|
||||
labels = container["Config"]["Labels"]
|
||||
container["Config"]["Labels"] = {}
|
||||
result = self.agent.execute(self.operation("instance.restart", {"unhealthyRecovery": True}, "unowned"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
container["Config"]["Labels"] = labels
|
||||
result = self.agent.execute(self.operation("instance.rebuild", {"unhealthyRecovery": True, "spec": spec, "nextGeneration": 1}, "same-generation"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
result = self.agent.execute(self.operation("instance.restart", {"unhealthyRecovery": True}, "stale", generation=0))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertFalse(any(call[:2] in (("container", "restart"), ("container", "stop"), ("container", "rm")) for call in self.docker.calls))
|
||||
|
||||
def assert_override_disruption(self, kind):
|
||||
spec = self.create()
|
||||
payload = {"safetyOverride": True}
|
||||
@@ -1241,6 +1347,88 @@ class AgentTests(unittest.TestCase):
|
||||
self.assertEqual(result["state"], "failed")
|
||||
self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
|
||||
|
||||
def test_cancel_creation_removes_completed_recovery_rebuild(self):
|
||||
spec = self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
rebuild = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2, "unhealthyRecovery": True}, "rebuild")
|
||||
self.assertEqual(self.agent.execute(rebuild)["state"], "completed")
|
||||
cancel = self.operation("instance.cancel-create", {"createOperationId": "create", "recoveryRebuildOperationId": "rebuild", "recoveryGeneration": 2}, "cancel")
|
||||
with patch.object(self.agent, "rcon", side_effect=AssertionError("Cancellation queried player status")):
|
||||
result = self.agent.execute(cancel)
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
self.assertFalse(json.loads(result["result"])["exists"])
|
||||
for generation in (1, 2):
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, generation)["exists"])
|
||||
self.assertFalse(self.agent.instance_dir(INSTANCE, generation).exists())
|
||||
self.assertEqual(json.loads((self.agent.instance_dir(INSTANCE, 1).parent / "current.json").read_text()), {"generation": 2, "removed": True})
|
||||
calls = list(self.docker.calls)
|
||||
self.assertEqual(self.agent.execute(cancel)["state"], "completed")
|
||||
self.assertEqual(self.docker.calls, calls)
|
||||
|
||||
def test_cancel_creation_fences_undelivered_recovery_rebuild(self):
|
||||
spec = self.create()
|
||||
cancel = self.operation("instance.cancel-create", {"createOperationId": "create", "recoveryRebuildOperationId": "late-rebuild", "recoveryGeneration": 2}, "cancel")
|
||||
self.assertEqual(self.agent.execute(cancel)["state"], "completed")
|
||||
rebuild = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2, "unhealthyRecovery": True}, "late-rebuild")
|
||||
result = self.agent.execute(rebuild)
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
|
||||
self.assertEqual(sum(call[:2] == ("container", "create") for call in self.docker.calls), 1)
|
||||
|
||||
def test_cancel_creation_reconciles_recovery_rebuild_removal(self):
|
||||
spec = self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"}
|
||||
self.docker.lose_reply_after = "start"
|
||||
rebuild = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2, "unhealthyRecovery": True}, "rebuild")
|
||||
self.assertEqual(self.agent.execute(rebuild)["state"], "unknown")
|
||||
self.docker.lose_reply_after = "rm"
|
||||
cancel = self.operation("instance.cancel-create", {"createOperationId": "create", "recoveryRebuildOperationId": "rebuild", "recoveryGeneration": 2}, "cancel")
|
||||
self.assertEqual(self.agent.execute(cancel)["state"], "unknown")
|
||||
self.agent.journal.close()
|
||||
self.agent = Agent(self.config, self.docker)
|
||||
self.assertEqual(self.agent.execute(cancel)["state"], "completed")
|
||||
for generation in (1, 2):
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, generation)["exists"])
|
||||
self.assertFalse(self.agent.instance_dir(INSTANCE, generation).exists())
|
||||
|
||||
def test_cancel_creation_rejects_mismatched_recovery_rebuild(self):
|
||||
spec = self.create()
|
||||
for index, (kind, instance_id, generation, changes) in enumerate((
|
||||
("instance.restart", INSTANCE, 1, {}),
|
||||
("instance.rebuild", "another-instance", 1, {}),
|
||||
("instance.rebuild", INSTANCE, 2, {}),
|
||||
("instance.rebuild", INSTANCE, 1, {"nextGeneration": 3}),
|
||||
("instance.rebuild", INSTANCE, 1, {"unhealthyRecovery": False}),
|
||||
)):
|
||||
rebuild = self.operation(kind, {"spec": spec, "nextGeneration": 2, "unhealthyRecovery": True, **changes}, f"rebuild-{index}", generation=generation)
|
||||
rebuild["instanceId"] = instance_id
|
||||
self.agent.journal.accept(rebuild)
|
||||
cancel = self.operation("instance.cancel-create", {"createOperationId": "create", "recoveryRebuildOperationId": rebuild["id"], "recoveryGeneration": 2}, f"cancel-{index}")
|
||||
result = self.agent.execute(cancel)
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("does not match", result["error"])
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["exists"])
|
||||
self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
|
||||
|
||||
def test_cancel_creation_rejects_unrelated_recovery_container_or_newer_generation(self):
|
||||
self.create()
|
||||
original = self.docker.containers[self.agent.container_name(INSTANCE, 1)]
|
||||
unrelated = json.loads(json.dumps(original))
|
||||
unrelated["Config"]["Labels"].update({module.LABEL + ".generation": "2", module.LABEL + ".operation": "unrelated"})
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 2)] = unrelated
|
||||
payload = {"createOperationId": "create", "recoveryRebuildOperationId": "rebuild", "recoveryGeneration": 2}
|
||||
result = self.agent.execute(self.operation("instance.cancel-create", payload, "unrelated-container"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("different creation", result["error"])
|
||||
marker = self.agent.instance_dir(INSTANCE, 1).parent / "current.json"
|
||||
module.atomic_json(marker, {"generation": 3})
|
||||
result = self.agent.execute(self.operation("instance.cancel-create", payload, "newer-generation"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("superseded", result["error"])
|
||||
self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
|
||||
self.assertEqual(json.loads(marker.read_text()), {"generation": 3})
|
||||
|
||||
def test_unowned_container_never_mutated(self):
|
||||
self.create()
|
||||
self.docker.containers[self.agent.container_name(INSTANCE, 1)]["Config"]["Labels"] = {}
|
||||
|
||||
Reference in New Issue
Block a user