Add confirmed admin override for provisioning player checks

agent/ec_agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.14"
+VERSION = "1.4.15"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1332,7 +1332,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1569,16 +1569,20 @@ class Agent:
             else:
                 self.docker.call("container", "start", name)
         elif kind in ("instance.stop", "instance.restart", "instance.remove", "instance.rebuild"):
+            # The authenticated controller records explicit administrator approval
+            # for this immutable operation. Ownership and generation checks still apply.
+            override_safety = payload.get("safetyOverride") is True
             if kind == "instance.rebuild":
                 next_generation = payload.get("nextGeneration", payload.get("newGeneration"))
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
                     raise AgentError("Rebuild must use a newer immutable generation")
                 spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration")}
                 self.validate_spec(spec)
-            self.safe_to_disrupt(instance_id, generation)
+            if not override_safety:
+                self.safe_to_disrupt(instance_id, generation)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
             if payload.get("stageId") and kind == "instance.restart":
-                before = self.activate(instance_id, generation, payload["stageId"])
+                before = self.activate(instance_id, generation, payload["stageId"], override_safety=override_safety)
             if kind in ("instance.remove", "instance.rebuild"):
                 if health["exists"]:
                     self.docker.call("container", "stop", "--time", "30", name)
@@ -1592,7 +1596,7 @@ class Agent:
                     shutil.rmtree(directory)
                 self.mark_removed(instance_id, generation)
             elif health["exists"]:
-                if kind == "instance.restart" and payload.get("stageId"):
+                if kind == "instance.restart" and payload.get("stageId") and not override_safety:
                     self.safe_to_disrupt(instance_id, generation)
                 self.docker.call("container", "stop" if kind.endswith("stop") else "restart", "--time", "30", name)
         else:
@@ -1663,7 +1667,7 @@ class Agent:
         atomic_json(marker, manifest)
         return {"stageId": stage_id, "files": len(planned), "state": "staged"}
 
-    def activate(self, instance_id, generation, stage_id):
+    def activate(self, instance_id, generation, stage_id, *, override_safety=False):
         directory = self.instance_dir(instance_id, generation)
         stage = directory / "stages" / identifier(stage_id)
         marker = stage / "manifest.json"
@@ -1681,7 +1685,8 @@ class Agent:
             before.append({"id": item.get("id"), "path": item["path"], "contentBase64": None if content is None else base64.b64encode(content).decode()})
         # Snapshot preparation may take time. Occupancy is rechecked at the
         # local activation boundary, after all file I/O and integrity checks.
-        self.safe_to_disrupt(instance_id, generation)
+        if not override_safety:
+            self.safe_to_disrupt(instance_id, generation)
         saved.update(state="activating", before=before)
         atomic_json(marker, saved)
         try:

agent/tests/test_agent.py

@@ -830,6 +830,64 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(self.agent.execute(self.operation("instance.stop"))["state"], "accepted")
         self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls))
 
+    def assert_override_disruption(self, kind):
+        spec = self.create()
+        payload = {"safetyOverride": True}
+        if kind == "instance.rebuild":
+            payload.update({**spec, "nextGeneration": 2})
+        operation = self.operation(kind, payload)
+        with patch.object(self.agent, "rcon", side_effect=AssertionError("Override queried player status")):
+            result = self.agent.execute(operation)
+            self.assertEqual(result["state"], "completed", result)
+            calls = list(self.docker.calls)
+            self.assertEqual(self.agent.execute(operation)["state"], "completed")
+            self.assertEqual(self.docker.calls, calls)
+        self.assertTrue((self.library / "content/game.ff").exists())
+
+    def test_admin_override_restarts_without_player_status(self):
+        self.assert_override_disruption("instance.restart")
+
+    def test_admin_override_stops_without_player_status(self):
+        self.assert_override_disruption("instance.stop")
+
+    def test_admin_override_removes_without_player_status(self):
+        self.assert_override_disruption("instance.remove")
+
+    def test_admin_override_rebuilds_without_player_status(self):
+        self.assert_override_disruption("instance.rebuild")
+
+    def test_admin_override_requires_boolean_true(self):
+        self.create()
+        self.assertIn("instance.safety-override", self.agent.inventory()["capabilities"])
+        for index, value in enumerate((False, None, "true", 1, {})):
+            with self.subTest(value=value):
+                self.agent.rcon = lambda *_: {"ok": True, "players": 2}
+                result = self.agent.execute(self.operation("instance.restart", {"safetyOverride": value}, f"attempt-{index}"))
+                self.assertEqual(result["state"], "accepted", result)
+        self.assertFalse(any(call[:2] == ("container", "restart") for call in self.docker.calls))
+
+    def test_admin_override_preserves_ownership_and_generation_checks(self):
+        self.create()
+        self.docker.containers[self.agent.container_name(INSTANCE, 1)]["Config"]["Labels"] = {}
+        self.assertEqual(self.agent.execute(self.operation("instance.remove", {"safetyOverride": True}))["state"], "failed")
+        self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls))
+        self.assertEqual(self.agent.execute(self.operation("instance.stop", {"safetyOverride": True}, "stale", generation=0))["state"], "failed")
+
+    def test_admin_override_activation_keeps_file_validation(self):
+        self.create()
+        target = self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg"
+        staged = {"stageId": "campaign", "files": [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"new").decode(), "checkBefore": True, "expectedBeforeSha": module.digest(b"original")}]}
+        self.agent.execute(self.operation("files.stage", staged, "stage"))
+        self.agent.rcon = lambda *_: {"ok": True, "players": 3}
+        target.write_bytes(b"changed")
+        failed = self.agent.execute(self.operation("instance.restart", {"stageId": "campaign", "safetyOverride": True}, "changed"))
+        self.assertEqual(failed["state"], "failed", failed)
+        self.assertFalse(any(call[:2] == ("container", "restart") for call in self.docker.calls))
+        target.write_bytes(b"original")
+        result = self.agent.execute(self.operation("instance.restart", {"stageId": "campaign", "safetyOverride": True}, "confirmed"))
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(target.read_bytes(), b"new")
+
     def test_files_stage_is_nondisruptive_activation_snapshots_and_rollback(self):
         self.create()
         target = self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg"