Require revocable authorization for automatic lobby removal

agent/ec_agent.py

@@ -33,7 +33,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.6.0"
+VERSION = "1.7.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -71,6 +71,10 @@ class Rejected(AgentError):
     """A definitive precondition failure, before any destructive action."""
 
 
+class AutoscalingInvalidated(Rejected):
+    """This removal needs a new policy decision, never a later local retry."""
+
+
 def systemctl(*args):
     try:
         result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
@@ -1356,7 +1360,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1581,6 +1585,24 @@ class Agent:
             raise AgentError("Staged file checksum mismatch")
         return content
 
+    def authorize_autoscaling_removal(self, instance_id, generation, payload):
+        authorization = payload.get("autoscaling")
+        if not isinstance(authorization, dict) or not self.executing_id or payload.get("safetyOverride") or payload.get("stageId"):
+            raise AutoscalingInvalidated("Invalid autoscaling removal authorization")
+        started = time.monotonic()
+        try:
+            response = self.request("POST", "/api/agent/operations/" + urllib.parse.quote(self.executing_id, safe="") + "/authorize-removal", authorization)
+        except AgentError:
+            raise AutoscalingInvalidated("Autoscaling removal authorization unavailable; request a new decision") from None
+        ttl = response.get("validForMs") if isinstance(response, dict) else None
+        if not isinstance(response, dict) or response.get("authorized") is not True or isinstance(ttl, bool) or not isinstance(ttl, (int, float)) or not 0 < ttl <= 5000:
+            raise AutoscalingInvalidated("Autoscaling removal authorization was withdrawn or expired")
+        # Occupancy can change during the controller round trip. Check again
+        # within the short claim window, immediately before the durable checkpoint.
+        self.safe_to_disrupt(instance_id, generation)
+        if (time.monotonic() - started) * 1000 >= ttl:
+            raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
+
     def lifecycle(self, kind, instance_id, generation, payload):
         if kind == "instance.cancel-create":
             return self.cancel_creation(instance_id, generation, payload)
@@ -1610,6 +1632,8 @@ class Agent:
             # The authenticated controller records explicit administrator approval
             # for this immutable operation. Ownership and generation checks still apply.
             override_safety = payload.get("safetyOverride") is True
+            if "autoscaling" in payload and (kind != "instance.remove" or override_safety):
+                raise AutoscalingInvalidated("Autoscaling only permits removal with player safety checks")
             if kind == "instance.rebuild":
                 next_generation = payload.get("nextGeneration", payload.get("newGeneration"))
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
@@ -1618,6 +1642,8 @@ class Agent:
                 self.validate_spec(spec)
             if not override_safety:
                 self.safe_to_disrupt(instance_id, generation)
+            if "autoscaling" in payload:
+                self.authorize_autoscaling_removal(instance_id, generation, payload)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
             if payload.get("stageId") and kind == "instance.restart":
                 before = self.activate(instance_id, generation, payload["stageId"], override_safety=override_safety)
@@ -2203,7 +2229,11 @@ class Agent:
         try:
             result = self.dispatch(operation)
         except Held as err:
+            if operation.get("kind") == "instance.remove" and "autoscaling" in operation.get("payload", {}):
+                return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error=str(err))
             return self.journal.update(operation["id"], "accepted", {"blocked": "occupied" if "Players" in str(err) else "unknown", "message": str(err)})
+        except AutoscalingInvalidated as err:
+            return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error=str(err))
         except Exception as err:
             # The command may already have changed Docker or files. Never infer
             # failure means nothing happened, and never retry without inspection.
@@ -2239,6 +2269,8 @@ class Agent:
         instance_id, generation = operation.get("instanceId"), operation.get("generation")
         checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
         try:
+            if kind == "instance.remove" and "autoscaling" in payload and not checkpoint:
+                return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision")
             if kind == "instance.cancel-create":
                 return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload))
             if kind == "instance.create":