Reconcile rebuilds interrupted before replacement creation

static/agent/ec-agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.8.3"
+VERSION = "1.8.4"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -2665,12 +2665,84 @@ class Agent:
         atomic_json(self.config_path, self.config)
         return True
 
+    def reconcile_uncreated_rebuild(self, operation, row):
+        """Settle removal only when the replacement create was never submitted.
+
+        The write lane and instance lock exclude the original Python execution.
+        A terminal journal receipt prevents any later replay of that command.
+        """
+        if row["state"] != "unknown":
+            return row
+        instance_id, generation = instance_identity(operation.get("instanceId"), operation.get("generation"))
+        payload = operation.get("payload", {})
+        next_generation = payload.get("nextGeneration", payload.get("newGeneration"))
+        if type(next_generation) is not int or next_generation <= generation:
+            return row
+        previous = json.loads(row["result"]) if row.get("result") else {}
+        if not isinstance(previous, dict):
+            return row
+        directory = scoped(self.root, f"instances/{instance_id}/{next_generation}")
+
+        def unchanged_generation():
+            try:
+                marker = scoped(self.root, f"instances/{instance_id}/current.json")
+                current = json.loads(marker.read_text())
+            except FileNotFoundError:
+                return True
+            return isinstance(current, dict) and type(current.get("generation")) is int and current["generation"] == generation and current.get("removed", False) is False
+
+        def both_missing():
+            for target in (generation, next_generation):
+                health = self.inspect(instance_id, target)
+                if health.get("exists") is not False or health.get("running") is not False or health.get("state") != "missing":
+                    return False
+            return True
+
+        if not unchanged_generation() or not both_missing():
+            return row
+        # Preparation can leave runtime/configuration files before create's
+        # checkpoint. A fresh operation must not inherit those partial files.
+        expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
+        try:
+            saved_spec = json.loads(scoped(directory, "spec.json").read_text())
+        except FileNotFoundError:
+            pass
+        else:
+            if saved_spec != expected_spec:
+                return row
+        try:
+            metadata = directory.lstat()
+        except FileNotFoundError:
+            pass
+        else:
+            if not stat_module.S_ISDIR(metadata.st_mode):
+                return row
+            shutil.rmtree(directory)
+        # Also sync when a previous reconciliation removed the directory but
+        # crashed before recording the terminal receipt.
+        try:
+            sync_parent(directory)
+        except FileNotFoundError:
+            pass
+        if not unchanged_generation() or not both_missing():
+            return row
+        original_error = row.get("error")
+        result = {**previous, "reconciled": True, "originalError": original_error,
+                  "rebuildFailure": {"phase": "disrupting", "generation": generation, "nextGeneration": next_generation,
+                                     "oldContainerMissing": True, "replacementContainerMissing": True, "replacementFilesRemoved": True}}
+        message = "Old container removed; replacement creation failed."
+        if original_error:
+            message += " " + original_error
+        return self.journal.update(operation["id"], "failed", result, error=message)
+
     def reconcile(self, operation, row):
         """Prove desired state with immutable ownership; never resubmit a command."""
         kind, payload = operation["kind"], operation.get("payload", {})
         instance_id, generation = operation.get("instanceId"), operation.get("generation")
         checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
         try:
+            if kind == "instance.rebuild" and checkpoint.get("phase") == "disrupting":
+                return self.reconcile_uncreated_rebuild(operation, row)
             if kind == "instance.remove" and "autoscaling" in payload and not checkpoint:
                 return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision")
             if kind == "instance.cancel-create":

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.8.3",
+  "version": "1.8.4",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "388569ffd4566dee63584970838d4d3ae7d2b8ef92675ae7627c31389dcbfec0",
+  "sha256": "7e7c0049161a6f64743210b787a8ec30bd32d726c30587962bc5c705d94dfbac",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }