Rebuild missing containers during maintenance recovery

agent/ec_agent.py

@@ -33,7 +33,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.8.1"
+VERSION = "1.8.2"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1913,7 +1913,9 @@ class Agent:
                     self.docker.call("container", "rm", name)
                 if kind == "instance.rebuild":
                     result = self.create(instance_id, next_generation, spec)
-                    shutil.rmtree(self.instance_dir(instance_id, generation), ignore_errors=False)
+                    old = self.instance_dir(instance_id, generation)
+                    if old.exists():
+                        shutil.rmtree(old, ignore_errors=False)
                     return result
                 directory = self.instance_dir(instance_id, generation)
                 if directory.exists():

agent/tests/test_agent.py

@@ -1444,6 +1444,44 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(self.agent.execute(self.operation("instance.stop", op_id="old-stop"))["state"], "failed")
         self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
 
+    def assert_missing_instance_rebuild(self, remove_directory=False):
+        payload = self.create()
+        self.docker.containers.pop(self.agent.container_name(INSTANCE, 1))
+        old = self.agent.instance_dir(INSTANCE, 1)
+        if remove_directory:
+            module.shutil.rmtree(old)
+        operation = self.operation("instance.rebuild", {"spec": payload, "nextGeneration": 2})
+        with patch.object(self.agent, "rcon", side_effect=AssertionError("Missing container queried player status")):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(json.loads(result["result"])["generation"], 2)
+        self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
+        self.assertFalse(old.exists())
+        self.assertEqual(json.loads((old.parent / "current.json").read_text()), {"generation": 2})
+        self.assertFalse(any(call[:2] in (("container", "stop"), ("container", "rm")) for call in self.docker.calls))
+        calls = list(self.docker.calls)
+        self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertEqual(self.docker.calls, calls)
+
+    def test_rebuild_recreates_missing_container(self):
+        self.assert_missing_instance_rebuild()
+
+    def test_rebuild_completes_when_old_instance_directory_is_also_missing(self):
+        self.assert_missing_instance_rebuild(remove_directory=True)
+
+    def test_rebuild_keeps_real_old_directory_cleanup_failures_unresolved(self):
+        payload = self.create()
+        self.docker.containers.pop(self.agent.container_name(INSTANCE, 1))
+        old = self.agent.instance_dir(INSTANCE, 1)
+        operation = self.operation("instance.rebuild", {"spec": payload, "nextGeneration": 2})
+        with patch.object(module.shutil, "rmtree", side_effect=PermissionError("Cleanup denied")):
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+            self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
+            self.assertTrue(old.exists())
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+        self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertFalse(old.exists())
+
     def test_traversal_and_symlinks_are_rejected(self):
         self.create()
         app = self.agent.instance_dir(INSTANCE, 1) / "app"