Cache agent inventory manifests by modification time

agent/ec_agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.10"
+VERSION = "1.4.11"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -918,6 +918,8 @@ class Agent:
         self.observation_keep_days = 0  # Older controllers do not authorize cleanup.
         self.library_progress = {}
         self.library_transfers = {}
+        self.inventory_manifest_cache = {}
+        self.inventory_manifest_lock = threading.Lock()
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
         self.executing_id = None
 
@@ -1191,6 +1193,25 @@ class Agent:
                 continue
         return samples
 
+    def inventory_manifest(self, marker):
+        # Inventory only: launch/download verification must still read disk.
+        # Include file identity so atomic replacements invalidate even when
+        # their size and modification time match the previous manifest.
+        with self.inventory_manifest_lock:
+            try:
+                metadata = marker.stat()
+                signature = (metadata.st_mtime_ns, metadata.st_size, metadata.st_dev, metadata.st_ino)
+                cached = self.inventory_manifest_cache.get(marker)
+                if cached is not None and cached[0] == signature:
+                    return cached[1]
+                self.inventory_manifest_cache.pop(marker, None)
+                value = json.loads(marker.read_text())
+                self.inventory_manifest_cache[marker] = (signature, value)
+                return value
+            except (OSError, ValueError):
+                self.inventory_manifest_cache.pop(marker, None)
+                raise
+
     def inventory(self):
         cpu_usage = self.cpu_usage_percent()
         usage = shutil.disk_usage(self.root)
@@ -1210,13 +1231,18 @@ class Agent:
         libraries, caches = [], []
         progress = {key: value.copy() for key, value in list(self.library_progress.items())}
         progress.update({key: transfer.snapshot() for key, transfer in list(self.library_transfers.items())})
-        for marker in (self.root / "libraries").glob("*/*/manifest.json"):
+        manifests = list((self.root / "libraries").glob("*/*/manifest.json"))
+        partial_manifests = list((self.root / "libraries").glob("*/.*.partial/source.json"))
+        with self.inventory_manifest_lock:
+            for marker in self.inventory_manifest_cache.keys() - set(manifests + partial_manifests):
+                del self.inventory_manifest_cache[marker]
+        for marker in manifests:
             # A crash between writing a manifest and its atomic promotion must
             # never expose the staging directory as a ready library.
             if marker.parent.name.startswith("."):
                 continue
             try:
-                value = json.loads(marker.read_text())
+                value = self.inventory_manifest(marker)
                 # Rechecking an existing cache must revoke readiness until its
                 # entire contents have passed verification again.
                 if value["libraryId"] not in progress and (marker.parent / "transfer.json").exists():
@@ -1233,9 +1259,9 @@ class Agent:
             except (OSError, ValueError, KeyError):
                 continue
         ready = {item["id"] for item in libraries}
-        for marker in (self.root / "libraries").glob("*/.*.partial/source.json"):
+        for marker in partial_manifests:
             try:
-                value = json.loads(marker.read_text())
+                value = self.inventory_manifest(marker)
                 library_id = identifier(value["libraryId"])
                 if library_id in ready or library_id in progress:
                     continue