Automate host preparation and speed up game file downloads

agent/ec_agent.py

@@ -7,6 +7,7 @@ unknown and is reported for reconciliation, never automatically executed twice.
 
 import argparse
 import base64
+from concurrent.futures import ThreadPoolExecutor, as_completed
 import contextlib
 import hashlib
 import fcntl
@@ -29,9 +30,10 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.2.0"
+VERSION = "1.2.1"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
+LIBRARY_DOWNLOAD_WORKERS = 4
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
 AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
 AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
@@ -788,13 +790,46 @@ class Agent:
             pass
         libraries, caches = [], []
         for marker in (self.root / "libraries").glob("*/*/manifest.json"):
+            # A crash between writing a manifest and its atomic promotion must
+            # never expose the staging directory as a ready library.
+            if marker.parent.name.startswith("."):
+                continue
             try:
                 value = json.loads(marker.read_text())
                 libraries.append({"id": value["libraryId"], "state": "ready"})
                 caches.append({"libraryVersionId": value["libraryId"], "status": "ready", "bytesDownloaded": value["size"], "bytesTotal": value["size"], "manifestSha256": value.get("manifestSha256")})
             except (OSError, ValueError, KeyError):
                 continue
-        progress = list(self.library_progress.values())
+        ready = {item["id"] for item in libraries}
+        progress = {key: value.copy() for key, value in list(self.library_progress.items()) if key not in ready}
+        for marker in (self.root / "libraries").glob("*/.*.partial/source.json"):
+            try:
+                value = json.loads(marker.read_text())
+                library_id = identifier(value["libraryId"])
+                if library_id in ready or library_id in progress:
+                    continue
+                expected = self.library_dir(value["game"], library_id).with_name("." + library_id + ".partial")
+                if marker.parent != expected or marker.is_symlink() or marker.parent.is_symlink():
+                    continue
+                files = value["files"]
+                canonical = json.dumps([{ "path": item["path"], "size": item["size"], "sha256": item["sha256"] } for item in files], separators=(",", ":"), ensure_ascii=False)
+                if digest(canonical.encode("utf-8")) != value["manifestSha256"]:
+                    continue
+                retained, size, paths = 0, 0, set()
+                for item in files:
+                    if item["path"] in paths or type(item["size"]) is not int or item["size"] < 0:
+                        raise AgentError("Invalid partial library manifest")
+                    paths.add(item["path"])
+                    target = scoped(marker.parent / "content", item["path"])
+                    size += item["size"]
+                    if target.exists() and stat_module.S_ISREG(target.stat().st_mode):
+                        retained += min(target.stat().st_size, item["size"])
+                # These are resumable bytes, not verified content. Reporting
+                # them survives restarts without requiring space for two copies.
+                progress[library_id] = {"libraryVersionId": library_id, "status": "missing", "bytesDownloaded": retained, "bytesTotal": size, "manifestSha256": value["manifestSha256"]}
+            except (AgentError, OSError, ValueError, KeyError, TypeError):
+                continue
+        progress = list(progress.values())
         libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress)
         caches.extend(progress)
         images = []
@@ -1195,6 +1230,65 @@ class Agent:
                 raise AgentError("Artifact redirects are not allowed")
         return urllib.request.build_opener(NoRedirect()).open(req, timeout=60)
 
+    def download_library_file(self, item, target, files_url, update_progress, aborted):
+        def check_active():
+            if aborted.is_set():
+                raise AgentError("Library download stopped after a transfer failed")
+
+        check_active()
+        target.parent.mkdir(parents=True, exist_ok=True)
+        offset = target.stat().st_size if target.exists() else 0
+        if offset > item["size"]:
+            target.unlink()
+            offset = 0
+        checksum = hashlib.sha256()
+        # Hash the saved prefix once so resumed transfers still verify the whole
+        # file. New bytes are hashed as they arrive, avoiding a second disk read.
+        if offset:
+            with target.open("rb") as stream:
+                for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+                    check_active()
+                    checksum.update(chunk)
+        if offset == item["size"] and target.exists():
+            if checksum.hexdigest() == item["sha256"]:
+                update_progress(item["path"], offset)
+                return
+            target.unlink()
+            offset = 0
+            checksum = hashlib.sha256()
+        update_progress(item["path"], offset)
+        check_active()
+        url = files_url.rstrip("/") + "/" + urllib.parse.quote(item["path"], safe="/")
+        with self.artifact_open("GET", url, extra_headers={"Range": f"bytes={offset}-"} if offset else {}) as response:
+            if response.status == 206:
+                expected = f"bytes {offset}-{item['size'] - 1}/{item['size']}"
+                if response.headers.get("Content-Range") != expected:
+                    raise AgentError("Invalid resumed download response")
+            elif response.status != 200:
+                raise AgentError("Invalid library download response")
+            elif offset:
+                # A server may ignore Range and send the whole file instead.
+                offset = 0
+                checksum = hashlib.sha256()
+                update_progress(item["path"], 0)
+            with target.open("ab" if offset else "wb") as output:
+                received = offset
+                while True:
+                    check_active()
+                    chunk = response.read(1024 * 1024)
+                    if not chunk:
+                        break
+                    received += len(chunk)
+                    if received > item["size"]:
+                        raise AgentError("Library download exceeded its manifest size")
+                    output.write(chunk)
+                    checksum.update(chunk)
+                    update_progress(item["path"], received)
+                output.flush()
+                os.fsync(output.fileno())
+        if target.stat().st_size != item["size"] or checksum.hexdigest() != item["sha256"]:
+            raise AgentError("Downloaded library file failed integrity verification")
+
     def library_download(self, payload):
         library_id = identifier(payload.get("libraryVersionId", payload.get("libraryId")))
         with self.artifact_open("GET", payload["manifestUrl"]) as response:
@@ -1221,55 +1315,46 @@ class Agent:
         source_marker = staging / "source.json"
         if source_marker.exists() and json.loads(source_marker.read_text()).get("manifestSha256") != manifest_sha:
             raise Rejected("Partial library download belongs to a different manifest")
-        atomic_json(source_marker, {"libraryId": library_id, "manifestSha256": manifest_sha})
         size = sum(item["size"] for item in files)
-        downloaded = 0
-        paths = set()
+        atomic_json(source_marker, {"libraryId": library_id, "game": manifest["game"], "size": size, "files": files, "manifestSha256": manifest_sha})
+        targets = {}
         progress = {"libraryVersionId": library_id, "status": "downloading", "bytesDownloaded": 0, "bytesTotal": size, "manifestSha256": manifest_sha}
         self.library_progress[library_id] = progress
         try:
             if shutil.disk_usage(staging).free < max(0, size - sum(path.stat().st_size for path in staging.rglob("*") if path.is_file())):
                 raise AgentError("Insufficient library storage")
+            # Validate every path before starting concurrent writes. In addition
+            # to duplicates, a manifest cannot treat a file as a parent folder.
             for item in files:
-                if item["path"] in paths or not isinstance(item["size"], int) or item["size"] < 0 or not re.fullmatch("[0-9a-f]{64}", item["sha256"]):
+                if item["path"] in targets or not isinstance(item["size"], int) or item["size"] < 0 or not re.fullmatch("[0-9a-f]{64}", item["sha256"]):
                     raise AgentError("Invalid library file manifest")
-                paths.add(item["path"])
-                target = scoped(staging / "content", item["path"])
-                target.parent.mkdir(parents=True, exist_ok=True)
-                offset = target.stat().st_size if target.exists() else 0
-                if offset > item["size"]:
-                    target.unlink()
-                    offset = 0
-                if offset == item["size"] and target.exists() and file_digest(target) != item["sha256"]:
-                    target.unlink()
-                    offset = 0
-                if offset < item["size"] or not target.exists():
-                    url = payload["filesBaseUrl"].rstrip("/") + "/" + urllib.parse.quote(item["path"], safe="/")
-                    with self.artifact_open("GET", url, extra_headers={"Range": f"bytes={offset}-"} if offset else {}) as response:
-                        if offset and response.status == 206:
-                            content_range = response.headers.get("Content-Range", "")
-                            if not content_range.startswith(f"bytes {offset}-"):
-                                raise AgentError("Invalid resumed download response")
-                        elif offset:
-                            offset = 0
-                        with target.open("ab" if offset else "wb") as output:
-                            received = offset
-                            while True:
-                                chunk = response.read(1024 * 1024)
-                                if not chunk:
-                                    break
-                                received += len(chunk)
-                                if received > item["size"]:
-                                    raise AgentError("Library download exceeded its manifest size")
-                                output.write(chunk)
-                                progress["bytesDownloaded"] = downloaded + received
-                            output.flush()
-                            os.fsync(output.fileno())
-                progress["status"] = "verifying"
-                if target.stat().st_size != item["size"] or file_digest(target) != item["sha256"]:
-                    raise AgentError("Downloaded library file failed integrity verification")
-                downloaded += item["size"]
-                progress.update(status="downloading", bytesDownloaded=downloaded)
+                targets[item["path"]] = scoped(staging / "content", item["path"])
+            for path in targets:
+                if any(str(parent) in targets for parent in PurePosixPath(path).parents):
+                    raise AgentError("Library file paths overlap")
+            progress_lock = threading.Lock()
+            offsets = {}
+            aborted = threading.Event()
+
+            def update_progress(path, offset):
+                with progress_lock:
+                    progress["bytesDownloaded"] += offset - offsets.get(path, 0)
+                    offsets[path] = offset
+
+            def transfer(item):
+                try:
+                    self.download_library_file(item, targets[item["path"]], payload["filesBaseUrl"], update_progress, aborted)
+                except Exception:
+                    aborted.set()
+                    raise
+
+            # Bound sockets, memory and disk pressure while allowing other files
+            # to progress during network round trips and per-file fsyncs.
+            with ThreadPoolExecutor(max_workers=min(LIBRARY_DOWNLOAD_WORKERS, len(files))) as pool:
+                futures = [pool.submit(transfer, item) for item in files]
+                for future in as_completed(futures):
+                    future.result()
+            progress["status"] = "verifying"
             saved = {"libraryId": library_id, "game": manifest["game"], "size": size, "files": files, "manifestSha256": manifest_sha}
             atomic_json(staging / "manifest.json", saved)
             for path in (staging / "content").rglob("*"):
@@ -1446,6 +1531,16 @@ class Agent:
                 if not self.config.get("pendingCredential") or self.recover_credential():
                     return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True})
                 return row
+            if kind == "runtime.pull":
+                # Docker resumes already downloaded layers. Pull again under
+                # the same operation ID: an existing tag can still refer to an
+                # older image and cannot prove an interrupted pull completed.
+                self.journal.update(operation["id"], "running")
+                try:
+                    result = self.dispatch(operation)
+                    return self.journal.update(operation["id"], "completed", {**result, "reconciled": True})
+                except Exception:
+                    return self.journal.update(operation["id"], "failed", error="Interrupted runtime download could not be resumed")
             if kind == "library.import":
                 try:
                     with self.artifact_open("GET", payload["completeUrl"]) as response:

agent/tests/test_agent.py

@@ -4,6 +4,7 @@ import io
 import json
 from pathlib import Path
 import tempfile
+import threading
 import unittest
 from unittest.mock import patch
 
@@ -107,6 +108,40 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(result["state"], "unknown")
         self.assertEqual(self.docker.calls, [])
 
+    def test_agent_restart_resumes_runtime_pull_even_when_an_older_tag_is_cached(self):
+        image = "ghcr.io/ayymoss/plutainer:latest"
+        operation = self.operation("runtime.pull", {"image": image})
+        self.agent.journal.accept(operation)
+        self.agent.journal.update(operation["id"], "running")
+        aliases = self.agent.root / "agent/runtime-images.json"
+        module.atomic_json(aliases, {image: "sha256:older-image"})
+        self.agent.journal.close()
+        self.agent = Agent(self.config, self.docker)
+        original = self.docker.call
+
+        def call(*args, **kwargs):
+            result = original(*args, **kwargs)
+            return "sha256:new-image" if args[:2] == ("image", "inspect") else result
+
+        self.docker.call = call
+        result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed")
+        self.assertEqual(json.loads(result["result"]), {"image": image, "ready": True, "reconciled": True})
+        self.assertEqual(json.loads(aliases.read_text())[image], "sha256:new-image")
+        self.agent.execute(operation)
+        self.assertEqual(sum(args[:2] == ("image", "pull") for args in self.docker.calls), 1)
+
+    def test_interrupted_runtime_pull_failure_is_reported_for_automatic_retry(self):
+        operation = self.operation("runtime.pull", {"image": "ghcr.io/ayymoss/plutainer:latest"})
+        self.agent.journal.accept(operation)
+        self.agent.journal.update(operation["id"], "running")
+        self.agent.journal.close()
+        self.agent = Agent(self.config, self.docker)
+        with patch.object(self.docker, "call", side_effect=AgentError("Registry unavailable")):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "failed")
+        self.assertIn("runtime download", result["error"])
+
     def test_occupied_and_unknown_wait_then_revalidate_same_operation(self):
         self.create()
         operation = self.operation("instance.restart")
@@ -198,8 +233,192 @@ class AgentTests(unittest.TestCase):
                 module.scoped(app, path)
 
     def manifest(self, content=b"complete-game-file"):
-        files = [{"path": "zone/test.ff", "size": len(content), "sha256": module.digest(content)}]
-        return {"id": "downloaded", "game": "t6", "files": files, "sizeBytes": len(content), "manifestSha256": module.digest(json.dumps(files, separators=(",", ":")).encode())}
+        return self.manifest_files({"zone/test.ff": content})
+
+    def manifest_files(self, contents):
+        files = [{"path": path, "size": len(content), "sha256": module.digest(content)} for path, content in contents.items()]
+        return {"id": "downloaded", "game": "t6", "files": files, "sizeBytes": sum(item["size"] for item in files), "manifestSha256": module.digest(json.dumps(files, separators=(",", ":")).encode())}
+
+    def test_downloads_use_bounded_parallel_transfers_and_aggregate_progress(self):
+        contents = {f"zone/file-{index}.ff": bytes([index]) * 31 for index in range(8)}
+        manifest = self.manifest_files(contents)
+        barrier = threading.Barrier(4)
+        lock = threading.Lock()
+        active = maximum = 0
+        snapshots = []
+        agent = self.agent
+
+        class ConcurrentResponse(Response):
+            def __exit__(self, *args):
+                nonlocal active
+                with lock:
+                    active -= 1
+                    snapshots.append(agent.library_progress["downloaded"]["bytesDownloaded"])
+                return super().__exit__(*args)
+
+        def fetch(method, path, data=None, extra_headers=None):
+            nonlocal active, maximum
+            if path == "/manifest":
+                return Response(json.dumps(manifest).encode())
+            with lock:
+                active += 1
+                maximum = max(maximum, active)
+            # A serial downloader cannot reach this barrier. Two batches also
+            # prove that an unlimited worker pool cannot open all eight files.
+            barrier.wait(timeout=5)
+            return ConcurrentResponse(contents[path.removeprefix("/files/")])
+
+        self.agent.artifact_open = fetch
+        result = self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
+        self.assertEqual(maximum, 4)
+        self.assertEqual(snapshots, sorted(snapshots))
+        self.assertEqual(max(snapshots), manifest["sizeBytes"])
+        self.assertEqual(result["size"], manifest["sizeBytes"])
+        for path, content in contents.items():
+            self.assertEqual((self.agent.library_dir("t6", "downloaded") / "content" / path).read_bytes(), content)
+
+    def test_fresh_download_hashes_stream_without_rereading_whole_file(self):
+        content = b"game-file" * 200000
+        manifest = self.manifest(content)
+        self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode() if path == "/manifest" else content)
+        with patch.object(module, "file_digest", side_effect=AssertionError("download reread the file")):
+            result = self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
+        self.assertEqual(result["state"], "ready")
+
+    def test_ignored_range_restarts_file_and_checksum(self):
+        content = b"complete-game-file"
+        manifest = self.manifest(content)
+        staging = self.agent.root / "libraries/t6/.downloaded.partial/content/zone"
+        staging.mkdir(parents=True)
+        (staging / "test.ff").write_bytes(b"bad-prefix")
+
+        def fetch(method, path, data=None, extra_headers=None):
+            if path == "/manifest":
+                return Response(json.dumps(manifest).encode())
+            self.assertEqual(extra_headers, {"Range": "bytes=10-"})
+            return Response(content)
+
+        self.agent.artifact_open = fetch
+        result = self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
+        self.assertEqual(result["state"], "ready")
+        self.assertEqual((self.agent.library_dir("t6", "downloaded") / "content/zone/test.ff").read_bytes(), content)
+
+    def test_resumed_download_verifies_saved_prefix(self):
+        content = b"complete-game-file"
+        manifest = self.manifest(content)
+        staging = self.agent.root / "libraries/t6/.downloaded.partial/content/zone"
+        staging.mkdir(parents=True)
+        (staging / "test.ff").write_bytes(b"corrupt!")
+        self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode()) if path == "/manifest" else Response(content[8:], 206, {"Content-Range": f"bytes 8-{len(content)-1}/{len(content)}"})
+        with self.assertRaisesRegex(AgentError, "integrity"):
+            self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
+        self.assertFalse(self.agent.library_dir("t6", "downloaded").exists())
+
+    def test_parallel_failure_retains_partial_files_for_verified_resume(self):
+        contents = {"zone/first.ff": b"first-game-file", "zone/second.ff": b"second-game-file"}
+        manifest = self.manifest_files(contents)
+        payload = {"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}
+
+        def failed_fetch(method, path, data=None, extra_headers=None):
+            if path == "/manifest":
+                return Response(json.dumps(manifest).encode())
+            return Response(contents[path.removeprefix("/files/")][:5])
+
+        self.agent.artifact_open = failed_fetch
+        with self.assertRaises(AgentError):
+            self.agent.library_download(payload)
+        self.assertFalse(self.agent.library_dir("t6", "downloaded").exists())
+        self.assertEqual(self.agent.library_progress["downloaded"]["status"], "failed")
+        resumed = []
+
+        def resume_fetch(method, path, data=None, extra_headers=None):
+            if path == "/manifest":
+                return Response(json.dumps(manifest).encode())
+            content = contents[path.removeprefix("/files/")]
+            if extra_headers:
+                self.assertEqual(extra_headers, {"Range": "bytes=5-"})
+                resumed.append(path)
+                return Response(content[5:], 206, {"Content-Range": f"bytes 5-{len(content)-1}/{len(content)}"})
+            return Response(content)
+
+        self.agent.artifact_open = resume_fetch
+        self.assertEqual(self.agent.library_download(payload)["state"], "ready")
+        self.assertTrue(resumed)
+        self.assertEqual(self.agent.verify_library(self.agent.library_dir("t6", "downloaded"))["size"], manifest["sizeBytes"])
+
+    def test_overlapping_paths_rejected_before_concurrent_downloads(self):
+        manifest = self.manifest_files({"zone": b"file", "zone/child.ff": b"child"})
+        calls = []
+
+        def fetch(method, path, data=None, extra_headers=None):
+            calls.append(path)
+            return Response(json.dumps(manifest).encode())
+
+        self.agent.artifact_open = fetch
+        with self.assertRaisesRegex(AgentError, "overlap"):
+            self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
+        self.assertEqual(calls, ["/manifest"])
+
+    def test_failed_transfer_restart_reports_retained_bytes_and_resumes_with_only_remaining_space(self):
+        content = b"complete-game-file"
+        manifest = self.manifest(content)
+        payload = {"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}
+        self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode() if path == "/manifest" else content[:8])
+        with self.assertRaises(AgentError):
+            self.agent.library_download(payload)
+        self.agent.journal.close()
+        self.agent = Agent(self.config, self.docker)
+        cache = next(item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == "downloaded")
+        self.assertEqual(cache, {"libraryVersionId": "downloaded", "status": "missing", "bytesDownloaded": 8, "bytesTotal": len(content), "manifestSha256": manifest["manifestSha256"]})
+
+        def fetch(method, path, data=None, extra_headers=None):
+            if path == "/manifest":
+                return Response(json.dumps(manifest).encode())
+            self.assertEqual(extra_headers, {"Range": "bytes=8-"})
+            return Response(content[8:], 206, {"Content-Range": f"bytes 8-{len(content)-1}/{len(content)}"})
+
+        self.agent.artifact_open = fetch
+        usage = module.shutil.disk_usage(self.agent.root)
+        with patch.object(module.shutil, "disk_usage", return_value=type(usage)(usage.total, usage.used, len(content) - 8)):
+            self.assertEqual(self.agent.library_download(payload)["state"], "ready")
+
+    def test_unpromoted_manifest_is_never_reported_as_ready_and_progress_is_not_duplicated(self):
+        content = b"complete-game-file"
+        manifest = self.manifest(content)
+        staging = self.agent.root / "libraries/t6/.downloaded.partial"
+        (staging / "content/zone").mkdir(parents=True)
+        # Oversized retained files can be replaced during resume. Their progress
+        # must still stay within the published file size.
+        (staging / "content/zone/test.ff").write_bytes(content + b"extra")
+        saved = {"libraryId": "downloaded", "game": "t6", "size": len(content), "files": manifest["files"], "manifestSha256": manifest["manifestSha256"]}
+        module.atomic_json(staging / "source.json", saved)
+        module.atomic_json(staging / "manifest.json", saved)
+        inventory = self.agent.inventory()
+        caches = [item for item in inventory["libraryCaches"] if item["libraryVersionId"] == "downloaded"]
+        self.assertEqual(len(caches), 1)
+        self.assertEqual(caches[0]["status"], "missing")
+        self.assertEqual(caches[0]["bytesDownloaded"], len(content))
+        self.assertNotIn({"id": "downloaded", "state": "ready"}, inventory["libraries"])
+        self.agent.library_progress["downloaded"] = {**caches[0], "status": "downloading", "bytesDownloaded": 3}
+        caches = [item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == "downloaded"]
+        self.assertEqual(len(caches), 1)
+        self.assertEqual(caches[0]["status"], "downloading")
+        self.assertEqual(caches[0]["bytesDownloaded"], 3)
+
+    def test_partial_inventory_rejects_changed_manifest_and_symlinked_content(self):
+        content = b"complete-game-file"
+        manifest = self.manifest(content)
+        staging = self.agent.root / "libraries/t6/.downloaded.partial"
+        (staging / "content/zone").mkdir(parents=True)
+        source = {"libraryId": "downloaded", "game": "t6", "files": manifest["files"], "manifestSha256": "0" * 64}
+        module.atomic_json(staging / "source.json", source)
+        (staging / "content/zone/test.ff").write_bytes(content)
+        self.assertFalse(any(item["libraryVersionId"] == "downloaded" for item in self.agent.inventory()["libraryCaches"]))
+        source["manifestSha256"] = manifest["manifestSha256"]
+        module.atomic_json(staging / "source.json", source)
+        (staging / "content/zone/test.ff").unlink()
+        (staging / "content/zone/test.ff").symlink_to(self.library / "content/game.ff")
+        self.assertFalse(any(item["libraryVersionId"] == "downloaded" for item in self.agent.inventory()["libraryCaches"]))
 
     def test_resume_verifies_and_atomically_publishes_library(self):
         content = b"complete-game-file"