josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Separate game file downloads and verification with progress timing

Browse Source
This commit is contained in:
josh
2026-10-04 12:51:05 -04:00
parent 68323c0baa
commit f8c60fd101
17 changed files with 1150 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/tests/test_agent.py
+266 -6
View File
@@ -277,13 +277,60 @@ class AgentTests(unittest.TestCase):
for path, content in contents.items():
self.assertEqual((self.agent.library_dir("t6", "downloaded") / "content" / path).read_bytes(), content)
def test_fresh_download_hashes_stream_without_rereading_whole_file(self):
content = b"game-file" * 200000
manifest = self.manifest(content)
self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode() if path == "/manifest" else content)
with patch.object(module, "file_digest", side_effect=AssertionError("download reread the file")):
def test_all_downloads_finish_before_any_file_is_hashed_and_verified_from_disk(self):
contents = {"zone/first.ff": b"game-file" * 200000, "zone/second.ff": b"second-file"}
manifest = self.manifest_files(contents)
self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode() if path == "/manifest" else contents[path.removeprefix("/files/")])
original_hash, original_open = module.hashlib.sha256, Path.open
reads = []
def hashing(*args, **kwargs):
progress = self.agent.library_progress.get("downloaded")
if progress:
self.assertEqual(progress["status"], "verifying")
self.assertEqual(progress["bytesDownloaded"], manifest["sizeBytes"])
return original_hash(*args, **kwargs)
def opening(path, mode="r", *args, **kwargs):
if mode == "rb" and path.name.endswith(".ff"):
self.assertFalse(self.agent.library_dir("t6", "downloaded").exists())
reads.append(path.name)
return original_open(path, mode, *args, **kwargs)
with patch.object(module.hashlib, "sha256", side_effect=hashing), patch.object(Path, "open", opening):
result = self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
self.assertEqual(result["state"], "ready")
self.assertCountEqual(reads, ["first.ff", "second.ff"])
cache = next(item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == "downloaded")
self.assertEqual(cache["bytesVerified"], manifest["sizeBytes"])
self.assertEqual(cache["transfer"]["phase"], "verifying")
def test_phase_boundaries_are_reported_even_between_heartbeats_and_status_errors_are_nonfatal(self):
content = b"game-file"
manifest = self.manifest(content)
reports = []
def fetch(method, path, data=None, extra_headers=None, timeout=60):
if path == "/status":
self.assertEqual(timeout, 5)
reports.append(json.loads(data))
raise AgentError("Controller temporarily unavailable")
self.assertEqual(timeout, 60)
return Response(json.dumps(manifest).encode() if path == "/manifest" else content)
self.agent.artifact_open = fetch
result = self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files", "statusUrl": "/status"})
self.assertEqual(result["state"], "ready")
self.assertEqual([item["status"] for item in reports], ["downloading", "verifying", "ready"])
self.assertEqual([item["bytesVerified"] for item in reports], [0, 0, len(content)])
self.assertEqual(reports[1]["bytesDownloaded"], len(content))
def test_status_socket_timeout_is_short_without_reducing_download_timeout(self):
with patch.object(module.urllib.request, "build_opener") as build_opener:
self.agent.artifact_open("GET", "/file")
self.assertEqual(build_opener.return_value.open.call_args.kwargs["timeout"], 60)
self.agent.artifact_open("POST", "/status", b"{}", timeout=5)
self.assertEqual(build_opener.return_value.open.call_args.kwargs["timeout"], 5)
def test_ignored_range_restarts_file_and_checksum(self):
content = b"complete-game-file"
@@ -313,6 +360,80 @@ class AgentTests(unittest.TestCase):
with self.assertRaisesRegex(AgentError, "integrity"):
self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
self.assertFalse(self.agent.library_dir("t6", "downloaded").exists())
self.assertFalse((staging / "test.ff").exists())
self.assertEqual(self.agent.library_progress["downloaded"]["bytesDownloaded"], 0)
requests = []
def fetch(method, path, data=None, extra_headers=None):
if path != "/manifest":
requests.append(extra_headers)
return Response(json.dumps(manifest).encode() if path == "/manifest" else content)
self.agent.artifact_open = fetch
self.assertEqual(self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})["state"], "ready")
self.assertEqual(requests, [{}])
def test_all_corrupt_complete_files_are_removed_so_retry_downloads_them_again(self):
contents = {"zone/first.ff": b"first", "zone/second.ff": b"second", "zone/good.ff": b"good"}
manifest = self.manifest_files(contents)
staging = self.agent.root / "libraries/t6/.downloaded.partial/content/zone"
staging.mkdir(parents=True)
for name, content in contents.items():
(staging / Path(name).name).write_bytes(content if name.endswith("good.ff") else b"x" * len(content))
requests = []
def fetch(method, path, data=None, extra_headers=None):
requests.append(path)
return Response(json.dumps(manifest).encode() if path == "/manifest" else contents[path.removeprefix("/files/")])
self.agent.artifact_open = fetch
payload = {"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}
with self.assertRaisesRegex(AgentError, "integrity"):
self.agent.library_download(payload)
self.assertEqual(requests, ["/manifest"])
self.assertEqual([path.name for path in staging.iterdir()], ["good.ff"])
self.assertEqual(self.agent.library_download(payload)["state"], "ready")
self.assertCountEqual(requests, ["/manifest", "/manifest", "/files/zone/first.ff", "/files/zone/second.ff"])
def test_verification_keeps_download_phase_complete_until_failure_reports_retained_bytes(self):
contents = {"zone/first.ff": b"first", "zone/second.ff": b"second"}
for mode in ("staged-corrupt", "existing-missing", "existing-short"):
with self.subTest(mode=mode):
manifest = {**self.manifest_files(contents), "id": mode}
directory = self.agent.library_dir("t6", mode)
if mode.startswith("existing"):
(directory / "content/zone").mkdir(parents=True)
(directory / "content/zone/second.ff").write_bytes(contents["zone/second.ff"])
if mode == "existing-short":
(directory / "content/zone/first.ff").write_bytes(b"f")
module.atomic_json(directory / "manifest.json", {"libraryId": mode, "game": "t6", "size": manifest["sizeBytes"], "files": manifest["files"], "manifestSha256": manifest["manifestSha256"]})
def fetch(method, path, **kwargs):
if path == "/manifest":
return Response(json.dumps(manifest).encode())
content = contents[path.removeprefix("/files/")]
return Response(b"x" * len(content) if path.endswith("first.ff") else content)
self.agent.artifact_open = fetch
original_open = Path.open
snapshots = []
def opening(path, open_mode="r", *args, **kwargs):
open_mode = kwargs.pop("mode", open_mode)
if open_mode == "rb" and path.name == "second.ff":
cache = next(item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == mode)
snapshots.append(cache)
self.assertEqual(cache["status"], "verifying")
self.assertEqual(cache["bytesDownloaded"], cache["bytesTotal"])
self.assertEqual(cache["transfer"]["phase"], "verifying")
return original_open(path, open_mode, *args, **kwargs)
with patch.object(Path, "open", opening), self.assertRaisesRegex(AgentError, "integrity"):
self.agent.library_download({"libraryVersionId": mode, "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
self.assertEqual(len(snapshots), 1)
failed = next(item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == mode)
self.assertEqual(failed["status"], "failed")
self.assertEqual(failed["bytesDownloaded"], len(contents["zone/second.ff"]) + (1 if mode == "existing-short" else 0))
def test_parallel_failure_retains_partial_files_for_verified_resume(self):
contents = {"zone/first.ff": b"first-game-file", "zone/second.ff": b"second-game-file"}
@@ -369,7 +490,13 @@ class AgentTests(unittest.TestCase):
self.agent.journal.close()
self.agent = Agent(self.config, self.docker)
cache = next(item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == "downloaded")
self.assertEqual(cache, {"libraryVersionId": "downloaded", "status": "missing", "bytesDownloaded": 8, "bytesTotal": len(content), "manifestSha256": manifest["manifestSha256"]})
self.assertEqual(cache["status"], "failed")
self.assertEqual(cache["bytesDownloaded"], 8)
self.assertEqual(cache["bytesVerified"], 0)
self.assertEqual(cache["bytesTotal"], len(content))
self.assertEqual(cache["manifestSha256"], manifest["manifestSha256"])
self.assertEqual(cache["transfer"]["phase"], "downloading")
self.assertIsNone(cache["transfer"]["etaSeconds"])
def fetch(method, path, data=None, extra_headers=None):
if path == "/manifest":
@@ -405,6 +532,139 @@ class AgentTests(unittest.TestCase):
self.assertEqual(caches[0]["status"], "downloading")
self.assertEqual(caches[0]["bytesDownloaded"], 3)
def test_interrupted_verification_restarts_from_disk_without_redownloading_and_keeps_elapsed(self):
content = b"complete-game-file"
manifest = self.manifest(content)
payload = {"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}
self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode() if path == "/manifest" else content)
def interrupted(files, targets, transfer, repair=False):
transfer.advance("bytesVerified", 5, 5)
transfer.persist(force=True)
raise KeyboardInterrupt("Simulated process termination")
with patch.object(self.agent, "verify_downloaded_files", side_effect=interrupted), self.assertRaises(KeyboardInterrupt):
self.agent.library_download(payload)
previous = self.agent.library_transfers["downloaded"].snapshot()
self.assertFalse(self.agent.library_dir("t6", "downloaded").exists())
self.agent.journal.close()
self.agent = Agent(self.config, self.docker)
interrupted_cache = next(item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == "downloaded")
self.assertEqual(interrupted_cache["status"], "missing")
self.assertEqual(interrupted_cache["bytesDownloaded"], len(content))
self.assertEqual(interrupted_cache["bytesVerified"], 5)
requests = []
def fetch(method, path, **kwargs):
requests.append(path)
return Response(json.dumps(manifest).encode())
original_verify = self.agent.verify_downloaded_files
def verifying(files, targets, transfer, repair=False):
self.assertEqual(transfer.snapshot()["bytesVerified"], 0)
self.assertIsNone(transfer.snapshot()["transfer"]["etaSeconds"])
return original_verify(files, targets, transfer, repair)
self.agent.artifact_open = fetch
with patch.object(self.agent, "verify_downloaded_files", side_effect=verifying):
self.assertEqual(self.agent.library_download(payload)["state"], "ready")
self.assertEqual(requests, ["/manifest"])
cache = next(item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == "downloaded")
self.assertEqual(cache["bytesVerified"], len(content))
self.assertEqual(cache["transfer"]["startedAt"], previous["transfer"]["startedAt"])
self.assertGreaterEqual(cache["transfer"]["elapsedSeconds"], interrupted_cache["transfer"]["elapsedSeconds"])
self.assertGreaterEqual(cache["transfer"]["phaseElapsedSeconds"], interrupted_cache["transfer"]["phaseElapsedSeconds"])
self.agent.journal.close()
self.agent = Agent(self.config, self.docker)
self.assertEqual(next(item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == "downloaded"), cache)
def test_crash_before_atomic_promotion_reverifies_read_only_staging_without_network_transfers(self):
content = b"complete-game-file"
manifest = self.manifest(content)
payload = {"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}
self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode() if path == "/manifest" else content)
original_replace = module.os.replace
def replace(source, destination):
if Path(destination) == self.agent.library_dir("t6", "downloaded"):
raise KeyboardInterrupt("Simulated process termination before promotion")
return original_replace(source, destination)
with patch.object(module.os, "replace", side_effect=replace), self.assertRaises(KeyboardInterrupt):
self.agent.library_download(payload)
staging = self.agent.root / "libraries/t6/.downloaded.partial/content"
self.assertEqual(staging.stat().st_mode & 0o777, 0o555)
self.agent.journal.close()
self.agent = Agent(self.config, self.docker)
requests = []
def fetch(method, path, **kwargs):
requests.append(path)
return Response(json.dumps(manifest).encode())
self.agent.artifact_open = fetch
self.assertEqual(self.agent.library_download(payload)["state"], "ready")
self.assertEqual(requests, ["/manifest"])
self.assertEqual((self.agent.library_dir("t6", "downloaded") / "content/zone/test.ff").read_bytes(), content)
def test_existing_cache_reverification_is_visible_and_failure_revokes_readiness_after_restart(self):
saved = json.loads((self.library / "manifest.json").read_text())
manifest = {**saved, "id": self.library_id}
self.agent.artifact_open = lambda *args, **kwargs: Response(json.dumps(manifest).encode())
original_verify = self.agent.verify_downloaded_files
def verifying(files, targets, transfer, repair=False):
inventory = self.agent.inventory()
cache = next(item for item in inventory["libraryCaches"] if item["libraryVersionId"] == self.library_id)
self.assertEqual(cache["status"], "verifying")
self.assertEqual(cache["bytesVerified"], 0)
self.assertNotIn({"id": self.library_id, "state": "ready"}, inventory["libraries"])
return original_verify(files, targets, transfer, repair)
(self.library / "content/game.ff").write_bytes(b"corrupt!!")
with patch.object(self.agent, "verify_downloaded_files", side_effect=verifying), self.assertRaisesRegex(AgentError, "integrity"):
self.agent.library_download({"libraryVersionId": self.library_id, "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
self.agent.journal.close()
self.agent = Agent(self.config, self.docker)
inventory = self.agent.inventory()
self.assertNotIn({"id": self.library_id, "state": "ready"}, inventory["libraries"])
self.assertEqual(next(item for item in inventory["libraryCaches"] if item["libraryVersionId"] == self.library_id)["status"], "failed")
def test_elapsed_and_phase_eta_use_new_work_only_and_freeze_at_completion_or_failure(self):
for terminal_status in ("ready", "failed"):
with self.subTest(terminal_status=terminal_status):
now = [100.0]
progress = {"libraryVersionId": "downloaded", "status": "downloading", "bytesDownloaded": 80, "bytesVerified": 0, "bytesTotal": 100}
previous = {"transfer": {"phase": "downloading", "startedAt": "2026-10-04T00:00:00Z", "elapsedSeconds": 20, "phaseElapsedSeconds": 20}}
with patch.object(module.time, "monotonic", side_effect=lambda: now[0]):
transfer = module.LibraryTransfer(progress, self.agent.root / "agent/timing.json", previous)
self.assertIsNone(transfer.snapshot()["transfer"]["etaSeconds"])
now[0] += 0.5
transfer.advance("bytesDownloaded", 5, 5)
self.assertIsNone(transfer.snapshot()["transfer"]["etaSeconds"])
now[0] += 1.5
snapshot = transfer.snapshot()["transfer"]
self.assertEqual(snapshot["bytesPerSecond"], 2.5)
self.assertEqual(snapshot["etaSeconds"], 6)
self.assertEqual(snapshot["elapsedSeconds"], 22)
now[0] += 1
transfer.advance("bytesDownloaded", 15, 15)
transfer.begin_verification()
self.assertEqual(transfer.snapshot()["transfer"]["phaseElapsedSeconds"], 0)
self.assertIsNone(transfer.snapshot()["transfer"]["etaSeconds"])
now[0] += 2
transfer.advance("bytesVerified", 40, 40)
snapshot = transfer.snapshot()["transfer"]
self.assertEqual(snapshot["bytesPerSecond"], 20)
self.assertEqual(snapshot["etaSeconds"], 3)
self.assertEqual(snapshot["elapsedSeconds"], 25)
transfer.finish(terminal_status)
final = transfer.snapshot()
now[0] += 100
self.assertEqual(transfer.snapshot(), final)
self.assertEqual(final["transfer"]["etaSeconds"], 0 if terminal_status == "ready" else None)
def test_partial_inventory_rejects_changed_manifest_and_symlinked_content(self):
content = b"complete-game-file"
manifest = self.manifest(content)