Automate host preparation and speed up game file downloads
This commit is contained in:
+221
-2
@@ -4,6 +4,7 @@ import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -107,6 +108,40 @@ class AgentTests(unittest.TestCase):
|
||||
self.assertEqual(result["state"], "unknown")
|
||||
self.assertEqual(self.docker.calls, [])
|
||||
|
||||
def test_agent_restart_resumes_runtime_pull_even_when_an_older_tag_is_cached(self):
|
||||
image = "ghcr.io/ayymoss/plutainer:latest"
|
||||
operation = self.operation("runtime.pull", {"image": image})
|
||||
self.agent.journal.accept(operation)
|
||||
self.agent.journal.update(operation["id"], "running")
|
||||
aliases = self.agent.root / "agent/runtime-images.json"
|
||||
module.atomic_json(aliases, {image: "sha256:older-image"})
|
||||
self.agent.journal.close()
|
||||
self.agent = Agent(self.config, self.docker)
|
||||
original = self.docker.call
|
||||
|
||||
def call(*args, **kwargs):
|
||||
result = original(*args, **kwargs)
|
||||
return "sha256:new-image" if args[:2] == ("image", "inspect") else result
|
||||
|
||||
self.docker.call = call
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "completed")
|
||||
self.assertEqual(json.loads(result["result"]), {"image": image, "ready": True, "reconciled": True})
|
||||
self.assertEqual(json.loads(aliases.read_text())[image], "sha256:new-image")
|
||||
self.agent.execute(operation)
|
||||
self.assertEqual(sum(args[:2] == ("image", "pull") for args in self.docker.calls), 1)
|
||||
|
||||
def test_interrupted_runtime_pull_failure_is_reported_for_automatic_retry(self):
|
||||
operation = self.operation("runtime.pull", {"image": "ghcr.io/ayymoss/plutainer:latest"})
|
||||
self.agent.journal.accept(operation)
|
||||
self.agent.journal.update(operation["id"], "running")
|
||||
self.agent.journal.close()
|
||||
self.agent = Agent(self.config, self.docker)
|
||||
with patch.object(self.docker, "call", side_effect=AgentError("Registry unavailable")):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "failed")
|
||||
self.assertIn("runtime download", result["error"])
|
||||
|
||||
def test_occupied_and_unknown_wait_then_revalidate_same_operation(self):
|
||||
self.create()
|
||||
operation = self.operation("instance.restart")
|
||||
@@ -198,8 +233,192 @@ class AgentTests(unittest.TestCase):
|
||||
module.scoped(app, path)
|
||||
|
||||
def manifest(self, content=b"complete-game-file"):
|
||||
files = [{"path": "zone/test.ff", "size": len(content), "sha256": module.digest(content)}]
|
||||
return {"id": "downloaded", "game": "t6", "files": files, "sizeBytes": len(content), "manifestSha256": module.digest(json.dumps(files, separators=(",", ":")).encode())}
|
||||
return self.manifest_files({"zone/test.ff": content})
|
||||
|
||||
def manifest_files(self, contents):
|
||||
files = [{"path": path, "size": len(content), "sha256": module.digest(content)} for path, content in contents.items()]
|
||||
return {"id": "downloaded", "game": "t6", "files": files, "sizeBytes": sum(item["size"] for item in files), "manifestSha256": module.digest(json.dumps(files, separators=(",", ":")).encode())}
|
||||
|
||||
def test_downloads_use_bounded_parallel_transfers_and_aggregate_progress(self):
|
||||
contents = {f"zone/file-{index}.ff": bytes([index]) * 31 for index in range(8)}
|
||||
manifest = self.manifest_files(contents)
|
||||
barrier = threading.Barrier(4)
|
||||
lock = threading.Lock()
|
||||
active = maximum = 0
|
||||
snapshots = []
|
||||
agent = self.agent
|
||||
|
||||
class ConcurrentResponse(Response):
|
||||
def __exit__(self, *args):
|
||||
nonlocal active
|
||||
with lock:
|
||||
active -= 1
|
||||
snapshots.append(agent.library_progress["downloaded"]["bytesDownloaded"])
|
||||
return super().__exit__(*args)
|
||||
|
||||
def fetch(method, path, data=None, extra_headers=None):
|
||||
nonlocal active, maximum
|
||||
if path == "/manifest":
|
||||
return Response(json.dumps(manifest).encode())
|
||||
with lock:
|
||||
active += 1
|
||||
maximum = max(maximum, active)
|
||||
# A serial downloader cannot reach this barrier. Two batches also
|
||||
# prove that an unlimited worker pool cannot open all eight files.
|
||||
barrier.wait(timeout=5)
|
||||
return ConcurrentResponse(contents[path.removeprefix("/files/")])
|
||||
|
||||
self.agent.artifact_open = fetch
|
||||
result = self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
|
||||
self.assertEqual(maximum, 4)
|
||||
self.assertEqual(snapshots, sorted(snapshots))
|
||||
self.assertEqual(max(snapshots), manifest["sizeBytes"])
|
||||
self.assertEqual(result["size"], manifest["sizeBytes"])
|
||||
for path, content in contents.items():
|
||||
self.assertEqual((self.agent.library_dir("t6", "downloaded") / "content" / path).read_bytes(), content)
|
||||
|
||||
def test_fresh_download_hashes_stream_without_rereading_whole_file(self):
|
||||
content = b"game-file" * 200000
|
||||
manifest = self.manifest(content)
|
||||
self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode() if path == "/manifest" else content)
|
||||
with patch.object(module, "file_digest", side_effect=AssertionError("download reread the file")):
|
||||
result = self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
|
||||
self.assertEqual(result["state"], "ready")
|
||||
|
||||
def test_ignored_range_restarts_file_and_checksum(self):
|
||||
content = b"complete-game-file"
|
||||
manifest = self.manifest(content)
|
||||
staging = self.agent.root / "libraries/t6/.downloaded.partial/content/zone"
|
||||
staging.mkdir(parents=True)
|
||||
(staging / "test.ff").write_bytes(b"bad-prefix")
|
||||
|
||||
def fetch(method, path, data=None, extra_headers=None):
|
||||
if path == "/manifest":
|
||||
return Response(json.dumps(manifest).encode())
|
||||
self.assertEqual(extra_headers, {"Range": "bytes=10-"})
|
||||
return Response(content)
|
||||
|
||||
self.agent.artifact_open = fetch
|
||||
result = self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
|
||||
self.assertEqual(result["state"], "ready")
|
||||
self.assertEqual((self.agent.library_dir("t6", "downloaded") / "content/zone/test.ff").read_bytes(), content)
|
||||
|
||||
def test_resumed_download_verifies_saved_prefix(self):
|
||||
content = b"complete-game-file"
|
||||
manifest = self.manifest(content)
|
||||
staging = self.agent.root / "libraries/t6/.downloaded.partial/content/zone"
|
||||
staging.mkdir(parents=True)
|
||||
(staging / "test.ff").write_bytes(b"corrupt!")
|
||||
self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode()) if path == "/manifest" else Response(content[8:], 206, {"Content-Range": f"bytes 8-{len(content)-1}/{len(content)}"})
|
||||
with self.assertRaisesRegex(AgentError, "integrity"):
|
||||
self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
|
||||
self.assertFalse(self.agent.library_dir("t6", "downloaded").exists())
|
||||
|
||||
def test_parallel_failure_retains_partial_files_for_verified_resume(self):
|
||||
contents = {"zone/first.ff": b"first-game-file", "zone/second.ff": b"second-game-file"}
|
||||
manifest = self.manifest_files(contents)
|
||||
payload = {"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}
|
||||
|
||||
def failed_fetch(method, path, data=None, extra_headers=None):
|
||||
if path == "/manifest":
|
||||
return Response(json.dumps(manifest).encode())
|
||||
return Response(contents[path.removeprefix("/files/")][:5])
|
||||
|
||||
self.agent.artifact_open = failed_fetch
|
||||
with self.assertRaises(AgentError):
|
||||
self.agent.library_download(payload)
|
||||
self.assertFalse(self.agent.library_dir("t6", "downloaded").exists())
|
||||
self.assertEqual(self.agent.library_progress["downloaded"]["status"], "failed")
|
||||
resumed = []
|
||||
|
||||
def resume_fetch(method, path, data=None, extra_headers=None):
|
||||
if path == "/manifest":
|
||||
return Response(json.dumps(manifest).encode())
|
||||
content = contents[path.removeprefix("/files/")]
|
||||
if extra_headers:
|
||||
self.assertEqual(extra_headers, {"Range": "bytes=5-"})
|
||||
resumed.append(path)
|
||||
return Response(content[5:], 206, {"Content-Range": f"bytes 5-{len(content)-1}/{len(content)}"})
|
||||
return Response(content)
|
||||
|
||||
self.agent.artifact_open = resume_fetch
|
||||
self.assertEqual(self.agent.library_download(payload)["state"], "ready")
|
||||
self.assertTrue(resumed)
|
||||
self.assertEqual(self.agent.verify_library(self.agent.library_dir("t6", "downloaded"))["size"], manifest["sizeBytes"])
|
||||
|
||||
def test_overlapping_paths_rejected_before_concurrent_downloads(self):
|
||||
manifest = self.manifest_files({"zone": b"file", "zone/child.ff": b"child"})
|
||||
calls = []
|
||||
|
||||
def fetch(method, path, data=None, extra_headers=None):
|
||||
calls.append(path)
|
||||
return Response(json.dumps(manifest).encode())
|
||||
|
||||
self.agent.artifact_open = fetch
|
||||
with self.assertRaisesRegex(AgentError, "overlap"):
|
||||
self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"})
|
||||
self.assertEqual(calls, ["/manifest"])
|
||||
|
||||
def test_failed_transfer_restart_reports_retained_bytes_and_resumes_with_only_remaining_space(self):
|
||||
content = b"complete-game-file"
|
||||
manifest = self.manifest(content)
|
||||
payload = {"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}
|
||||
self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode() if path == "/manifest" else content[:8])
|
||||
with self.assertRaises(AgentError):
|
||||
self.agent.library_download(payload)
|
||||
self.agent.journal.close()
|
||||
self.agent = Agent(self.config, self.docker)
|
||||
cache = next(item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == "downloaded")
|
||||
self.assertEqual(cache, {"libraryVersionId": "downloaded", "status": "missing", "bytesDownloaded": 8, "bytesTotal": len(content), "manifestSha256": manifest["manifestSha256"]})
|
||||
|
||||
def fetch(method, path, data=None, extra_headers=None):
|
||||
if path == "/manifest":
|
||||
return Response(json.dumps(manifest).encode())
|
||||
self.assertEqual(extra_headers, {"Range": "bytes=8-"})
|
||||
return Response(content[8:], 206, {"Content-Range": f"bytes 8-{len(content)-1}/{len(content)}"})
|
||||
|
||||
self.agent.artifact_open = fetch
|
||||
usage = module.shutil.disk_usage(self.agent.root)
|
||||
with patch.object(module.shutil, "disk_usage", return_value=type(usage)(usage.total, usage.used, len(content) - 8)):
|
||||
self.assertEqual(self.agent.library_download(payload)["state"], "ready")
|
||||
|
||||
def test_unpromoted_manifest_is_never_reported_as_ready_and_progress_is_not_duplicated(self):
|
||||
content = b"complete-game-file"
|
||||
manifest = self.manifest(content)
|
||||
staging = self.agent.root / "libraries/t6/.downloaded.partial"
|
||||
(staging / "content/zone").mkdir(parents=True)
|
||||
# Oversized retained files can be replaced during resume. Their progress
|
||||
# must still stay within the published file size.
|
||||
(staging / "content/zone/test.ff").write_bytes(content + b"extra")
|
||||
saved = {"libraryId": "downloaded", "game": "t6", "size": len(content), "files": manifest["files"], "manifestSha256": manifest["manifestSha256"]}
|
||||
module.atomic_json(staging / "source.json", saved)
|
||||
module.atomic_json(staging / "manifest.json", saved)
|
||||
inventory = self.agent.inventory()
|
||||
caches = [item for item in inventory["libraryCaches"] if item["libraryVersionId"] == "downloaded"]
|
||||
self.assertEqual(len(caches), 1)
|
||||
self.assertEqual(caches[0]["status"], "missing")
|
||||
self.assertEqual(caches[0]["bytesDownloaded"], len(content))
|
||||
self.assertNotIn({"id": "downloaded", "state": "ready"}, inventory["libraries"])
|
||||
self.agent.library_progress["downloaded"] = {**caches[0], "status": "downloading", "bytesDownloaded": 3}
|
||||
caches = [item for item in self.agent.inventory()["libraryCaches"] if item["libraryVersionId"] == "downloaded"]
|
||||
self.assertEqual(len(caches), 1)
|
||||
self.assertEqual(caches[0]["status"], "downloading")
|
||||
self.assertEqual(caches[0]["bytesDownloaded"], 3)
|
||||
|
||||
def test_partial_inventory_rejects_changed_manifest_and_symlinked_content(self):
|
||||
content = b"complete-game-file"
|
||||
manifest = self.manifest(content)
|
||||
staging = self.agent.root / "libraries/t6/.downloaded.partial"
|
||||
(staging / "content/zone").mkdir(parents=True)
|
||||
source = {"libraryId": "downloaded", "game": "t6", "files": manifest["files"], "manifestSha256": "0" * 64}
|
||||
module.atomic_json(staging / "source.json", source)
|
||||
(staging / "content/zone/test.ff").write_bytes(content)
|
||||
self.assertFalse(any(item["libraryVersionId"] == "downloaded" for item in self.agent.inventory()["libraryCaches"]))
|
||||
source["manifestSha256"] = manifest["manifestSha256"]
|
||||
module.atomic_json(staging / "source.json", source)
|
||||
(staging / "content/zone/test.ff").unlink()
|
||||
(staging / "content/zone/test.ff").symlink_to(self.library / "content/game.ff")
|
||||
self.assertFalse(any(item["libraryVersionId"] == "downloaded" for item in self.agent.inventory()["libraryCaches"]))
|
||||
|
||||
def test_resume_verifies_and_atomically_publishes_library(self):
|
||||
content = b"complete-game-file"
|
||||
|
||||
Reference in New Issue
Block a user