josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Separate game file downloads and verification with progress timing

Browse Source
This commit is contained in:
josh
2026-10-04 12:51:05 -04:00
parent 68323c0baa
commit f8c60fd101
17 changed files with 1150 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/ec_agent.py
+198 -50
View File
@@ -30,7 +30,7 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.2.1"
VERSION = "1.2.2"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
LIBRARY_DOWNLOAD_WORKERS = 4
@@ -587,6 +587,80 @@ class Uninstaller:
time.sleep(15)
class LibraryTransfer:
"""Durable phase progress; throughput counts only work in this attempt."""
def __init__(self, progress, marker, previous=None):
self.progress = progress
self.marker = marker
self.lock = threading.RLock()
prior = (previous or {}).get("transfer") or {}
self.prior_phase = prior.get("phase")
self.prior_phase_elapsed = prior.get("phaseElapsedSeconds", 0)
self.started_at = prior.get("startedAt") or time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
self.elapsed = prior.get("elapsedSeconds", 0)
self.phase_elapsed = prior.get("phaseElapsedSeconds", 0) if prior.get("phase") == progress["status"] else 0
self.started = self.phase_started = time.monotonic()
self.phase = progress["status"]
self.work = 0
self.finished = False
self.failed_bytes_downloaded = None
self.persisted_at = self.started
self.persist(force=True)
def snapshot(self):
with self.lock:
if not self.finished:
now = time.monotonic()
sample_seconds = max(0, now - self.phase_started)
rate = self.work / sample_seconds if sample_seconds >= 1 and self.work else None
done = self.progress["bytesDownloaded"] if self.phase == "downloading" else self.progress["bytesVerified"]
self.progress["transfer"] = {
"phase": self.phase, "startedAt": self.started_at,
"elapsedSeconds": self.elapsed + max(0, now - self.started),
"phaseElapsedSeconds": self.phase_elapsed + sample_seconds,
"bytesPerSecond": rate,
"etaSeconds": max(0, self.progress["bytesTotal"] - done) / rate if rate else None,
}
return {**self.progress, "transfer": self.progress["transfer"].copy()}
def persist(self, force=False):
with self.lock:
now = time.monotonic()
self.snapshot()
if force or now - self.persisted_at >= 5:
atomic_json(self.marker, self.progress)
self.persisted_at = now
def advance(self, key, count, work):
with self.lock:
self.progress[key] += count
self.work += work
self.persist()
def begin_verification(self):
with self.lock:
self.progress.update(status="verifying", bytesVerified=0)
if self.phase != "verifying":
self.phase_elapsed = self.prior_phase_elapsed if self.prior_phase == "verifying" else 0
self.phase = "verifying"
self.phase_started = time.monotonic()
self.work = 0
self.persist(force=True)
def finish(self, status, error=None):
with self.lock:
self.snapshot()
self.finished = True
self.progress["status"] = status
if status == "failed" and self.failed_bytes_downloaded is not None:
self.progress["bytesDownloaded"] = self.failed_bytes_downloaded
self.progress["transfer"]["etaSeconds"] = 0 if status == "ready" else None
if error:
self.progress["error"] = error
self.persist(force=True)
class Agent:
def __init__(self, config, docker=None):
self.config = config
@@ -599,6 +673,7 @@ class Agent:
self.stopping = threading.Event()
self.current_operation = None
self.library_progress = {}
self.library_transfers = {}
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
self.executing_id = None
@@ -789,6 +864,8 @@ class Agent:
except (AgentError, ValueError):
pass
libraries, caches = [], []
progress = {key: value.copy() for key, value in list(self.library_progress.items())}
progress.update({key: transfer.snapshot() for key, transfer in list(self.library_transfers.items())})
for marker in (self.root / "libraries").glob("*/*/manifest.json"):
# A crash between writing a manifest and its atomic promotion must
# never expose the staging directory as a ready library.
@@ -796,12 +873,22 @@ class Agent:
continue
try:
value = json.loads(marker.read_text())
# Rechecking an existing cache must revoke readiness until its
# entire contents have passed verification again.
if value["libraryId"] not in progress and (marker.parent / "transfer.json").exists():
saved_progress = json.loads((marker.parent / "transfer.json").read_text())
if saved_progress.get("status") != "ready":
progress[value["libraryId"]] = {**saved_progress, "status": "failed" if saved_progress.get("status") == "failed" else "missing"}
if value["libraryId"] in progress:
continue
libraries.append({"id": value["libraryId"], "state": "ready"})
caches.append({"libraryVersionId": value["libraryId"], "status": "ready", "bytesDownloaded": value["size"], "bytesTotal": value["size"], "manifestSha256": value.get("manifestSha256")})
cache = {"libraryVersionId": value["libraryId"], "status": "ready", "bytesDownloaded": value["size"], "bytesVerified": value["size"], "bytesTotal": value["size"], "manifestSha256": value.get("manifestSha256")}
if value.get("transfer"):
cache["transfer"] = value["transfer"]
caches.append(cache)
except (OSError, ValueError, KeyError):
continue
ready = {item["id"] for item in libraries}
progress = {key: value.copy() for key, value in list(self.library_progress.items()) if key not in ready}
for marker in (self.root / "libraries").glob("*/.*.partial/source.json"):
try:
value = json.loads(marker.read_text())
@@ -826,7 +913,15 @@ class Agent:
retained += min(target.stat().st_size, item["size"])
# These are resumable bytes, not verified content. Reporting
# them survives restarts without requiring space for two copies.
progress[library_id] = {"libraryVersionId": library_id, "status": "missing", "bytesDownloaded": retained, "bytesTotal": size, "manifestSha256": value["manifestSha256"]}
cache = {"libraryVersionId": library_id, "status": "missing", "bytesDownloaded": retained, "bytesVerified": 0, "bytesTotal": size, "manifestSha256": value["manifestSha256"]}
transfer_marker = marker.parent / "transfer.json"
if transfer_marker.exists():
saved_progress = json.loads(transfer_marker.read_text())
if saved_progress.get("manifestSha256") == value["manifestSha256"]:
cache.update(bytesVerified=saved_progress.get("bytesVerified", 0), transfer=saved_progress.get("transfer"))
if saved_progress.get("status") == "failed":
cache.update(status="failed", error=saved_progress.get("error"))
progress[library_id] = cache
except (AgentError, OSError, ValueError, KeyError, TypeError):
continue
progress = list(progress.values())
@@ -1221,14 +1316,14 @@ class Agent:
raise AgentError("Library transport must use HTTPS")
return url
def artifact_open(self, method, path, data=None, extra_headers=None):
def artifact_open(self, method, path, data=None, extra_headers=None, timeout=60):
headers = {"Authorization": "Bearer " + self.config["credential"], **(extra_headers or {})}
req = urllib.request.Request(self.artifact_url(path), data=data, method=method, headers=headers)
# Redirects cannot forward host credentials to another origin.
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
raise AgentError("Artifact redirects are not allowed")
return urllib.request.build_opener(NoRedirect()).open(req, timeout=60)
return urllib.request.build_opener(NoRedirect()).open(req, timeout=timeout)
def download_library_file(self, item, target, files_url, update_progress, aborted):
def check_active():
@@ -1241,21 +1336,11 @@ class Agent:
if offset > item["size"]:
target.unlink()
offset = 0
checksum = hashlib.sha256()
# Hash the saved prefix once so resumed transfers still verify the whole
# file. New bytes are hashed as they arrive, avoiding a second disk read.
if offset:
with target.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
check_active()
checksum.update(chunk)
# Saved prefixes and complete files are intentionally not read here.
# The second phase verifies every byte after all transfers have ended.
if offset == item["size"] and target.exists():
if checksum.hexdigest() == item["sha256"]:
update_progress(item["path"], offset)
return
target.unlink()
offset = 0
checksum = hashlib.sha256()
update_progress(item["path"], offset)
return
update_progress(item["path"], offset)
check_active()
url = files_url.rstrip("/") + "/" + urllib.parse.quote(item["path"], safe="/")
@@ -1269,7 +1354,6 @@ class Agent:
elif offset:
# A server may ignore Range and send the whole file instead.
offset = 0
checksum = hashlib.sha256()
update_progress(item["path"], 0)
with target.open("ab" if offset else "wb") as output:
received = offset
@@ -1282,13 +1366,48 @@ class Agent:
if received > item["size"]:
raise AgentError("Library download exceeded its manifest size")
output.write(chunk)
checksum.update(chunk)
update_progress(item["path"], received)
output.flush()
os.fsync(output.fileno())
if target.stat().st_size != item["size"] or checksum.hexdigest() != item["sha256"]:
if target.stat().st_size != item["size"]:
raise AgentError("Downloaded library file size failed integrity verification")
def verify_downloaded_files(self, files, targets, transfer, repair=False):
invalid = False
retained = transfer.progress["bytesDownloaded"]
for item in files:
target = targets[item["path"]]
checksum = hashlib.sha256()
actual_size = target.stat().st_size if target.is_file() else 0
valid_size = target.is_file() and actual_size == item["size"]
if valid_size:
with target.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
checksum.update(chunk)
transfer.advance("bytesVerified", len(chunk), len(chunk))
if not valid_size or checksum.hexdigest() != item["sha256"]:
invalid = True
if repair and target.exists():
# A corrupt complete file (including a corrupt saved
# prefix) must not be resumed indefinitely on every retry.
target.unlink()
actual_size = 0
retained -= item["size"] - min(actual_size, item["size"])
if invalid:
# Verification describes a completed download phase. Keep that
# total stable while other files are checked, then publish retained
# bytes atomically with failure (including removed corrupt files).
transfer.failed_bytes_downloaded = retained
raise AgentError("Downloaded library file failed integrity verification")
def publish_library_progress(self, payload, transfer):
if payload.get("statusUrl"):
# Phase boundaries are useful even when shorter than the heartbeat
# interval. Telemetry failure must never undo a verified download.
with contextlib.suppress(AgentError, OSError, ValueError):
with self.artifact_open("POST", payload["statusUrl"], json.dumps(transfer.snapshot()).encode(), {"Content-Type": "application/json"}, timeout=5) as response:
response.read()
def library_download(self, payload):
library_id = identifier(payload.get("libraryVersionId", payload.get("libraryId")))
with self.artifact_open("GET", payload["manifestUrl"]) as response:
@@ -1305,12 +1424,12 @@ class Agent:
if payload.get("manifestSha256") and payload["manifestSha256"] != manifest_sha:
raise AgentError("Library manifest differs from the requested immutable version")
directory = self.library_dir(manifest["game"], library_id)
if directory.exists():
saved = self.verify_library(directory)
existing = directory.exists()
if existing:
saved = json.loads((directory / "manifest.json").read_text())
if saved.get("manifestSha256") != manifest_sha:
raise AgentError("Published library changed unexpectedly")
return {"libraryId": library_id, "state": "ready", "size": saved["size"]}
staging = directory.with_name("." + library_id + ".partial")
staging = directory if existing else directory.with_name("." + library_id + ".partial")
staging.mkdir(parents=True, exist_ok=True)
source_marker = staging / "source.json"
if source_marker.exists() and json.loads(source_marker.read_text()).get("manifestSha256") != manifest_sha:
@@ -1318,28 +1437,39 @@ class Agent:
size = sum(item["size"] for item in files)
atomic_json(source_marker, {"libraryId": library_id, "game": manifest["game"], "size": size, "files": files, "manifestSha256": manifest_sha})
targets = {}
progress = {"libraryVersionId": library_id, "status": "downloading", "bytesDownloaded": 0, "bytesTotal": size, "manifestSha256": manifest_sha}
progress = {"libraryVersionId": library_id, "status": "verifying" if existing else "downloading", "bytesDownloaded": size if existing else 0, "bytesVerified": 0, "bytesTotal": size, "manifestSha256": manifest_sha}
self.library_progress[library_id] = progress
marker = staging / "transfer.json"
previous = json.loads(marker.read_text()) if marker.exists() else None
if previous and (previous.get("manifestSha256") != manifest_sha or previous.get("status") == "ready"):
previous = None
timing = LibraryTransfer(progress, marker, previous)
self.library_transfers[library_id] = timing
try:
if shutil.disk_usage(staging).free < max(0, size - sum(path.stat().st_size for path in staging.rglob("*") if path.is_file())):
raise AgentError("Insufficient library storage")
# Validate every path before starting concurrent writes. In addition
# to duplicates, a manifest cannot treat a file as a parent folder.
for item in files:
if item["path"] in targets or not isinstance(item["size"], int) or item["size"] < 0 or not re.fullmatch("[0-9a-f]{64}", item["sha256"]):
if item["path"] in targets or type(item["size"]) is not int or item["size"] < 0 or not re.fullmatch("[0-9a-f]{64}", item["sha256"]):
raise AgentError("Invalid library file manifest")
targets[item["path"]] = scoped(staging / "content", item["path"])
for path in targets:
if any(str(parent) in targets for parent in PurePosixPath(path).parents):
raise AgentError("Library file paths overlap")
progress_lock = threading.Lock()
offsets = {}
offsets = {item["path"]: min(targets[item["path"]].stat().st_size, item["size"]) if targets[item["path"]].exists() else 0 for item in files}
if not existing:
progress["bytesDownloaded"] = sum(offsets.values())
if not existing and shutil.disk_usage(staging).free < size - progress["bytesDownloaded"]:
raise AgentError("Insufficient library storage")
timing.persist(force=True)
self.publish_library_progress(payload, timing)
aborted = threading.Event()
def update_progress(path, offset):
with progress_lock:
progress["bytesDownloaded"] += offset - offsets.get(path, 0)
delta = offset - offsets.get(path, 0)
offsets[path] = offset
timing.advance("bytesDownloaded", delta, max(0, delta))
def transfer(item):
try:
@@ -1350,21 +1480,44 @@ class Agent:
# Bound sockets, memory and disk pressure while allowing other files
# to progress during network round trips and per-file fsyncs.
with ThreadPoolExecutor(max_workers=min(LIBRARY_DOWNLOAD_WORKERS, len(files))) as pool:
futures = [pool.submit(transfer, item) for item in files]
for future in as_completed(futures):
future.result()
progress["status"] = "verifying"
if not existing:
# A crash after setting read-only permissions but before the
# atomic rename leaves a valid, resumable staging directory.
content_root = staging / "content"
if content_root.exists():
os.chmod(content_root, 0o755)
for path in content_root.rglob("*"):
if path.is_symlink():
raise AgentError("Symlinks are not allowed in library content")
os.chmod(path, 0o755 if path.is_dir() else 0o644)
with ThreadPoolExecutor(max_workers=min(LIBRARY_DOWNLOAD_WORKERS, len(files))) as pool:
futures = [pool.submit(transfer, item) for item in files]
for future in as_completed(futures):
future.result()
timing.begin_verification()
self.publish_library_progress(payload, timing)
self.verify_downloaded_files(files, targets, timing, repair=not existing)
saved = {"libraryId": library_id, "game": manifest["game"], "size": size, "files": files, "manifestSha256": manifest_sha}
atomic_json(staging / "manifest.json", saved)
for path in (staging / "content").rglob("*"):
os.chmod(path, 0o555 if path.is_dir() else 0o444)
os.chmod(staging / "content", 0o555)
os.replace(staging, directory)
if not existing:
for path in (staging / "content").rglob("*"):
if path.is_symlink():
raise AgentError("Symlinks are not allowed in library content")
os.chmod(path, 0o555 if path.is_dir() else 0o444)
os.chmod(staging / "content", 0o555)
os.replace(staging, directory)
timing.marker = directory / "transfer.json"
timing.finish("ready")
saved["transfer"] = timing.snapshot()["transfer"]
saved["bytesVerified"] = size
atomic_json(directory / "manifest.json", saved)
self.publish_library_progress(payload, timing)
self.library_progress.pop(library_id, None)
self.library_transfers.pop(library_id, None)
return {"libraryId": library_id, "state": "ready", "size": size, "manifestSha256": manifest_sha}
except Exception:
progress.update(status="failed", error="Library download or integrity verification failed")
timing.finish("failed", "Library download or integrity verification failed")
self.publish_library_progress(payload, timing)
raise
def library_import(self, payload):
@@ -1561,14 +1714,9 @@ class Agent:
finally:
self.executing_id = None
if kind == "library.download":
library_id = payload.get("libraryVersionId", payload.get("libraryId"))
for game in ("t5", "t6"):
directory = self.library_dir(game, library_id)
if directory.exists():
saved = self.verify_library(directory)
return self.journal.update(operation["id"], "completed", {"libraryId": library_id, "state": "ready", "size": saved["size"], "manifestSha256": saved.get("manifestSha256"), "reconciled": True})
# This is a resumable immutable transfer, not a repeated
# container command: verified completed files remain untouched.
# container command. Full files skip transfer; verification
# always starts again so interrupted reads cannot prove ready.
self.journal.update(operation["id"], "running")
try:
return self.journal.update(operation["id"], "completed", self.library_download(payload))