bfd6771a9a
Successor to the Josh Steam prototypes. Single-VM Docker Compose stack with
the load-bearing core/ logic ported from JoshSteam CDN with bug fixes.
Contents:
- backend/ FastAPI + Celery (same image, two entrypoints)
core/ hdiff, librsync, chain_replay, manifest, compression,
discord, steam, unrealpak, paths
api/ auth, catalog, admin, builds (skeletons) + downloads (real)
worker/ Celery factory replacing the missing prototype Tasks/__init__.py
db/ SQLAlchemy models + Alembic initial migration
- admin-web/ SvelteKit + Tailwind skeleton
- client/ Tauri 2 + Svelte skeleton (Mist placeholder UI)
- mistpipe/ click-based admin CLI with subcommand stubs
- docs/ ARCHITECTURE, DECISIONS (9 ADRs), RUNBOOK
- docker-compose.yml + dev overlay + .github/workflows
Bugs fixed during port:
- Routes/download.py:2 stray backslash on import line
- Utils/celery.py inspect.reserved() missing parens + double active() typo
- Hardcoded OneDrive/Desktop paths replaced with pydantic-settings config
- Discord webhook URL + RabbitMQ password moved to env vars
- Missing Tasks/__init__.py reconstructed as worker/__init__.py
Out of scope for this commit: route bodies, UI screens, mistpipe subcommand
bodies, real image builds.
142 lines
5.0 KiB
Python
142 lines
5.0 KiB
Python
"""Per-file SHA-256 manifests + per-game version-history helpers.
|
|
|
|
Ported from: JoshSteam CDN/Utils/manifest.py
|
|
Bug fixes during port:
|
|
- Hardcoded GAMES_DIR removed; uses `mist.core.paths` instead.
|
|
|
|
NOTE: the version-history-as-JSON functions (`update_game_version_manifest`,
|
|
`get_latest_version_from_game_manifest`, `is_direct_update`) are kept as
|
|
**legacy fallbacks**. The Postgres `Version` model is the new source of
|
|
truth. Callsites are expected to migrate; the JSON path remains for
|
|
emergency manual operations on the NAS.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from mist.core import paths
|
|
|
|
|
|
def file_checksum(file_path: Path | str) -> str:
|
|
"""SHA-256 of a file, computed in 4KB chunks."""
|
|
sha256_hash = hashlib.sha256()
|
|
with open(file_path, "rb") as f:
|
|
for byte_block in iter(lambda: f.read(4096), b""):
|
|
sha256_hash.update(byte_block)
|
|
return sha256_hash.hexdigest()
|
|
|
|
|
|
def generate_manifest(game_title: str, version: str, depot_path: Path | str) -> Path:
|
|
"""Walk `depot_path`, compute SHA-256 for every file, write to manifests/<version>.json."""
|
|
depot_path = Path(depot_path)
|
|
manifest: dict[str, dict[str, str]] = {}
|
|
for root, _, files in __import__("os").walk(depot_path):
|
|
for file in files:
|
|
file_path = Path(root) / file
|
|
checksum = file_checksum(file_path)
|
|
relative_path = str(file_path.relative_to(depot_path))
|
|
manifest[relative_path] = {"checksum": checksum, "path": relative_path}
|
|
|
|
out_path = paths.manifest_path(game_title, version)
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(out_path, "w") as f:
|
|
json.dump(manifest, f, indent=4)
|
|
return out_path
|
|
|
|
|
|
def load_manifest(manifest_file: Path | str) -> dict[str, dict[str, str]]:
|
|
"""Load a per-version manifest file."""
|
|
with open(manifest_file) as f:
|
|
return json.load(f)
|
|
|
|
|
|
# ---- Legacy version-history JSON helpers (preserved for manual ops) ----
|
|
|
|
|
|
def update_game_version_manifest(game_title: str, version: str) -> None:
|
|
"""Append a version to the per-game version-history JSON. Legacy."""
|
|
history_file = paths.game_version_history_path(game_title)
|
|
if history_file.exists():
|
|
with open(history_file) as f:
|
|
history: list[str] = json.load(f)
|
|
else:
|
|
history = []
|
|
history.append(version)
|
|
history_file.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(history_file, "w") as f:
|
|
json.dump(history, f, indent=4)
|
|
|
|
|
|
def get_latest_version_from_game_manifest(game_title: str) -> str | None:
|
|
"""Last entry in the legacy version-history JSON."""
|
|
history_file = paths.game_version_history_path(game_title)
|
|
if not history_file.exists():
|
|
return None
|
|
with open(history_file) as f:
|
|
versions: list[str] = json.load(f)
|
|
return versions[-1] if versions else None
|
|
|
|
|
|
def get_earliest_version_from_game_manifest(game_title: str) -> str | None:
|
|
"""First entry in the legacy version-history JSON."""
|
|
history_file = paths.game_version_history_path(game_title)
|
|
if not history_file.exists():
|
|
return None
|
|
with open(history_file) as f:
|
|
versions: list[str] = json.load(f)
|
|
return versions[0] if versions else None
|
|
|
|
|
|
def is_direct_update(game_title: str, version1: str, version2: str) -> bool:
|
|
"""True iff version1 is the immediate predecessor of version2. Legacy."""
|
|
history_file = paths.game_version_history_path(game_title)
|
|
if not history_file.exists():
|
|
return False
|
|
with open(history_file) as f:
|
|
versions: list[str] = json.load(f)
|
|
if version1 not in versions or version2 not in versions:
|
|
return False
|
|
return versions.index(version1) + 1 == versions.index(version2)
|
|
|
|
|
|
# ---- Manifest verification ----
|
|
|
|
|
|
def verify_files(game_dir: Path | str, manifest_file: Path | str) -> dict[str, Any]:
|
|
"""Verify every file under `game_dir` against the SHA-256s in `manifest_file`.
|
|
|
|
Returns a dict with `missing_files`, `mismatched_checksums`, `all_files_verified`.
|
|
"""
|
|
game_dir = Path(game_dir)
|
|
manifest_file = Path(manifest_file)
|
|
if not manifest_file.exists():
|
|
raise FileNotFoundError(f"Manifest not found at {manifest_file}")
|
|
|
|
with open(manifest_file) as f:
|
|
manifest_data: dict[str, dict[str, str]] = json.load(f)
|
|
|
|
results: dict[str, Any] = {
|
|
"missing_files": [],
|
|
"mismatched_checksums": [],
|
|
"all_files_verified": True,
|
|
}
|
|
|
|
for _, file_info in manifest_data.items():
|
|
file_path = game_dir / file_info["path"]
|
|
if not file_path.exists():
|
|
results["missing_files"].append(str(file_path))
|
|
results["all_files_verified"] = False
|
|
continue
|
|
actual = file_checksum(file_path)
|
|
if actual != file_info["checksum"]:
|
|
results["mismatched_checksums"].append(
|
|
{"file": str(file_path), "expected": file_info["checksum"], "actual": actual}
|
|
)
|
|
results["all_files_verified"] = False
|
|
|
|
return results
|