Initial Mist scaffold
Successor to the Josh Steam prototypes. Single-VM Docker Compose stack with
the load-bearing core/ logic ported from JoshSteam CDN with bug fixes.
Contents:
- backend/ FastAPI + Celery (same image, two entrypoints)
core/ hdiff, librsync, chain_replay, manifest, compression,
discord, steam, unrealpak, paths
api/ auth, catalog, admin, builds (skeletons) + downloads (real)
worker/ Celery factory replacing the missing prototype Tasks/__init__.py
db/ SQLAlchemy models + Alembic initial migration
- admin-web/ SvelteKit + Tailwind skeleton
- client/ Tauri 2 + Svelte skeleton (Mist placeholder UI)
- mistpipe/ click-based admin CLI with subcommand stubs
- docs/ ARCHITECTURE, DECISIONS (9 ADRs), RUNBOOK
- docker-compose.yml + dev overlay + .github/workflows
Bugs fixed during port:
- Routes/download.py:2 stray backslash on import line
- Utils/celery.py inspect.reserved() missing parens + double active() typo
- Hardcoded OneDrive/Desktop paths replaced with pydantic-settings config
- Discord webhook URL + RabbitMQ password moved to env vars
- Missing Tasks/__init__.py reconstructed as worker/__init__.py
Out of scope for this commit: route bodies, UI screens, mistpipe subcommand
bodies, real image builds.
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
"""Per-file SHA-256 manifests + per-game version-history helpers.
|
||||
|
||||
Ported from: JoshSteam CDN/Utils/manifest.py
|
||||
Bug fixes during port:
|
||||
- Hardcoded GAMES_DIR removed; uses `mist.core.paths` instead.
|
||||
|
||||
NOTE: the version-history-as-JSON functions (`update_game_version_manifest`,
|
||||
`get_latest_version_from_game_manifest`, `is_direct_update`) are kept as
|
||||
**legacy fallbacks**. The Postgres `Version` model is the new source of
|
||||
truth. Callsites are expected to migrate; the JSON path remains for
|
||||
emergency manual operations on the NAS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from mist.core import paths
|
||||
|
||||
|
||||
def file_checksum(file_path: Path | str) -> str:
|
||||
"""SHA-256 of a file, computed in 4KB chunks."""
|
||||
sha256_hash = hashlib.sha256()
|
||||
with open(file_path, "rb") as f:
|
||||
for byte_block in iter(lambda: f.read(4096), b""):
|
||||
sha256_hash.update(byte_block)
|
||||
return sha256_hash.hexdigest()
|
||||
|
||||
|
||||
def generate_manifest(game_title: str, version: str, depot_path: Path | str) -> Path:
|
||||
"""Walk `depot_path`, compute SHA-256 for every file, write to manifests/<version>.json."""
|
||||
depot_path = Path(depot_path)
|
||||
manifest: dict[str, dict[str, str]] = {}
|
||||
for root, _, files in __import__("os").walk(depot_path):
|
||||
for file in files:
|
||||
file_path = Path(root) / file
|
||||
checksum = file_checksum(file_path)
|
||||
relative_path = str(file_path.relative_to(depot_path))
|
||||
manifest[relative_path] = {"checksum": checksum, "path": relative_path}
|
||||
|
||||
out_path = paths.manifest_path(game_title, version)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(out_path, "w") as f:
|
||||
json.dump(manifest, f, indent=4)
|
||||
return out_path
|
||||
|
||||
|
||||
def load_manifest(manifest_file: Path | str) -> dict[str, dict[str, str]]:
|
||||
"""Load a per-version manifest file."""
|
||||
with open(manifest_file) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# ---- Legacy version-history JSON helpers (preserved for manual ops) ----
|
||||
|
||||
|
||||
def update_game_version_manifest(game_title: str, version: str) -> None:
|
||||
"""Append a version to the per-game version-history JSON. Legacy."""
|
||||
history_file = paths.game_version_history_path(game_title)
|
||||
if history_file.exists():
|
||||
with open(history_file) as f:
|
||||
history: list[str] = json.load(f)
|
||||
else:
|
||||
history = []
|
||||
history.append(version)
|
||||
history_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(history_file, "w") as f:
|
||||
json.dump(history, f, indent=4)
|
||||
|
||||
|
||||
def get_latest_version_from_game_manifest(game_title: str) -> str | None:
|
||||
"""Last entry in the legacy version-history JSON."""
|
||||
history_file = paths.game_version_history_path(game_title)
|
||||
if not history_file.exists():
|
||||
return None
|
||||
with open(history_file) as f:
|
||||
versions: list[str] = json.load(f)
|
||||
return versions[-1] if versions else None
|
||||
|
||||
|
||||
def get_earliest_version_from_game_manifest(game_title: str) -> str | None:
|
||||
"""First entry in the legacy version-history JSON."""
|
||||
history_file = paths.game_version_history_path(game_title)
|
||||
if not history_file.exists():
|
||||
return None
|
||||
with open(history_file) as f:
|
||||
versions: list[str] = json.load(f)
|
||||
return versions[0] if versions else None
|
||||
|
||||
|
||||
def is_direct_update(game_title: str, version1: str, version2: str) -> bool:
|
||||
"""True iff version1 is the immediate predecessor of version2. Legacy."""
|
||||
history_file = paths.game_version_history_path(game_title)
|
||||
if not history_file.exists():
|
||||
return False
|
||||
with open(history_file) as f:
|
||||
versions: list[str] = json.load(f)
|
||||
if version1 not in versions or version2 not in versions:
|
||||
return False
|
||||
return versions.index(version1) + 1 == versions.index(version2)
|
||||
|
||||
|
||||
# ---- Manifest verification ----
|
||||
|
||||
|
||||
def verify_files(game_dir: Path | str, manifest_file: Path | str) -> dict[str, Any]:
|
||||
"""Verify every file under `game_dir` against the SHA-256s in `manifest_file`.
|
||||
|
||||
Returns a dict with `missing_files`, `mismatched_checksums`, `all_files_verified`.
|
||||
"""
|
||||
game_dir = Path(game_dir)
|
||||
manifest_file = Path(manifest_file)
|
||||
if not manifest_file.exists():
|
||||
raise FileNotFoundError(f"Manifest not found at {manifest_file}")
|
||||
|
||||
with open(manifest_file) as f:
|
||||
manifest_data: dict[str, dict[str, str]] = json.load(f)
|
||||
|
||||
results: dict[str, Any] = {
|
||||
"missing_files": [],
|
||||
"mismatched_checksums": [],
|
||||
"all_files_verified": True,
|
||||
}
|
||||
|
||||
for _, file_info in manifest_data.items():
|
||||
file_path = game_dir / file_info["path"]
|
||||
if not file_path.exists():
|
||||
results["missing_files"].append(str(file_path))
|
||||
results["all_files_verified"] = False
|
||||
continue
|
||||
actual = file_checksum(file_path)
|
||||
if actual != file_info["checksum"]:
|
||||
results["mismatched_checksums"].append(
|
||||
{"file": str(file_path), "expected": file_info["checksum"], "actual": actual}
|
||||
)
|
||||
results["all_files_verified"] = False
|
||||
|
||||
return results
|
||||
Reference in New Issue
Block a user