Initial Mist scaffold
admin-web / build (push) Successful in 22s
backend / test (push) Failing after 52s
mistpipe / test (push) Successful in 10s
admin-web / build-and-push (push) Failing after 5s
backend / build-and-push (push) Has been skipped

Successor to the Josh Steam prototypes. Single-VM Docker Compose stack with
the load-bearing core/ logic ported from JoshSteam CDN with bug fixes.

Contents:
- backend/  FastAPI + Celery (same image, two entrypoints)
            core/  hdiff, librsync, chain_replay, manifest, compression,
                   discord, steam, unrealpak, paths
            api/   auth, catalog, admin, builds (skeletons) + downloads (real)
            worker/  Celery factory replacing the missing prototype Tasks/__init__.py
            db/    SQLAlchemy models + Alembic initial migration
- admin-web/  SvelteKit + Tailwind skeleton
- client/    Tauri 2 + Svelte skeleton (Mist placeholder UI)
- mistpipe/  click-based admin CLI with subcommand stubs
- docs/      ARCHITECTURE, DECISIONS (9 ADRs), RUNBOOK
- docker-compose.yml + dev overlay + .github/workflows

Bugs fixed during port:
- Routes/download.py:2 stray backslash on import line
- Utils/celery.py inspect.reserved() missing parens + double active() typo
- Hardcoded OneDrive/Desktop paths replaced with pydantic-settings config
- Discord webhook URL + RabbitMQ password moved to env vars
- Missing Tasks/__init__.py reconstructed as worker/__init__.py

Out of scope for this commit: route bodies, UI screens, mistpipe subcommand
bodies, real image builds.
This commit is contained in:
2026-06-07 19:39:25 -04:00
commit bfd6771a9a
76 changed files with 3890 additions and 0 deletions
+141
View File
@@ -0,0 +1,141 @@
"""Per-file SHA-256 manifests + per-game version-history helpers.
Ported from: JoshSteam CDN/Utils/manifest.py
Bug fixes during port:
- Hardcoded GAMES_DIR removed; uses `mist.core.paths` instead.
NOTE: the version-history-as-JSON functions (`update_game_version_manifest`,
`get_latest_version_from_game_manifest`, `is_direct_update`) are kept as
**legacy fallbacks**. The Postgres `Version` model is the new source of
truth. Callsites are expected to migrate; the JSON path remains for
emergency manual operations on the NAS.
"""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from typing import Any
from mist.core import paths
def file_checksum(file_path: Path | str) -> str:
"""SHA-256 of a file, computed in 4KB chunks."""
sha256_hash = hashlib.sha256()
with open(file_path, "rb") as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
return sha256_hash.hexdigest()
def generate_manifest(game_title: str, version: str, depot_path: Path | str) -> Path:
"""Walk `depot_path`, compute SHA-256 for every file, write to manifests/<version>.json."""
depot_path = Path(depot_path)
manifest: dict[str, dict[str, str]] = {}
for root, _, files in __import__("os").walk(depot_path):
for file in files:
file_path = Path(root) / file
checksum = file_checksum(file_path)
relative_path = str(file_path.relative_to(depot_path))
manifest[relative_path] = {"checksum": checksum, "path": relative_path}
out_path = paths.manifest_path(game_title, version)
out_path.parent.mkdir(parents=True, exist_ok=True)
with open(out_path, "w") as f:
json.dump(manifest, f, indent=4)
return out_path
def load_manifest(manifest_file: Path | str) -> dict[str, dict[str, str]]:
"""Load a per-version manifest file."""
with open(manifest_file) as f:
return json.load(f)
# ---- Legacy version-history JSON helpers (preserved for manual ops) ----
def update_game_version_manifest(game_title: str, version: str) -> None:
"""Append a version to the per-game version-history JSON. Legacy."""
history_file = paths.game_version_history_path(game_title)
if history_file.exists():
with open(history_file) as f:
history: list[str] = json.load(f)
else:
history = []
history.append(version)
history_file.parent.mkdir(parents=True, exist_ok=True)
with open(history_file, "w") as f:
json.dump(history, f, indent=4)
def get_latest_version_from_game_manifest(game_title: str) -> str | None:
"""Last entry in the legacy version-history JSON."""
history_file = paths.game_version_history_path(game_title)
if not history_file.exists():
return None
with open(history_file) as f:
versions: list[str] = json.load(f)
return versions[-1] if versions else None
def get_earliest_version_from_game_manifest(game_title: str) -> str | None:
"""First entry in the legacy version-history JSON."""
history_file = paths.game_version_history_path(game_title)
if not history_file.exists():
return None
with open(history_file) as f:
versions: list[str] = json.load(f)
return versions[0] if versions else None
def is_direct_update(game_title: str, version1: str, version2: str) -> bool:
"""True iff version1 is the immediate predecessor of version2. Legacy."""
history_file = paths.game_version_history_path(game_title)
if not history_file.exists():
return False
with open(history_file) as f:
versions: list[str] = json.load(f)
if version1 not in versions or version2 not in versions:
return False
return versions.index(version1) + 1 == versions.index(version2)
# ---- Manifest verification ----
def verify_files(game_dir: Path | str, manifest_file: Path | str) -> dict[str, Any]:
"""Verify every file under `game_dir` against the SHA-256s in `manifest_file`.
Returns a dict with `missing_files`, `mismatched_checksums`, `all_files_verified`.
"""
game_dir = Path(game_dir)
manifest_file = Path(manifest_file)
if not manifest_file.exists():
raise FileNotFoundError(f"Manifest not found at {manifest_file}")
with open(manifest_file) as f:
manifest_data: dict[str, dict[str, str]] = json.load(f)
results: dict[str, Any] = {
"missing_files": [],
"mismatched_checksums": [],
"all_files_verified": True,
}
for _, file_info in manifest_data.items():
file_path = game_dir / file_info["path"]
if not file_path.exists():
results["missing_files"].append(str(file_path))
results["all_files_verified"] = False
continue
actual = file_checksum(file_path)
if actual != file_info["checksum"]:
results["mismatched_checksums"].append(
{"file": str(file_path), "expected": file_info["checksum"], "actual": actual}
)
results["all_files_verified"] = False
return results