Initial Mist scaffold

Successor to the Josh Steam prototypes. Single-VM Docker Compose stack with
the load-bearing core/ logic ported from JoshSteam CDN with bug fixes.

Contents:
- backend/  FastAPI + Celery (same image, two entrypoints)
            core/  hdiff, librsync, chain_replay, manifest, compression,
                   discord, steam, unrealpak, paths
            api/   auth, catalog, admin, builds (skeletons) + downloads (real)
            worker/  Celery factory replacing the missing prototype Tasks/__init__.py
            db/    SQLAlchemy models + Alembic initial migration
- admin-web/  SvelteKit + Tailwind skeleton
- client/    Tauri 2 + Svelte skeleton (Mist placeholder UI)
- mistpipe/  click-based admin CLI with subcommand stubs
- docs/      ARCHITECTURE, DECISIONS (9 ADRs), RUNBOOK
- docker-compose.yml + dev overlay + .github/workflows

Bugs fixed during port:
- Routes/download.py:2 stray backslash on import line
- Utils/celery.py inspect.reserved() missing parens + double active() typo
- Hardcoded OneDrive/Desktop paths replaced with pydantic-settings config
- Discord webhook URL + RabbitMQ password moved to env vars
- Missing Tasks/__init__.py reconstructed as worker/__init__.py

Out of scope for this commit: route bodies, UI screens, mistpipe subcommand
bodies, real image builds.

backend/Dockerfile

@@ -0,0 +1,35 @@
+# Mist backend image — used by both `api` and `worker` containers.
+# The api container runs uvicorn (the default ENTRYPOINT); the worker container
+# overrides the command to `celery -A mist.worker worker --loglevel=INFO`.
+
+FROM python:3.12-slim AS base
+
+# System deps for py7zr, librsync, hdiff, build tools.
+# hdiffz/hpatchz/rdiff binaries are expected to be installed here at /usr/local/bin.
+# In dev they can be bind-mounted; in prod they're baked in. Placeholder COPY below.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        librsync-dev \
+        rdiff \
+        curl \
+        ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# TODO: copy hdiffz/hpatchz binaries from a known release tarball (build-time stage)
+#       For now, the operator is expected to provide them; or build from source.
+
+WORKDIR /app
+
+COPY pyproject.toml ./pyproject.toml
+COPY src ./src
+COPY alembic.ini ./alembic.ini
+
+RUN pip install --no-cache-dir -e .
+
+ENV PYTHONPATH=/app/src
+ENV PYTHONUNBUFFERED=1
+
+EXPOSE 8000
+
+# Default entrypoint = api. The worker container overrides this in docker-compose.yml.
+CMD ["uvicorn", "mist.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

backend/alembic.ini

@@ -0,0 +1,40 @@
+[alembic]
+script_location = src/mist/db/migrations
+prepend_sys_path = src
+version_path_separator = os
+
+# sqlalchemy.url is set programmatically in env.py from mist.config.settings
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

backend/pyproject.toml

@@ -0,0 +1,65 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "mist"
+version = "0.1.0"
+description = "Mist backend — FastAPI app + Celery worker for the Mist private game distribution platform"
+readme = "../README.md"
+requires-python = ">=3.12"
+license = { text = "Proprietary" }
+authors = [{ name = "Josh" }]
+
+dependencies = [
+    # Web framework
+    "fastapi>=0.115",
+    "uvicorn[standard]>=0.32",
+    "python-multipart>=0.0.12",     # multipart form parsing for builds upload
+    "pydantic>=2.9",
+    "pydantic-settings>=2.6",
+
+    # DB / migrations
+    "sqlalchemy>=2.0",
+    "alembic>=1.13",
+    "psycopg[binary]>=3.2",
+
+    # Auth
+    "argon2-cffi>=23.1",
+    "python-jose[cryptography]>=3.3",
+
+    # Background work
+    "celery>=5.4",
+    "redis>=5.1",
+    "kombu>=5.4",
+
+    # Content / compression
+    "zstandard>=0.23",
+    "py7zr>=0.22",
+
+    # Outbound HTTP
+    "httpx>=0.27",
+    "requests>=2.32",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.3",
+    "pytest-asyncio>=0.24",
+    "ruff>=0.7",
+    "mypy>=1.13",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 110
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "N"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

backend/src/mist/__init__.py

@@ -0,0 +1,10 @@
+"""Mist — private game distribution platform.
+
+Package layout:
+    api/      FastAPI routers + app factory
+    worker/   Celery app + tasks
+    core/     Load-bearing domain logic (delta-patching, manifests, etc.)
+    db/      SQLAlchemy models + Alembic migrations
+"""
+
+__version__ = "0.1.0"

backend/src/mist/api/__init__.py

@@ -0,0 +1 @@
+"""Mist HTTP API — FastAPI routers + app factory."""

backend/src/mist/api/admin.py

@@ -0,0 +1,72 @@
+"""Admin routes — game CRUD, user provisioning, build-job inspection.
+
+Skeleton implementation; all routes require admin scope.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel
+
+from mist.api.deps import require_admin
+from mist.db.models import User
+
+router = APIRouter(dependencies=[Depends(require_admin)])
+
+
+class CreateGameRequest(BaseModel):
+    title: str
+    app_id: int | None = None
+    is_private: bool = False
+
+
+class UpdateGameRequest(BaseModel):
+    description_override: str | None = None
+    header_image_override: str | None = None
+    is_private: bool | None = None
+
+
+class CreateUserRequest(BaseModel):
+    username: str
+    password: str
+    is_admin: bool = False
+
+
+@router.post("/games", status_code=201)
+def create_game(_body: CreateGameRequest, _admin: User = Depends(require_admin)) -> dict:
+    # TODO: create Game row, kick off Steam appdetails fetch if app_id present
+    raise HTTPException(status_code=501, detail="admin.create_game not implemented yet")
+
+
+@router.patch("/games/{game_id}")
+def update_game(game_id: int, _body: UpdateGameRequest, _admin: User = Depends(require_admin)) -> dict:
+    raise HTTPException(status_code=501, detail="admin.update_game not implemented yet")
+
+
+@router.delete("/games/{game_id}", status_code=204)
+def delete_game(game_id: int, _admin: User = Depends(require_admin)) -> None:
+    # TODO: soft-delete (set deleted_at)
+    raise HTTPException(status_code=501, detail="admin.delete_game not implemented yet")
+
+
+@router.post("/games/{game_id}/resync-steam")
+def resync_steam(game_id: int, _admin: User = Depends(require_admin)) -> dict:
+    # TODO: re-fetch Steam appdetails for this game
+    raise HTTPException(status_code=501, detail="admin.resync_steam not implemented yet")
+
+
+@router.post("/users", status_code=201)
+def create_user(_body: CreateUserRequest, _admin: User = Depends(require_admin)) -> dict:
+    # TODO: argon2-hash password, insert User row
+    raise HTTPException(status_code=501, detail="admin.create_user not implemented yet")
+
+
+@router.get("/users")
+def list_users(_admin: User = Depends(require_admin)) -> list[dict]:
+    # TODO: list users
+    raise HTTPException(status_code=501, detail="admin.list_users not implemented yet")
+
+
+@router.get("/build-jobs")
+def list_build_jobs(_admin: User = Depends(require_admin)) -> list[dict]:
+    raise HTTPException(status_code=501, detail="admin.list_build_jobs not implemented yet")

backend/src/mist/api/app.py

@@ -0,0 +1,60 @@
+"""FastAPI app factory.
+
+Run with: `uvicorn mist.api.app:app --host 0.0.0.0 --port 8000`
+"""
+
+from __future__ import annotations
+
+import logging
+from contextlib import asynccontextmanager
+from collections.abc import AsyncIterator
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from mist.api import admin, auth, builds, catalog, downloads
+from mist.config import settings
+from mist.core import paths as core_paths
+
+log = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+    logging.basicConfig(level=settings.log_level)
+    log.info("Mist API starting; environment=%s", settings.environment)
+    core_paths.ensure_dirs()
+    yield
+    log.info("Mist API stopping")
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(title="Mist API", version="0.1.0", lifespan=lifespan)
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.cors_origins or ["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    app.include_router(auth.router, prefix="/auth", tags=["auth"])
+    app.include_router(catalog.router, prefix="/catalog", tags=["catalog"])
+    app.include_router(admin.router, prefix="/admin", tags=["admin"])
+    app.include_router(builds.router, prefix="/builds", tags=["builds"])
+    app.include_router(downloads.router, prefix="/download", tags=["downloads"])
+
+    @app.get("/healthz", tags=["health"])
+    def healthz() -> dict[str, bool]:
+        return {"ok": True}
+
+    @app.get("/readyz", tags=["health"])
+    def readyz() -> dict[str, bool]:
+        # TODO: actually check db / redis / rabbitmq reachability
+        return {"ok": True}
+
+    return app
+
+
+app = create_app()

backend/src/mist/api/auth.py

@@ -0,0 +1,42 @@
+"""Auth routes — login + identity.
+
+Skeleton implementation; argon2 password verify and JWT issuance to be filled in.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel
+
+from mist.api.deps import get_current_user
+from mist.db.models import User
+
+router = APIRouter()
+
+
+class LoginRequest(BaseModel):
+    username: str
+    password: str
+
+
+class LoginResponse(BaseModel):
+    access_token: str
+    token_type: str = "bearer"
+
+
+class MeResponse(BaseModel):
+    id: int
+    username: str
+    is_admin: bool
+
+
+@router.post("/login", response_model=LoginResponse)
+def login(_body: LoginRequest) -> LoginResponse:
+    # TODO: look up user by username, verify password_hash with argon2_cffi,
+    # then mint a JWT with sub=user.id and scope=admin if user.is_admin.
+    raise HTTPException(status_code=501, detail="login not implemented yet")
+
+
+@router.get("/me", response_model=MeResponse)
+def me(user: User = Depends(get_current_user)) -> MeResponse:
+    return MeResponse(id=user.id, username=user.username, is_admin=user.is_admin)

backend/src/mist/api/builds.py

@@ -0,0 +1,74 @@
+"""Build routes — receive uploads from `mistpipe`, query update status, kick off delta-gen.
+
+Skeleton implementation.
+
+NOTE: this is where the fix for the prototype's request_update.py:53 bug applies.
+The old code compared dict values to raw hashes:
+    from_version_manifest[file] != hash      ← wrong (dict vs scalar)
+The correct comparison is on the `checksum` key:
+    from_version_manifest[file]['checksum'] != hash['checksum']
+Implement that pattern when filling in update routing.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, UploadFile
+from pydantic import BaseModel
+
+from mist.api.deps import get_current_user, require_admin
+from mist.db.models import User
+
+router = APIRouter()
+
+
+class PushUpdateRequest(BaseModel):
+    game_title: str
+    version: str
+    app_id: int | None = None
+    patch_notes_url: str | None = None
+
+
+class RequestUpdateResponse(BaseModel):
+    mode: str  # "cache" | "generate_direct" | "generate_indirect"
+    file: str | None = None
+    task_id: str | None = None
+    files_to_signature: list[str] | None = None
+
+
+@router.post("/upload", dependencies=[Depends(require_admin)])
+async def upload(_file: UploadFile, _admin: User = Depends(require_admin)) -> dict:
+    """Receive a full game-version bundle from `mistpipe push`."""
+    # TODO: stream upload to /mnt/nas, queue push_update task, return BuildJob
+    raise HTTPException(status_code=501, detail="builds.upload not implemented yet")
+
+
+@router.get("/jobs/{job_id}")
+def get_job(job_id: int, _admin: User = Depends(require_admin)) -> dict:
+    raise HTTPException(status_code=501, detail="builds.get_job not implemented yet")
+
+
+@router.get("/request-update/{game_title}/{from_version}/{to_version}", response_model=RequestUpdateResponse)
+def request_update(
+    game_title: str, from_version: str, to_version: str, _user: User = Depends(get_current_user)
+) -> RequestUpdateResponse:
+    """Client asks: how do I get from `from_version` to `to_version`?
+
+    Returns one of:
+        - cache hit (file ready to download)
+        - generate direct (server queues a hdiff direct-delta zip task)
+        - generate indirect (client must POST signatures next)
+    """
+    # TODO: implement direct/indirect decision logic; apply request_update.py:53 fix
+    raise HTTPException(status_code=501, detail="builds.request_update not implemented yet")
+
+
+@router.post("/generate-indirect/{game_title}/{from_version}/{to_version}")
+async def generate_indirect_update(
+    game_title: str,
+    from_version: str,
+    to_version: str,
+    _signatures: dict,
+    _user: User = Depends(get_current_user),
+) -> dict:
+    """Client posts a `signatures` dict of {file: hex_signature}; server queues delta-gen."""
+    raise HTTPException(status_code=501, detail="builds.generate_indirect_update not implemented yet")

backend/src/mist/api/catalog.py

@@ -0,0 +1,46 @@
+"""Catalog routes — game browsing for end users.
+
+Skeleton implementation; honors `is_private` filtering once wired.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel
+
+from mist.api.deps import get_current_user
+from mist.db.models import User
+
+router = APIRouter()
+
+
+class GameSummary(BaseModel):
+    id: int
+    title: str
+    app_id: int | None
+    header_image: str | None
+    short_description: str | None
+    is_private: bool
+    latest_version: str | None
+
+
+class GameDetail(GameSummary):
+    versions: list[str]
+
+
+@router.get("/games", response_model=list[GameSummary])
+def list_games(_user: User = Depends(get_current_user)) -> list[GameSummary]:
+    # TODO: list games visible to the current user (public + admin sees private)
+    raise HTTPException(status_code=501, detail="catalog.list_games not implemented yet")
+
+
+@router.get("/games/{game_id}", response_model=GameDetail)
+def get_game(game_id: int, _user: User = Depends(get_current_user)) -> GameDetail:
+    # TODO: fetch single game with version list
+    raise HTTPException(status_code=501, detail="catalog.get_game not implemented yet")
+
+
+@router.get("/games/{game_id}/manifests/{version}")
+def get_manifest(game_id: int, version: str, _user: User = Depends(get_current_user)) -> dict:
+    # TODO: serve per-version manifest JSON
+    raise HTTPException(status_code=501, detail="catalog.get_manifest not implemented yet")

backend/src/mist/api/deps.py

@@ -0,0 +1,61 @@
+"""Shared FastAPI dependencies — DB session, JWT auth, admin gate."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from fastapi import Depends, HTTPException, status
+from fastapi.security import OAuth2PasswordBearer
+from jose import JWTError, jwt
+from sqlalchemy.orm import Session
+
+from mist.config import settings
+from mist.db.base import SessionLocal
+from mist.db.models import User
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/auth/login", auto_error=False)
+
+
+def get_db() -> Iterator[Session]:
+    session = SessionLocal()
+    try:
+        yield session
+    finally:
+        session.close()
+
+
+def _decode_token(token: str) -> dict:
+    try:
+        return jwt.decode(token, settings.jwt_secret, algorithms=[settings.jwt_alg])
+    except JWTError as e:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=f"Invalid token: {e}",
+            headers={"WWW-Authenticate": "Bearer"},
+        ) from e
+
+
+def get_current_user(
+    token: str | None = Depends(oauth2_scheme),
+    db: Session = Depends(get_db),
+) -> User:
+    if not token:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Not authenticated",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    payload = _decode_token(token)
+    user_id = payload.get("sub")
+    if user_id is None:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Token missing sub")
+    user = db.get(User, int(user_id))
+    if user is None:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found")
+    return user
+
+
+def require_admin(user: User = Depends(get_current_user)) -> User:
+    if not user.is_admin:
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only")
+    return user

backend/src/mist/api/downloads.py

@@ -0,0 +1,86 @@
+"""Downloads — resumable HTTP `Range` serving of cached artifacts.
+
+Ported from: JoshSteam CDN/Routes/download.py
+Bug fixes during port:
+    - Original line 2 had a stray backslash continuation breaking the import
+    - Ported from Flask to FastAPI (StreamingResponse + Request.headers)
+    - Hardcoded CACHE_DIR replaced with settings.cache_dir
+    - Auth gate added (any authenticated user can download; private-game checking
+      happens at /catalog level before the client is told what file to fetch)
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi.responses import FileResponse, StreamingResponse
+
+from mist.api.deps import get_current_user
+from mist.config import settings
+from mist.db.models import User
+
+router = APIRouter()
+
+CHUNK_SIZE = 8192
+
+
+def _file_iter(file_path: Path, start: int, end: int) -> Iterator[bytes]:
+    with open(file_path, "rb") as f:
+        f.seek(start)
+        remaining = end - start + 1
+        while remaining > 0:
+            chunk = f.read(min(CHUNK_SIZE, remaining))
+            if not chunk:
+                break
+            yield chunk
+            remaining -= len(chunk)
+
+
+@router.get("/{file}")
+def download(file: str, request: Request, _user: User = Depends(get_current_user)) -> StreamingResponse | FileResponse:
+    """Serve a cached file with HTTP `Range` resume support."""
+    # Defensive: prevent path traversal
+    if "/" in file or "\\" in file or ".." in file:
+        raise HTTPException(status_code=400, detail="Invalid file name")
+
+    file_path = settings.cache_dir / file
+    if not file_path.exists() or not file_path.is_file():
+        raise HTTPException(status_code=404, detail="File not found")
+
+    file_size = file_path.stat().st_size
+    range_header = request.headers.get("range")
+
+    if not range_header:
+        return FileResponse(
+            file_path,
+            media_type="application/zstd",
+            filename=file,
+        )
+
+    # Parse "bytes=START-END" (END optional)
+    try:
+        units, _, rng = range_header.partition("=")
+        if units.strip().lower() != "bytes":
+            raise ValueError("only bytes ranges supported")
+        start_s, _, end_s = rng.strip().partition("-")
+        start = int(start_s)
+        end = int(end_s) if end_s else file_size - 1
+        end = min(end, file_size - 1)
+        if start < 0 or start > end:
+            raise ValueError("bad range")
+    except ValueError:
+        raise HTTPException(status_code=416, detail="Invalid Range header") from None
+
+    headers = {
+        "Content-Range": f"bytes {start}-{end}/{file_size}",
+        "Accept-Ranges": "bytes",
+        "Content-Length": str(end - start + 1),
+    }
+    return StreamingResponse(
+        _file_iter(file_path, start, end),
+        status_code=206,
+        media_type="application/zstd",
+        headers=headers,
+    )

backend/src/mist/config.py

@@ -0,0 +1,69 @@
+"""Mist configuration — every path and secret comes from env vars.
+
+Loaded once at process start. Importable as `from mist.config import settings`.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+
+    # ---- Environment ----
+    environment: str = "development"
+    log_level: str = "INFO"
+
+    # ---- Database ----
+    database_url: str = "postgresql+psycopg://mist:changeme@localhost:5432/mist"
+
+    # ---- Redis ----
+    redis_url: str = "redis://localhost:6379/0"
+
+    # ---- Celery ----
+    celery_broker_url: str = "amqp://mist:changeme@localhost:5672//"
+    celery_result_backend: str = "redis://localhost:6379/1"
+
+    # ---- API ----
+    api_host: str = "0.0.0.0"
+    api_port: int = 8000
+    jwt_secret: str = "change-me"
+    jwt_alg: str = "HS256"
+    jwt_ttl_minutes: int = 60 * 12
+    cors_allowed_origins: str = ""  # comma-separated
+
+    # ---- Paths ----
+    games_dir: Path = Field(default=Path("/mnt/nas/mist/games"))
+    cache_dir: Path = Field(default=Path("/mist/cache"))
+    temp_dir: Path = Field(default=Path("/mist/tmp"))
+
+    # ---- Patch tool binaries ----
+    hdiffz_path: Path = Field(default=Path("/usr/local/bin/hdiffz"))
+    hpatchz_path: Path = Field(default=Path("/usr/local/bin/hpatchz"))
+    rdiff_path: Path = Field(default=Path("/usr/local/bin/rdiff"))
+    unrealpak_path: Path | None = None  # optional; only needed on Unreal-game ingest hosts
+
+    # ---- External integrations ----
+    discord_webhook_url: str | None = None
+    discord_bot_username: str = "Mist"
+    discord_bot_avatar_url: str = (
+        "https://www.pcgamesn.com/wp-content/sites/pcgamesn/2018/10/gabe_newell_meme.jpg"
+    )
+    steam_api_key: str | None = None
+
+    @property
+    def cors_origins(self) -> list[str]:
+        return [o.strip() for o in self.cors_allowed_origins.split(",") if o.strip()]
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    return Settings()
+
+
+settings = get_settings()

backend/src/mist/core/__init__.py

@@ -0,0 +1,13 @@
+"""Mist core — load-bearing domain logic.
+
+These modules are ported from the original Josh Steam prototypes:
+    hdiff         — direct-delta generation via HDiffPatch (`hdiffz`)
+    librsync      — indirect-delta generation via librsync (`rdiff`)
+    chain_replay  — cold reconstruction of arbitrary historical versions
+    manifest      — SHA-256 per-file manifests
+    compression   — zstd streaming compression, 7z extraction
+    discord       — Discord webhook announcements
+    steam         — Steam appdetails pull-through
+    unrealpak     — Unreal Engine .pak extract/repack helper
+    paths         — centralized path resolution (uses settings)
+"""

backend/src/mist/core/chain_replay.py

@@ -0,0 +1,237 @@
+"""Cold reconstruction of arbitrary historical versions via delta-chain replay.
+
+Ported from: JoshSteam CDN/Utils/prepare_game_version.py
+Bug fixes / changes during port:
+    - All hardcoded paths (GAMES_DIR/CACHE_DIR/TEMP_DIR/hdiff_path) replaced with
+      `mist.core.paths` helpers + `settings`
+    - Type hints
+    - Behavior preserved: find closest cached version, copy it, walk forward
+      applying hdiff patches at each step, verify final state, cache result
+
+The key insight: we never need to store every version's full files. The base
+version (.7z) + the delta chain gives us any historical version on demand,
+and the result gets cached for next time.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from mist.config import settings
+from mist.core import paths
+from mist.core.compression import extract_7z
+from mist.core.manifest import verify_files
+
+
+# ---- Version-history helpers (legacy JSON path; callers should prefer the DB) ----
+
+
+def get_base_version(game_title: str) -> str:
+    path = paths.game_version_history_path(game_title)
+    if not path.exists():
+        raise FileNotFoundError(f"Version file for {game_title} not found at {path}")
+    with open(path) as f:
+        versions: list[str] = json.load(f)
+    if not versions:
+        raise ValueError(f"No versions found for {game_title}")
+    return versions[0]
+
+
+def get_versions_up_to(game_title: str, target_version: str) -> list[str]:
+    path = paths.game_version_history_path(game_title)
+    if not path.exists():
+        raise FileNotFoundError(f"Version file for {game_title} not found at {path}")
+    with open(path) as f:
+        versions: list[str] = json.load(f)
+    try:
+        idx = versions.index(target_version)
+    except ValueError:
+        return []
+    return versions[0 : idx + 1]
+
+
+def get_versions_in_range(game_title: str, from_version: str, to_version: str) -> list[str]:
+    path = paths.game_version_history_path(game_title)
+    if not path.exists():
+        raise FileNotFoundError(f"Version file for {game_title} not found at {path}")
+    with open(path) as f:
+        versions: list[str] = json.load(f)
+    try:
+        from_idx = versions.index(from_version)
+        to_idx = versions.index(to_version)
+    except ValueError:
+        return []
+    return versions[from_idx + 1 : to_idx + 1]
+
+
+# ---- Cache lookup ----
+
+
+def find_game_folder_in_cache(game_title: str, version: str) -> bool:
+    """Is the reconstructed-version folder for (title, version) present in the cache?"""
+    return paths.cached_game_version_dir(game_title, version).exists()
+
+
+def find_closest_cached_version(game_title: str, version_list: list[str]) -> str | None:
+    """Of the candidate versions in `version_list`, return the latest one we've already
+    reconstructed and have sitting in cache (if any)."""
+    cache_dir = settings.cache_dir
+    cached_versions: list[str] = []
+    if not cache_dir.exists():
+        return None
+    for folder in os.listdir(cache_dir):
+        if not folder.startswith(game_title):
+            continue
+        folder_version = folder[len(game_title) :].strip()
+        if folder_version in version_list:
+            cached_versions.append(folder_version)
+    if not cached_versions:
+        return None
+    cached_versions.sort(key=lambda v: version_list.index(v))
+    return cached_versions[-1]
+
+
+# ---- Patch application ----
+
+
+def _copy_folder(src: Path, dest: Path) -> None:
+    if not src.exists():
+        raise FileNotFoundError(f"Source folder not found: {src}")
+    shutil.copytree(src, dest)
+
+
+def _apply_hdiff_patch(
+    patch: str, data: dict[str, str], temp_delta_folder: Path, temp_game_dir: Path
+) -> bool:
+    patch_path = temp_delta_folder / patch
+    relative_path = data.get("relative_path")
+    if not relative_path:
+        return False
+    original_file_path = temp_game_dir / relative_path
+    if not original_file_path.exists():
+        return False
+    temp_file_path = temp_delta_folder / f"{patch}.tmp"
+    try:
+        subprocess.run(
+            [str(settings.hpatchz_path), "-f", str(original_file_path), str(patch_path), str(temp_file_path)],
+            check=True,
+        )
+        os.replace(temp_file_path, original_file_path)
+        return True
+    except (subprocess.CalledProcessError, OSError):
+        return False
+
+
+def apply_update(game_title: str, version: str, temp_game_dir: Path) -> None:
+    """Apply the delta_manifest for `version` to the working folder `temp_game_dir`.
+
+    Copies the delta folder out of the NAS into temp_delta_folder, applies in parallel,
+    handles new/deleted files, then deletes the temp_delta_folder.
+    """
+    delta_folder = paths.deltas_dir(game_title, version)
+    temp_delta_folder = settings.temp_dir / f"{game_title} Update" / version
+    if not delta_folder.exists():
+        return
+
+    _copy_folder(delta_folder, temp_delta_folder)
+    with open(temp_delta_folder / "delta_manifest.json") as f:
+        delta_manifest: dict[str, Any] = json.load(f)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
+        futures = [
+            executor.submit(_apply_hdiff_patch, patch, data, temp_delta_folder, temp_game_dir)
+            for patch, data in delta_manifest["modified_files"].items()
+        ]
+        for f in concurrent.futures.as_completed(futures):
+            f.result()
+
+    new_files_folder = temp_delta_folder / "new_files"
+    for new_file in delta_manifest["new_files"]:
+        src = new_files_folder / new_file["file"]
+        dst = temp_game_dir / new_file["relative_path"]
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy(src, dst)
+
+    for deleted in delta_manifest["deleted_files"]:
+        target = temp_game_dir / deleted["relative_path"]
+        if target.exists():
+            target.unlink()
+
+    shutil.rmtree(temp_delta_folder)
+
+
+# ---- Orchestrator ----
+
+
+def prepare_game_version(game_title: str, version: str) -> bool:
+    """Reconstruct (game_title, version) into the cache folder and verify it.
+
+    Strategy:
+        1. If version == base, copy/extract the base archive.
+        2. Otherwise: find the closest cached version, copy it, walk forward.
+        3. If nothing is cached, extract the base and walk forward from there.
+        4. Verify against the per-version manifest. On success, move to cache.
+    """
+    base_version = get_base_version(game_title)
+    update_path: list[str] | None = None
+    temp_game_dir: Path
+
+    if version == base_version:
+        if find_game_folder_in_cache(game_title, base_version):
+            _copy_folder(
+                paths.cached_game_version_dir(game_title, base_version),
+                settings.temp_dir / f"{game_title} {version}",
+            )
+            temp_game_dir = settings.temp_dir / f"{game_title} {version}"
+        else:
+            base_archive = paths.base_archive_path(game_title)
+            temp_game_dir = settings.temp_dir / f"{game_title} {base_version}"
+            if not base_archive.exists():
+                return False
+            temp_game_dir.mkdir(parents=True, exist_ok=True)
+            extract_7z(base_archive, temp_game_dir)
+    else:
+        update_path_from_base = get_versions_up_to(game_title, version)
+        closest = find_closest_cached_version(game_title, update_path_from_base)
+        if closest:
+            _copy_folder(
+                paths.cached_game_version_dir(game_title, closest),
+                settings.temp_dir / f"{game_title} {closest}",
+            )
+            temp_game_dir = settings.temp_dir / f"{game_title} {closest}"
+            update_path = get_versions_in_range(game_title, closest, version)
+        else:
+            base_archive = paths.base_archive_path(game_title)
+            temp_game_dir = settings.temp_dir / f"{game_title} {base_version}"
+            if not base_archive.exists():
+                return False
+            temp_game_dir.mkdir(parents=True, exist_ok=True)
+            extract_7z(base_archive, temp_game_dir)
+            update_path = get_versions_up_to(game_title, version)
+            if update_path:
+                update_path.pop(0)  # we already have the base extracted
+
+    if update_path:
+        for next_version in update_path:
+            apply_update(game_title, next_version, temp_game_dir)
+            new_folder = settings.temp_dir / f"{game_title} {next_version}"
+            os.rename(temp_game_dir, new_folder)
+            temp_game_dir = new_folder
+        update_workspace = settings.temp_dir / f"{game_title} Update"
+        if update_workspace.exists():
+            shutil.rmtree(update_workspace)
+
+    requested_manifest = paths.manifest_path(game_title, version)
+    results = verify_files(temp_game_dir, requested_manifest)
+    if results["all_files_verified"]:
+        output_dir = paths.cached_game_version_dir(game_title, version)
+        shutil.move(str(temp_game_dir), str(output_dir))
+        return True
+    shutil.rmtree(temp_game_dir)
+    return False

backend/src/mist/core/compression.py

@@ -0,0 +1,56 @@
+"""zstd streaming compression + 7z extraction.
+
+Ported from: JoshSteam CDN/Utils/compression.py
+No behavior changes; just type hints and path-objects.
+"""
+
+from __future__ import annotations
+
+import os
+import tarfile
+from pathlib import Path
+from typing import BinaryIO
+
+import py7zr
+import zstandard as zstd
+
+
+def generate_tar_zstd_stream(depot_path: Path | str, output_stream: BinaryIO) -> None:
+    """Stream tarball of `depot_path` contents into an output binary stream."""
+    depot_path = Path(depot_path)
+    with tarfile.open(fileobj=output_stream, mode="w|") as tar:
+        for root, _, files in os.walk(depot_path):
+            for file in files:
+                file_path = Path(root) / file
+                relative_path = file_path.relative_to(depot_path)
+                tar.add(file_path, arcname=str(relative_path))
+
+
+def compress_and_save_zstd(
+    depot_path: Path | str, temp_zstd_path: Path | str, final_zstd_path: Path | str
+) -> None:
+    """Tar+zstd `depot_path` to a temp file, then atomically rename to final.
+
+    No intermediate uncompressed .tar is created on disk.
+    """
+    temp_zstd_path = Path(temp_zstd_path)
+    final_zstd_path = Path(final_zstd_path)
+    temp_zstd_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(temp_zstd_path, "wb") as out_file:
+        cctx = zstd.ZstdCompressor(level=3)
+        with cctx.stream_writer(out_file) as compressor:
+            generate_tar_zstd_stream(depot_path, compressor)
+
+    os.replace(temp_zstd_path, final_zstd_path)
+
+
+def extract_7z(archive_path: Path | str, destination_path: Path | str) -> None:
+    """Extract a .7z archive into `destination_path`."""
+    archive_path = Path(archive_path)
+    destination_path = Path(destination_path)
+    if not archive_path.exists():
+        raise FileNotFoundError(f"The archive {archive_path} does not exist.")
+    destination_path.mkdir(parents=True, exist_ok=True)
+    with py7zr.SevenZipFile(archive_path, mode="r") as archive:
+        archive.extractall(path=destination_path)

backend/src/mist/core/discord.py

@@ -0,0 +1,95 @@
+"""Discord webhook announcements.
+
+Ported from: JoshSteam CDN/Utils/discord.py
+Bug fixes during port:
+    - Webhook URL moved from hardcoded source to settings.discord_webhook_url
+    - Bot username changed from "Josh Steam" to settings.discord_bot_username
+    - Now no-ops gracefully if webhook URL is unset (logs warning)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+import requests
+
+from mist.config import settings
+
+log = logging.getLogger(__name__)
+
+
+def _send(embed: dict[str, Any]) -> None:
+    """Send a single embed to the Discord webhook. No-ops if webhook not configured."""
+    if not settings.discord_webhook_url:
+        log.warning("Discord webhook not configured; skipping send")
+        return
+
+    payload = {
+        "embeds": [embed],
+        "username": settings.discord_bot_username,
+        "avatar_url": settings.discord_bot_avatar_url,
+    }
+
+    try:
+        requests.post(
+            settings.discord_webhook_url,
+            data=json.dumps(payload),
+            headers={"Content-Type": "application/json"},
+            timeout=10,
+        )
+    except requests.RequestException as e:
+        log.warning("Discord webhook send failed: %s", e)
+
+
+def announce_new_game(
+    game_title: str,
+    version: str,
+    game_description: str | None,
+    game_image_url: str | None,
+    game_size: str,
+) -> None:
+    embed: dict[str, Any] = {
+        "title": f"New Game Added:\n{game_title}",
+        "description": game_description or "",
+        "color": 3447003,
+        "fields": [],
+        "footer": {"text": f"Version: {version}\nGame Size: {game_size}"},
+    }
+    if game_image_url:
+        embed["image"] = {"url": game_image_url}
+    _send(embed)
+
+
+def announce_update(
+    game_title: str,
+    version: str,
+    patch_notes_url: str | None,
+    game_description: str | None,
+    game_image_url: str | None,
+    update_size: str,
+) -> None:
+    embed: dict[str, Any] = {
+        "title": f"Update Available:\n{game_title}",
+        "description": game_description or "",
+        "color": 3447003,
+        "fields": [],
+        "footer": {"text": f"Version: {version}\nUpdate Size: {update_size}"},
+    }
+    if game_image_url:
+        embed["image"] = {"url": game_image_url}
+    if patch_notes_url:
+        embed["fields"].append(
+            {"name": "Patch Notes", "value": f"[Click here to read the patch notes]({patch_notes_url})"}
+        )
+    _send(embed)
+
+
+def announce_update_failure(game_title: str, version: str) -> None:
+    embed = {
+        "title": f"Update failed! {game_title} — {version}",
+        "color": 15158332,  # red
+        "fields": [],
+    }
+    _send(embed)

backend/src/mist/core/hdiff.py

@@ -0,0 +1,174 @@
+"""Direct-delta generation via HDiffPatch (`hdiffz`).
+
+Ported from: JoshSteam CDN/Utils/direct_patching.py
+Bug fixes / changes during port:
+    - All hardcoded paths replaced with `mist.config.settings` references
+    - Uses `mist.core.paths` helpers for game-tree layout
+    - Type hints throughout
+    - Behavior preserved: same hdiffz options, same delta_manifest format
+
+A "direct" update is between two consecutive versions whose deltas were
+pre-generated at push time. This module is what generates them.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from mist.config import settings
+from mist.core import paths
+
+
+def _generate_hdiff_patch(old_file: Path, new_file: Path, delta_file: Path) -> bool:
+    """Run hdiffz to produce a single binary patch. Returns True on success."""
+    try:
+        subprocess.run(
+            [
+                str(settings.hdiffz_path),
+                "-SD-256k",  # Single Data, 256KB step
+                "-p-8",  # 8 threads
+                "-s-64k",  # 64KB stream block
+                "-c-lzma2-9",  # LZMA2 level 9
+                str(old_file),
+                str(new_file),
+                str(delta_file),
+            ],
+            check=True,
+        )
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
+def _generate_single_delta(
+    file: str,
+    new_file_info: dict[str, str],
+    old_manifest: dict[str, dict[str, str]],
+    depot_path: Path,
+    import_path: Path,
+    deltas_path: Path,
+    new_files_path: Path,
+) -> dict[str, Any] | bool | None:
+    """Decide per-file action (new / modified / unchanged) and produce artifacts.
+
+    Returns:
+        dict with `type` ∈ {'new', 'modified'} and `file_info` for the delta_manifest,
+        or None if the file is unchanged,
+        or False if delta generation failed (caller should abort).
+    """
+    old_info = old_manifest.get(file)
+    new_checksum = new_file_info["checksum"]
+
+    # Unchanged
+    if old_info and old_info["checksum"] == new_checksum:
+        return None
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+    file_basename = os.path.basename(new_file_info["path"])
+
+    # New file
+    if old_info is None:
+        src = import_path / new_file_info["path"]
+        dest = new_files_path / f"{file_basename}_{timestamp}"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, dest)
+        return {
+            "type": "new",
+            "file_info": {
+                "file": dest.name,
+                "original_filename": file_basename,
+                "relative_path": new_file_info["path"],
+                "checksum": new_checksum,
+            },
+        }
+
+    # Modified file
+    old_file_path = depot_path / old_info["path"]
+    new_file_path = import_path / new_file_info["path"]
+    delta_filename = f"{file.replace(os.sep, '_').replace('/', '_')}_{timestamp}.patch"
+    delta_path = deltas_path / delta_filename
+
+    if _generate_hdiff_patch(old_file_path, new_file_path, delta_path):
+        return {
+            "type": "modified",
+            "file_info": {"file": delta_filename, "relative_path": new_file_info["path"]},
+        }
+    return False
+
+
+def generate_delta_patches(
+    game_title: str, old_version: str, new_version: str, import_path: Path | str
+) -> bool:
+    """Generate hdiff patches + delta_manifest.json for old_version -> new_version.
+
+    Reads manifests from the NAS, writes patches under deltas/<new_version>/.
+    """
+    import_path = Path(import_path)
+    depot_path = paths.depot_dir(game_title)
+    old_manifest_path = paths.manifest_path(game_title, old_version)
+    new_manifest_path = paths.manifest_path(game_title, new_version)
+    deltas_path = paths.deltas_dir(game_title, new_version)
+    new_files_path = paths.new_files_dir(game_title, new_version)
+
+    deltas_path.mkdir(parents=True, exist_ok=True)
+    new_files_path.mkdir(parents=True, exist_ok=True)
+
+    with open(old_manifest_path) as f:
+        old_manifest: dict[str, dict[str, str]] = json.load(f)
+    with open(new_manifest_path) as f:
+        new_manifest: dict[str, dict[str, str]] = json.load(f)
+
+    delta_manifest: dict[str, Any] = {
+        "modified_files": {},
+        "new_files": [],
+        "deleted_files": [],
+    }
+
+    with ThreadPoolExecutor(max_workers=6) as executor:
+        futures = [
+            executor.submit(
+                _generate_single_delta,
+                file,
+                data,
+                old_manifest,
+                depot_path,
+                import_path,
+                deltas_path,
+                new_files_path,
+            )
+            for file, data in new_manifest.items()
+        ]
+        for future in as_completed(futures):
+            result = future.result()
+            if result is False:
+                executor.shutdown(wait=False, cancel_futures=True)
+                return False
+            if result is None:
+                continue
+            if result["type"] == "new":
+                delta_manifest["new_files"].append(result["file_info"])
+            else:
+                delta_manifest["modified_files"][result["file_info"]["file"]] = result["file_info"]
+
+    # Deletions: present in old, absent in new
+    for file in old_manifest:
+        if file not in new_manifest:
+            delta_manifest["deleted_files"].append(
+                {
+                    "file": os.path.basename(old_manifest[file]["path"]),
+                    "relative_path": old_manifest[file]["path"],
+                    "checksum": old_manifest[file]["checksum"],
+                }
+            )
+
+    with open(paths.delta_manifest_path(game_title, new_version), "w") as f:
+        json.dump(delta_manifest, f, indent=4)
+
+    return True

backend/src/mist/core/librsync.py

@@ -0,0 +1,88 @@
+"""Indirect-delta generation via librsync (`rdiff`).
+
+Ported from: JoshSteam CDN/Utils/indirect_patching.py
+Bug fixes / changes during port:
+    - Hardcoded rdiff.exe path replaced with settings.rdiff_path
+    - Hardcoded TEMP_DIR replaced with settings.temp_dir
+    - Type hints
+    - Behavior preserved: hex-encoded signatures over the wire, .dlt deltas
+
+An "indirect" update is between arbitrary versions where the deltas were
+NOT pre-generated. Flow:
+    1. Client sends librsync `signature` of each file it has (hex-encoded)
+    2. Server `generate_delta(file, signature_hex, ...)` produces a .dlt
+    3. Client applies the .dlt with `rdiff patch`
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from datetime import datetime
+from pathlib import Path
+
+from mist.config import settings
+
+
+def copy_new_file(new_file: str, new_files_folder: Path | str, game_dir: Path | str) -> dict[str, str]:
+    """Copy a new (added) file into the new_files/ side-channel folder."""
+    new_files_folder = Path(new_files_folder)
+    game_dir = Path(game_dir)
+
+    new_file_path = game_dir / new_file
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+    file_basename = os.path.basename(new_file)
+    new_dest_path = new_files_folder / f"{file_basename}_{timestamp}"
+
+    new_dest_path.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(new_file_path, new_dest_path)
+
+    return {
+        "file": new_dest_path.name,
+        "original_filename": file_basename,
+        "relative_path": new_file,
+    }
+
+
+def generate_delta(
+    file: str,
+    signature_hex: str,
+    deltas_folder: Path | str,
+    from_version: str,
+    game_dir: Path | str,
+) -> dict[str, str]:
+    """Produce a librsync delta for `file` given the client's signature.
+
+    Writes the hex-decoded signature to a temp .sig, runs `rdiff delta`,
+    deletes the temp .sig, returns the delta metadata.
+    """
+    deltas_folder = Path(deltas_folder)
+    game_dir = Path(game_dir)
+
+    file_basename = os.path.basename(file)
+    signature_file = settings.temp_dir / f"{file_basename} {from_version}.sig"
+    settings.temp_dir.mkdir(parents=True, exist_ok=True)
+
+    local_file = game_dir / file
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+    delta_filename = f"{file_basename}_{timestamp}.dlt"
+    delta_file = deltas_folder / delta_filename
+
+    with open(signature_file, "wb") as f:
+        f.write(bytes.fromhex(signature_hex))
+
+    try:
+        subprocess.run(
+            [str(settings.rdiff_path), "delta", str(signature_file), str(local_file), str(delta_file)],
+            check=True,
+        )
+    finally:
+        if signature_file.exists():
+            signature_file.unlink()
+
+    return {
+        "file": delta_filename,
+        "original_filename": file_basename,
+        "relative_path": file,
+    }

backend/src/mist/core/manifest.py

@@ -0,0 +1,141 @@
+"""Per-file SHA-256 manifests + per-game version-history helpers.
+
+Ported from: JoshSteam CDN/Utils/manifest.py
+Bug fixes during port:
+    - Hardcoded GAMES_DIR removed; uses `mist.core.paths` instead.
+
+NOTE: the version-history-as-JSON functions (`update_game_version_manifest`,
+`get_latest_version_from_game_manifest`, `is_direct_update`) are kept as
+**legacy fallbacks**. The Postgres `Version` model is the new source of
+truth. Callsites are expected to migrate; the JSON path remains for
+emergency manual operations on the NAS.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from mist.core import paths
+
+
+def file_checksum(file_path: Path | str) -> str:
+    """SHA-256 of a file, computed in 4KB chunks."""
+    sha256_hash = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        for byte_block in iter(lambda: f.read(4096), b""):
+            sha256_hash.update(byte_block)
+    return sha256_hash.hexdigest()
+
+
+def generate_manifest(game_title: str, version: str, depot_path: Path | str) -> Path:
+    """Walk `depot_path`, compute SHA-256 for every file, write to manifests/<version>.json."""
+    depot_path = Path(depot_path)
+    manifest: dict[str, dict[str, str]] = {}
+    for root, _, files in __import__("os").walk(depot_path):
+        for file in files:
+            file_path = Path(root) / file
+            checksum = file_checksum(file_path)
+            relative_path = str(file_path.relative_to(depot_path))
+            manifest[relative_path] = {"checksum": checksum, "path": relative_path}
+
+    out_path = paths.manifest_path(game_title, version)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w") as f:
+        json.dump(manifest, f, indent=4)
+    return out_path
+
+
+def load_manifest(manifest_file: Path | str) -> dict[str, dict[str, str]]:
+    """Load a per-version manifest file."""
+    with open(manifest_file) as f:
+        return json.load(f)
+
+
+# ---- Legacy version-history JSON helpers (preserved for manual ops) ----
+
+
+def update_game_version_manifest(game_title: str, version: str) -> None:
+    """Append a version to the per-game version-history JSON. Legacy."""
+    history_file = paths.game_version_history_path(game_title)
+    if history_file.exists():
+        with open(history_file) as f:
+            history: list[str] = json.load(f)
+    else:
+        history = []
+    history.append(version)
+    history_file.parent.mkdir(parents=True, exist_ok=True)
+    with open(history_file, "w") as f:
+        json.dump(history, f, indent=4)
+
+
+def get_latest_version_from_game_manifest(game_title: str) -> str | None:
+    """Last entry in the legacy version-history JSON."""
+    history_file = paths.game_version_history_path(game_title)
+    if not history_file.exists():
+        return None
+    with open(history_file) as f:
+        versions: list[str] = json.load(f)
+    return versions[-1] if versions else None
+
+
+def get_earliest_version_from_game_manifest(game_title: str) -> str | None:
+    """First entry in the legacy version-history JSON."""
+    history_file = paths.game_version_history_path(game_title)
+    if not history_file.exists():
+        return None
+    with open(history_file) as f:
+        versions: list[str] = json.load(f)
+    return versions[0] if versions else None
+
+
+def is_direct_update(game_title: str, version1: str, version2: str) -> bool:
+    """True iff version1 is the immediate predecessor of version2. Legacy."""
+    history_file = paths.game_version_history_path(game_title)
+    if not history_file.exists():
+        return False
+    with open(history_file) as f:
+        versions: list[str] = json.load(f)
+    if version1 not in versions or version2 not in versions:
+        return False
+    return versions.index(version1) + 1 == versions.index(version2)
+
+
+# ---- Manifest verification ----
+
+
+def verify_files(game_dir: Path | str, manifest_file: Path | str) -> dict[str, Any]:
+    """Verify every file under `game_dir` against the SHA-256s in `manifest_file`.
+
+    Returns a dict with `missing_files`, `mismatched_checksums`, `all_files_verified`.
+    """
+    game_dir = Path(game_dir)
+    manifest_file = Path(manifest_file)
+    if not manifest_file.exists():
+        raise FileNotFoundError(f"Manifest not found at {manifest_file}")
+
+    with open(manifest_file) as f:
+        manifest_data: dict[str, dict[str, str]] = json.load(f)
+
+    results: dict[str, Any] = {
+        "missing_files": [],
+        "mismatched_checksums": [],
+        "all_files_verified": True,
+    }
+
+    for _, file_info in manifest_data.items():
+        file_path = game_dir / file_info["path"]
+        if not file_path.exists():
+            results["missing_files"].append(str(file_path))
+            results["all_files_verified"] = False
+            continue
+        actual = file_checksum(file_path)
+        if actual != file_info["checksum"]:
+            results["mismatched_checksums"].append(
+                {"file": str(file_path), "expected": file_info["checksum"], "actual": actual}
+            )
+            results["all_files_verified"] = False
+
+    return results

backend/src/mist/core/paths.py

@@ -0,0 +1,82 @@
+"""Centralized path resolution.
+
+Every callsite that needs to know where a game file lives goes through here.
+No path is hardcoded anywhere else in the backend.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from mist.config import settings
+
+
+def game_dir(title: str) -> Path:
+    """Root folder for a single game on the NAS."""
+    return settings.games_dir / title
+
+
+def depot_dir(title: str) -> Path:
+    """Current latest-version files for a game."""
+    return game_dir(title) / "depot"
+
+
+def base_archive_path(title: str) -> Path:
+    """Immutable original .7z that all chain-replay starts from."""
+    return game_dir(title) / "base_version.7z"
+
+
+def manifests_dir(title: str) -> Path:
+    return game_dir(title) / "manifests"
+
+
+def manifest_path(title: str, version: str) -> Path:
+    """Per-version SHA-256 file manifest."""
+    return manifests_dir(title) / f"{version}.json"
+
+
+def game_version_history_path(title: str) -> Path:
+    """Ordered list of versions for a game (legacy JSON; migrating to DB)."""
+    return manifests_dir(title) / f"{title}.json"
+
+
+def deltas_dir(title: str, to_version: str) -> Path:
+    """Pre-generated direct-update delta artifacts for `to_version`."""
+    return game_dir(title) / "deltas" / to_version
+
+
+def delta_manifest_path(title: str, to_version: str) -> Path:
+    return deltas_dir(title, to_version) / "delta_manifest.json"
+
+
+def new_files_dir(title: str, to_version: str) -> Path:
+    return deltas_dir(title, to_version) / "new_files"
+
+
+def cache_archive_path(title: str, version: str) -> Path:
+    """Ready-to-serve full-game .tar.zst archive in the hot cache."""
+    return settings.cache_dir / f"{title} {version}.tar.zst"
+
+
+def cache_direct_update_path(title: str, from_version: str, to_version: str) -> Path:
+    return settings.cache_dir / f"{title}_{from_version}_{to_version}_Direct.tar.zst"
+
+
+def cache_indirect_update_path(title: str, from_version: str, to_version: str) -> Path:
+    return settings.cache_dir / f"{title}_{from_version}_{to_version}_Indirect.tar.zst"
+
+
+def cached_game_version_dir(title: str, version: str) -> Path:
+    """Reconstructed-historical-version folder living in cache."""
+    return settings.cache_dir / f"{title} {version}"
+
+
+def temp_game_dir(title: str, version: str) -> Path:
+    """Scratch folder for chain-replay work."""
+    return settings.temp_dir / f"{title} {version}"
+
+
+def ensure_dirs() -> None:
+    """Create the standard cache + temp dirs if missing. NAS is assumed pre-mounted."""
+    settings.cache_dir.mkdir(parents=True, exist_ok=True)
+    settings.temp_dir.mkdir(parents=True, exist_ok=True)

backend/src/mist/core/steam.py

@@ -0,0 +1,50 @@
+"""Steam appdetails pull-through.
+
+Ported from: JoshSteam CDN/Utils/steam_api.py
+The Steam Storefront API is public and unauthenticated. We optionally pass
+`settings.steam_api_key` if present (used elsewhere in the Steam ecosystem
+for higher rate limits, harmless to include).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import requests
+
+from mist.config import settings
+
+
+def _get_steam_app_details(app_id: int | str) -> dict[str, Any] | None:
+    """Fetch raw appdetails for an app_id. Returns None on any failure."""
+    url = f"https://store.steampowered.com/api/appdetails?appids={app_id}"
+    params: dict[str, str] = {}
+    if settings.steam_api_key:
+        params["key"] = settings.steam_api_key
+
+    try:
+        response = requests.get(url, params=params, timeout=10)
+    except requests.RequestException:
+        return None
+
+    if response.status_code != 200:
+        return None
+
+    data = response.json()
+    payload = data.get(str(app_id))
+    if not payload or not payload.get("success"):
+        return None
+    return payload.get("data")
+
+
+def get_steam_app_info(app_id: int | str) -> tuple[str | None, str | None]:
+    """Return (short_description, header_image_url) for an app_id.
+
+    Returns (None, None) if Steam doesn't know about the app or the call failed.
+    """
+    details = _get_steam_app_details(app_id)
+    if not details:
+        return None, None
+    short_description = (details.get("short_description") or "").replace(""", '"') or None
+    header_image = details.get("header_image") or None
+    return short_description, header_image

backend/src/mist/core/unrealpak.py

@@ -0,0 +1,133 @@
+"""Unreal Engine .pak extract / repack helper.
+
+Ported from: Pak City/test.py (+ find_pak_file.py, find_dotpack_folders.py)
+
+Purpose: Unreal Engine ships massive Oodle-compressed .pak files. Running
+delta tools against them produces garbage ratios (input already entropy-maxed)
+and treats every byte as changed on any modification. The fix is to extract
+.pak files larger than a threshold before delta-gen, treat the contents as
+loose files, then repack on the client after install/update.
+
+Flow:
+    1. find_large_pak_files(game_dir) — scan for .pak >= 64MB
+    2. extract_pak(unrealpak_path, pak_file, extracted_dir) — UnrealPak -Extract
+    3. delete the original .pak; folder is renamed *DOTpak so we know to repack
+    4. generate_data_to_pack(extracted_dir, data_to_pack_file) — generate dataToPack manifest
+    5. (later, on client) repack_pak(...) using Oodle compression
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from mist.config import settings
+
+DEFAULT_PAK_SIZE_LIMIT_MB = 64
+
+
+def _unrealpak() -> Path:
+    """Return the configured UnrealPak.exe path, raising if not set."""
+    if settings.unrealpak_path is None:
+        raise RuntimeError(
+            "UNREALPAK_PATH is not set; this host cannot extract/repack Unreal .pak files."
+        )
+    return settings.unrealpak_path
+
+
+def find_large_pak_files(directory: Path | str, size_limit_mb: int = DEFAULT_PAK_SIZE_LIMIT_MB) -> list[Path]:
+    """Recursively find every .pak file >= size_limit_mb under `directory`."""
+    directory = Path(directory)
+    size_limit_bytes = size_limit_mb * 1024 * 1024
+    found: list[Path] = []
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if not file.lower().endswith(".pak"):
+                continue
+            file_path = Path(root) / file
+            try:
+                if file_path.stat().st_size > size_limit_bytes:
+                    found.append(file_path)
+            except OSError:
+                continue
+    return found
+
+
+def find_dotpak_folders(directory: Path | str) -> list[Path]:
+    """Find every folder marked as an extracted .pak (suffix 'DOTpak')."""
+    directory = Path(directory)
+    found: list[Path] = []
+    for root, dirs, _ in os.walk(directory):
+        for d in dirs:
+            if d.endswith("DOTpak"):
+                found.append(Path(root) / d)
+    return found
+
+
+def _run(args: list[str]) -> str:
+    """Run a subprocess command and return its stdout, raising on nonzero exit."""
+    result = subprocess.run(args, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise RuntimeError(f"Command failed: {' '.join(args)}\n{result.stderr}")
+    return result.stdout
+
+
+def extract_pak(pak_file: Path | str, output_dir: Path | str) -> None:
+    """Extract a .pak file into output_dir using UnrealPak -Extract."""
+    pak_file = Path(pak_file)
+    output_dir = Path(output_dir)
+    if output_dir.exists():
+        shutil.rmtree(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    _run([str(_unrealpak()), str(pak_file), "-Extract", str(output_dir)])
+
+
+def generate_data_to_pack(input_dir: Path | str, output_file: Path | str) -> None:
+    """Write a UnrealPak dataToPack.txt manifest for repacking."""
+    input_dir = Path(input_dir)
+    output_file = Path(output_file)
+    with open(output_file, "w") as f:
+        for root, _, files in os.walk(input_dir):
+            if not files:
+                continue
+            for file in files:
+                if file == output_file.name:
+                    continue
+                file_path = Path(root) / file
+                relative_path = file_path.relative_to(input_dir)
+                target_dir = str(relative_path.parent).replace("\\", "/")
+                f.write(f'"{file_path}" "../../../{target_dir}/"\n')
+
+
+def repack_pak(new_pak_file: Path | str, data_to_pack_file: Path | str) -> None:
+    """Repack a folder back into a .pak with Oodle compression."""
+    new_pak_file = Path(new_pak_file)
+    data_to_pack_file = Path(data_to_pack_file)
+    _run(
+        [
+            str(_unrealpak()),
+            str(new_pak_file),
+            f"-Create={data_to_pack_file}",
+            "-compress",
+            "-compressionformat=Oodle",
+        ]
+    )
+
+
+def extract_all_large_paks(game_dir: Path | str, size_limit_mb: int = DEFAULT_PAK_SIZE_LIMIT_MB) -> list[Path]:
+    """Convenience: walk a game dir, extract every large .pak, delete originals.
+
+    Returns the list of `*DOTpak/` folders created.
+    """
+    game_dir = Path(game_dir)
+    extracted: list[Path] = []
+    for pak_file in find_large_pak_files(game_dir, size_limit_mb):
+        dir_path = pak_file.parent
+        base_name = pak_file.name.replace(".pak", "DOTpak")
+        extracted_dir = dir_path / base_name
+        extract_pak(pak_file, extracted_dir)
+        pak_file.unlink()
+        extracted.append(extracted_dir)
+    return extracted

backend/src/mist/db/__init__.py

@@ -0,0 +1 @@
+"""Mist database layer — SQLAlchemy models, session factory, Alembic migrations."""

backend/src/mist/db/base.py

@@ -0,0 +1,27 @@
+"""SQLAlchemy engine, session factory, declarative base."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
+
+from mist.config import settings
+
+
+class Base(DeclarativeBase):
+    """Shared declarative base for every model in mist.db.models."""
+
+
+engine = create_engine(settings.database_url, pool_pre_ping=True, future=True)
+SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
+
+
+def get_session() -> Iterator[Session]:
+    """FastAPI dep — yields a Session, closes on request teardown."""
+    session = SessionLocal()
+    try:
+        yield session
+    finally:
+        session.close()

backend/src/mist/db/migrations/env.py

@@ -0,0 +1,48 @@
+"""Alembic environment — pulls the DB URL from mist.config and the metadata from mist.db.models."""
+
+from __future__ import annotations
+
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+from mist.config import settings
+from mist.db.base import Base
+from mist.db import models  # noqa: F401 — import side-effect registers tables on Base.metadata
+
+config = context.config
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+config.set_main_option("sqlalchemy.url", settings.database_url)
+
+
+def run_migrations_offline() -> None:
+    context.configure(
+        url=settings.database_url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

backend/src/mist/db/migrations/script.py.mako

@@ -0,0 +1,28 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: str | None = ${repr(down_revision)}
+branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
+depends_on: str | Sequence[str] | None = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

backend/src/mist/db/migrations/versions/0001_initial.py

@@ -0,0 +1,91 @@
+"""Initial schema — users, games, versions, build_jobs.
+
+Revision ID: 0001
+Revises:
+Create Date: 2026-06-07
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0001"
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "users",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("username", sa.String(64), nullable=False, unique=True),
+        sa.Column("password_hash", sa.String(255), nullable=False),
+        sa.Column("is_admin", sa.Boolean(), nullable=False, server_default=sa.text("false")),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+    )
+    op.create_index("ix_users_username", "users", ["username"], unique=True)
+
+    op.create_table(
+        "games",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("title", sa.String(255), nullable=False, unique=True),
+        sa.Column("app_id", sa.Integer(), nullable=True),
+        sa.Column("description_override", sa.Text(), nullable=True),
+        sa.Column("header_image_override", sa.String(1024), nullable=True),
+        sa.Column("is_private", sa.Boolean(), nullable=False, server_default=sa.text("false")),
+        sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+    )
+    op.create_index("ix_games_title", "games", ["title"], unique=True)
+    op.create_index("ix_games_is_private", "games", ["is_private"])
+    op.create_index("ix_games_deleted_at", "games", ["deleted_at"])
+
+    op.create_table(
+        "versions",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("game_id", sa.Integer(), sa.ForeignKey("games.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("version_string", sa.String(64), nullable=False),
+        sa.Column("ordinal", sa.Integer(), nullable=False),
+        sa.Column("manifest_hash", sa.String(64), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        sa.UniqueConstraint("game_id", "version_string", name="uq_versions_game_id_version_string"),
+    )
+    op.create_index("ix_versions_game_id_ordinal", "versions", ["game_id", "ordinal"])
+
+    build_job_kind = sa.Enum(
+        "import_new_game",
+        "push_update",
+        "generate_direct_update",
+        "generate_indirect_update",
+        "prepare_full_game",
+        name="buildjobkind",
+    )
+    build_job_state = sa.Enum("pending", "running", "success", "failure", name="buildjobstate")
+
+    op.create_table(
+        "build_jobs",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("celery_task_id", sa.String(128), nullable=True),
+        sa.Column("game_id", sa.Integer(), sa.ForeignKey("games.id", ondelete="SET NULL"), nullable=True),
+        sa.Column("kind", build_job_kind, nullable=False),
+        sa.Column("state", build_job_state, nullable=False, server_default="pending"),
+        sa.Column("detail", sa.Text(), nullable=True),
+        sa.Column("error", sa.Text(), nullable=True),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+    )
+    op.create_index("ix_build_jobs_celery_task_id", "build_jobs", ["celery_task_id"])
+
+
+def downgrade() -> None:
+    op.drop_table("build_jobs")
+    op.execute("DROP TYPE IF EXISTS buildjobstate")
+    op.execute("DROP TYPE IF EXISTS buildjobkind")
+    op.drop_table("versions")
+    op.drop_table("games")
+    op.drop_table("users")

backend/src/mist/db/models.py

@@ -0,0 +1,113 @@
+"""Mist data model.
+
+Four tables for MVP:
+    users        — accounts (provisioned by admin)
+    games        — catalog entries, one per game
+    versions     — per-game ordered version history (replaces legacy <title>.json)
+    build_jobs   — async work tracking for delta-gen, archive prep, etc.
+"""
+
+from __future__ import annotations
+
+import enum
+from datetime import datetime
+
+from sqlalchemy import (
+    Boolean,
+    DateTime,
+    Enum,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Text,
+    UniqueConstraint,
+    func,
+)
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from mist.db.base import Base
+
+
+class User(Base):
+    __tablename__ = "users"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    username: Mapped[str] = mapped_column(String(64), unique=True, nullable=False, index=True)
+    password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
+    is_admin: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+
+
+class Game(Base):
+    __tablename__ = "games"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    title: Mapped[str] = mapped_column(String(255), unique=True, nullable=False, index=True)
+    app_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    description_override: Mapped[str | None] = mapped_column(Text, nullable=True)
+    header_image_override: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    is_private: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False, index=True)
+    deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+
+    versions: Mapped[list[Version]] = relationship(
+        back_populates="game", cascade="all, delete-orphan", order_by="Version.ordinal"
+    )
+
+
+class Version(Base):
+    __tablename__ = "versions"
+    __table_args__ = (
+        UniqueConstraint("game_id", "version_string", name="uq_versions_game_id_version_string"),
+        Index("ix_versions_game_id_ordinal", "game_id", "ordinal"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    game_id: Mapped[int] = mapped_column(ForeignKey("games.id", ondelete="CASCADE"), nullable=False)
+    version_string: Mapped[str] = mapped_column(String(64), nullable=False)
+    ordinal: Mapped[int] = mapped_column(Integer, nullable=False)
+    manifest_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+
+    game: Mapped[Game] = relationship(back_populates="versions")
+
+
+class BuildJobKind(str, enum.Enum):
+    IMPORT_NEW_GAME = "import_new_game"
+    PUSH_UPDATE = "push_update"
+    GENERATE_DIRECT_UPDATE = "generate_direct_update"
+    GENERATE_INDIRECT_UPDATE = "generate_indirect_update"
+    PREPARE_FULL_GAME = "prepare_full_game"
+
+
+class BuildJobState(str, enum.Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    SUCCESS = "success"
+    FAILURE = "failure"
+
+
+class BuildJob(Base):
+    __tablename__ = "build_jobs"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    celery_task_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
+    game_id: Mapped[int | None] = mapped_column(ForeignKey("games.id", ondelete="SET NULL"), nullable=True)
+    kind: Mapped[BuildJobKind] = mapped_column(Enum(BuildJobKind), nullable=False)
+    state: Mapped[BuildJobState] = mapped_column(
+        Enum(BuildJobState), nullable=False, default=BuildJobState.PENDING
+    )
+    detail: Mapped[str | None] = mapped_column(Text, nullable=True)
+    error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )

backend/src/mist/worker/__init__.py

@@ -0,0 +1,56 @@
+"""Celery app factory.
+
+Reconstructed from the missing JoshSteam CDN/Tasks/__init__.py source.
+Bug fixes during port:
+    - The prototype's `check_if_task_exists` had `celery.control.inspect.reserved()`
+      (missing parens) and assigned `inspect().active()` to both `active_tasks` and
+      `reserved_tasks`. Both fixed in `task_already_queued` below.
+
+Run the worker with:
+    celery -A mist.worker worker --loglevel=INFO
+"""
+
+from __future__ import annotations
+
+from celery import Celery
+
+from mist.config import settings
+
+
+def make_celery() -> Celery:
+    app = Celery(
+        "mist",
+        broker=settings.celery_broker_url,
+        backend=settings.celery_result_backend,
+        include=["mist.worker.tasks", "mist.worker.notifications"],
+    )
+    app.conf.update(
+        task_track_started=True,
+        task_acks_late=True,
+        worker_prefetch_multiplier=1,
+        broker_connection_retry_on_startup=True,
+    )
+    return app
+
+
+celery_app = make_celery()
+
+
+def task_already_queued(task_id: str) -> bool:
+    """Is there already an active or reserved task with this `task_id`?
+
+    Fixed version of the prototype's check_if_task_exists.
+    """
+    inspector = celery_app.control.inspect()
+    active = inspector.active() or {}
+    reserved = inspector.reserved() or {}
+
+    for _, tasks in active.items():
+        for t in tasks:
+            if t.get("id") == task_id:
+                return True
+    for _, tasks in reserved.items():
+        for t in tasks:
+            if t.get("id") == task_id:
+                return True
+    return False

backend/src/mist/worker/notifications.py

@@ -0,0 +1,16 @@
+"""Notification fan-out — consumes events, dispatches to Discord (and future channels).
+
+Skeleton: real implementation will subscribe to a topic on RabbitMQ and call
+mist.core.discord.* for each event type. For MVP, push_update / import_new_game
+can call core.discord directly without going through this layer.
+"""
+
+from __future__ import annotations
+
+from mist.worker import celery_app
+
+
+@celery_app.task(name="mist.notifications.dispatch")
+def dispatch(_event_type: str, _payload: dict) -> None:
+    """Route a domain event to its notification channel(s)."""
+    raise NotImplementedError("notifications.dispatch not implemented yet")

backend/src/mist/worker/tasks.py

@@ -0,0 +1,63 @@
+"""Celery task definitions.
+
+Each task is a thin wrapper around `mist.core.*` functions. The patching IP
+lives in `core/`; this file is just plumbing.
+"""
+
+from __future__ import annotations
+
+from mist.worker import celery_app
+
+
+@celery_app.task(name="mist.tasks.import_new_game")
+def import_new_game(_data: dict) -> dict:
+    """Initial import of a game's base version from an uploaded archive.
+
+    Expected flow once implemented:
+        1. Move uploaded archive to NAS as base_version.7z
+        2. Extract to depot/
+        3. Generate per-version manifest via core.manifest.generate_manifest
+        4. Pre-cache the full-game .tar.zst via core.compression.compress_and_save_zstd
+        5. Insert Game + Version rows; call core.discord.announce_new_game
+    """
+    raise NotImplementedError("import_new_game not implemented yet")
+
+
+@celery_app.task(name="mist.tasks.push_update")
+def push_update(_data: dict) -> dict:
+    """Receive a new version's full files, generate per-version manifest +
+    hdiff direct deltas vs previous version, swap depot.
+
+    Replaces the prototype's push_update task. The chain is:
+        core.manifest.generate_manifest
+        core.hdiff.generate_delta_patches
+        (move new files into depot/)
+        Version row insert
+        core.discord.announce_update
+    """
+    raise NotImplementedError("push_update not implemented yet")
+
+
+@celery_app.task(name="mist.tasks.generate_direct_update")
+def generate_direct_update(_game_title: str, _from_version: str, _to_version: str) -> dict:
+    """Pack the pre-built deltas for a direct (consecutive) update into a .tar.zst."""
+    raise NotImplementedError("generate_direct_update not implemented yet")
+
+
+@celery_app.task(name="mist.tasks.generate_indirect_update")
+def generate_indirect_update(
+    _game_title: str, _from_version: str, _to_version: str, _signatures_path: str
+) -> dict:
+    """Generate librsync .dlt deltas based on client-provided signatures, pack into .tar.zst.
+
+    Replaces the prototype's Tasks/generate_indirect_update.py. Calls
+    core.librsync.generate_delta per file. May call core.chain_replay.prepare_game_version
+    if `to_version` is not the latest and not cached.
+    """
+    raise NotImplementedError("generate_indirect_update not implemented yet")
+
+
+@celery_app.task(name="mist.tasks.prepare_full_game_archive")
+def prepare_full_game_archive(_game_title: str, _version: str) -> dict:
+    """Reconstruct (if needed) and pack `(game_title, version)` as a .tar.zst into cache."""
+    raise NotImplementedError("prepare_full_game_archive not implemented yet")

backend/tests/conftest.py

@@ -0,0 +1 @@
+"""Test configuration. Empty for now — present so pytest discovers the tests/ dir."""

backend/tests/test_smoke.py

@@ -0,0 +1,26 @@
+"""Smoke test: every module imports cleanly.
+
+If a dep is missing, a syntax error slipped through, or a circular import got
+introduced, this test fails. Cheap insurance.
+"""
+
+
+def test_core_modules_import() -> None:
+    from mist.core import (  # noqa: F401
+        chain_replay,
+        compression,
+        discord,
+        hdiff,
+        librsync,
+        manifest,
+        paths,
+        steam,
+        unrealpak,
+    )
+
+
+def test_config_loads() -> None:
+    from mist.config import settings
+
+    assert settings.environment in {"development", "staging", "production"} or settings.environment
+    # don't assert specific values; just that it loaded