commit bfd6771a9abd56a68ac056211872c7ad287ad7fe Author: Josh Date: Sun Jun 7 19:39:25 2026 -0400 Initial Mist scaffold Successor to the Josh Steam prototypes. Single-VM Docker Compose stack with the load-bearing core/ logic ported from JoshSteam CDN with bug fixes. Contents: - backend/ FastAPI + Celery (same image, two entrypoints) core/ hdiff, librsync, chain_replay, manifest, compression, discord, steam, unrealpak, paths api/ auth, catalog, admin, builds (skeletons) + downloads (real) worker/ Celery factory replacing the missing prototype Tasks/__init__.py db/ SQLAlchemy models + Alembic initial migration - admin-web/ SvelteKit + Tailwind skeleton - client/ Tauri 2 + Svelte skeleton (Mist placeholder UI) - mistpipe/ click-based admin CLI with subcommand stubs - docs/ ARCHITECTURE, DECISIONS (9 ADRs), RUNBOOK - docker-compose.yml + dev overlay + .github/workflows Bugs fixed during port: - Routes/download.py:2 stray backslash on import line - Utils/celery.py inspect.reserved() missing parens + double active() typo - Hardcoded OneDrive/Desktop paths replaced with pydantic-settings config - Discord webhook URL + RabbitMQ password moved to env vars - Missing Tasks/__init__.py reconstructed as worker/__init__.py Out of scope for this commit: route bodies, UI screens, mistpipe subcommand bodies, real image builds. diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..863357a --- /dev/null +++ b/.env.example @@ -0,0 +1,52 @@ +# Copy this file to `.env` and fill in real values. Never commit `.env`. + +# ---- Database ---- +POSTGRES_USER=mist +POSTGRES_PASSWORD=changeme-postgres +POSTGRES_DB=mist +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +DATABASE_URL=postgresql+psycopg://mist:changeme-postgres@postgres:5432/mist + +# ---- Redis ---- +REDIS_URL=redis://redis:6379/0 + +# ---- RabbitMQ / Celery ---- +RABBITMQ_DEFAULT_USER=mist +RABBITMQ_DEFAULT_PASS=changeme-rabbitmq +CELERY_BROKER_URL=amqp://mist:changeme-rabbitmq@rabbitmq:5672// +CELERY_RESULT_BACKEND=redis://redis:6379/1 + +# ---- API ---- +API_HOST=0.0.0.0 +API_PORT=8000 +JWT_SECRET=changeme-generate-a-long-random-string +JWT_ALG=HS256 +JWT_TTL_MINUTES=720 +CORS_ALLOWED_ORIGINS=https://store.example.com,https://admin.example.com + +# ---- Paths (inside the container) ---- +# NAS mount inside container; source of truth for game files +GAMES_DIR=/mnt/nas/mist/games +# Hot cache for prepared .tar.zst archives (Docker volume) +CACHE_DIR=/mist/cache +# Working / temp dir for in-flight delta-gen (Docker volume or tmpfs) +TEMP_DIR=/mist/tmp + +# ---- Patch tool binaries (inside the container) ---- +# These are installed in the backend Dockerfile. +HDIFFZ_PATH=/usr/local/bin/hdiffz +HPATCHZ_PATH=/usr/local/bin/hpatchz +RDIFF_PATH=/usr/local/bin/rdiff +# Optional — only needed on the build host that ingests Unreal games +UNREALPAK_PATH= + +# ---- External integrations ---- +DISCORD_WEBHOOK_URL= +DISCORD_BOT_USERNAME=Mist +DISCORD_BOT_AVATAR_URL=https://www.pcgamesn.com/wp-content/sites/pcgamesn/2018/10/gabe_newell_meme.jpg +STEAM_API_KEY= + +# ---- Misc ---- +LOG_LEVEL=INFO +ENVIRONMENT=development diff --git a/.github/workflows/admin-web.yml b/.github/workflows/admin-web.yml new file mode 100644 index 0000000..914c679 --- /dev/null +++ b/.github/workflows/admin-web.yml @@ -0,0 +1,48 @@ +name: admin-web + +on: + push: + branches: [main] + paths: + - "admin-web/**" + - ".github/workflows/admin-web.yml" + pull_request: + paths: + - "admin-web/**" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - working-directory: admin-web + run: | + npm install --no-audit --no-fund + npm run check + npm run build + + build-and-push: + needs: build + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/setup-buildx-action@v3 + - uses: docker/build-push-action@v6 + with: + context: ./admin-web + push: true + tags: | + ghcr.io/${{ github.repository_owner }}/mist-admin-web:latest + ghcr.io/${{ github.repository_owner }}/mist-admin-web:${{ github.sha }} diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml new file mode 100644 index 0000000..4583385 --- /dev/null +++ b/.github/workflows/backend.yml @@ -0,0 +1,52 @@ +name: backend + +on: + push: + branches: [main] + paths: + - "backend/**" + - ".github/workflows/backend.yml" + pull_request: + paths: + - "backend/**" + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install + working-directory: backend + run: pip install -e .[dev] + - name: Smoke tests + working-directory: backend + run: pytest -q + - name: Lint + working-directory: backend + run: ruff check src tests + + build-and-push: + needs: test + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/setup-buildx-action@v3 + - uses: docker/build-push-action@v6 + with: + context: ./backend + push: true + tags: | + ghcr.io/${{ github.repository_owner }}/mist-backend:latest + ghcr.io/${{ github.repository_owner }}/mist-backend:${{ github.sha }} diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml new file mode 100644 index 0000000..a45d087 --- /dev/null +++ b/.github/workflows/client.yml @@ -0,0 +1,30 @@ +name: client + +on: + push: + tags: ["client-v*"] + workflow_dispatch: + +jobs: + build: + strategy: + fail-fast: false + matrix: + platform: [windows-latest] # add ubuntu-22.04 / macos-latest later + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - uses: dtolnay/rust-toolchain@stable + - name: Install client deps + working-directory: client + run: npm install --no-audit --no-fund + - name: Build Tauri + working-directory: client + run: npx tauri build + - uses: actions/upload-artifact@v4 + with: + name: mist-client-${{ matrix.platform }} + path: client/src-tauri/target/release/bundle/** diff --git a/.github/workflows/mistpipe.yml b/.github/workflows/mistpipe.yml new file mode 100644 index 0000000..bd08548 --- /dev/null +++ b/.github/workflows/mistpipe.yml @@ -0,0 +1,26 @@ +name: mistpipe + +on: + push: + branches: [main] + paths: + - "mistpipe/**" + - ".github/workflows/mistpipe.yml" + pull_request: + paths: + - "mistpipe/**" + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install + working-directory: mistpipe + run: pip install -e . + - name: Import check + working-directory: mistpipe + run: python -c "from mistpipe.cli import cli; print('mistpipe CLI imports cleanly')" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f04f5d --- /dev/null +++ b/.gitignore @@ -0,0 +1,75 @@ +# Secrets +.env +.env.local +.env.*.local + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +.eggs/ +.venv/ +venv/ +env/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.tox/ +.coverage +.coverage.* +htmlcov/ +dist/ +build/ +*.whl + +# Node +node_modules/ +.svelte-kit/ +.vite/ +.turbo/ +*.tsbuildinfo +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +.pnpm-store/ + +# Rust / Tauri +target/ +Cargo.lock +client/src-tauri/gen/ + +# Editor / OS +.DS_Store +Thumbs.db +.idea/ +.vscode/ +*.swp +*~ + +# Build artifacts +*.tar.zst +*.zst +*.7z +*.tmp +*.patch +*.sig +*.dlt + +# Local data / cache (should never be in repo) +data/ +cache/ +tmp/ +games/ +mist-data/ + +# Logs +*.log +logs/ + +# Docker +docker-compose.override.yml diff --git a/README.md b/README.md new file mode 100644 index 0000000..b6099b5 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# Mist + +A private Steam-clone for distributing games and updates to a small group of friends. Successor to the "Josh Steam" prototypes. + +## What it does + +- Hosts a catalog of games with version history +- Generates **delta patches** between versions (hdiff for direct/consecutive jumps, librsync-style for indirect/arbitrary jumps) so friends download only what changed +- Serves resumable downloads +- Ships a desktop client (Tauri) so friends can browse, install, and update games like they would on Steam +- Ships an admin web portal for managing the catalog +- Ships a CLI (`mistpipe`) for uploading new game versions from your own machine + +## Repo layout + +``` +backend/ # FastAPI app + Celery worker (same image, two entrypoints) +admin-web/ # SvelteKit admin portal +client/ # Tauri desktop client (Rust shell + Svelte UI) +mistpipe/ # Python CLI for admin uploads +docs/ # ARCHITECTURE, DECISIONS, RUNBOOK +.github/ # CI workflows +docker-compose.yml # production-ish stack +docker-compose.dev.yml # dev overlay (hot reload, bind mounts) +``` + +## Quickstart (development) + +```sh +cp .env.example .env +# edit .env, at minimum set passwords + JWT_SECRET + +docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build +``` + +API at `http://localhost:8000`, admin web at `http://localhost:5173`, RabbitMQ UI at `http://localhost:15672`. + +For the client and CLI, see `client/README.md` and `mistpipe/README.md`. + +## Status + +Initial scaffold. The load-bearing `backend/src/mist/core/` modules (hdiff, librsync, chain-replay, manifest, compression, discord, steam, unrealpak) are ported from the original Josh Steam prototypes with bugs fixed and hardcoded paths replaced. Route handlers, UI screens, and CLI subcommands are skeletons — to be implemented in subsequent phases. + +See `docs/ARCHITECTURE.md` for the system design and `docs/DECISIONS.md` for the decision log. diff --git a/admin-web/Dockerfile b/admin-web/Dockerfile new file mode 100644 index 0000000..79edb12 --- /dev/null +++ b/admin-web/Dockerfile @@ -0,0 +1,14 @@ +# Build stage +FROM node:22-alpine AS build +WORKDIR /app +COPY package.json ./ +RUN npm install --no-audit --no-fund +COPY . . +RUN npm run build + +# Serve stage +FROM nginx:1.27-alpine +COPY --from=build /app/build /usr/share/nginx/html +EXPOSE 80 +# Default nginx config serves /usr/share/nginx/html; for SPA fallback, drop in a +# minimal config here later if SvelteKit routes need the fallback to index.html. diff --git a/admin-web/package.json b/admin-web/package.json new file mode 100644 index 0000000..facf154 --- /dev/null +++ b/admin-web/package.json @@ -0,0 +1,27 @@ +{ + "name": "mist-admin-web", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite dev --host 0.0.0.0 --port 5173", + "build": "vite build", + "preview": "vite preview", + "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", + "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch" + }, + "devDependencies": { + "@sveltejs/adapter-static": "^3.0.0", + "@sveltejs/kit": "^2.7.0", + "@sveltejs/vite-plugin-svelte": "^4.0.0", + "@tailwindcss/typography": "^0.5.15", + "autoprefixer": "^10.4.20", + "postcss": "^8.4.49", + "svelte": "^5.0.0", + "svelte-check": "^4.0.0", + "tailwindcss": "^3.4.14", + "tslib": "^2.7.0", + "typescript": "^5.6.0", + "vite": "^5.4.0" + } +} diff --git a/admin-web/src/app.html b/admin-web/src/app.html new file mode 100644 index 0000000..5f514e2 --- /dev/null +++ b/admin-web/src/app.html @@ -0,0 +1,13 @@ + + + + + + + Mist Admin + %sveltekit.head% + + +
%sveltekit.body%
+ + diff --git a/admin-web/src/lib/api.ts b/admin-web/src/lib/api.ts new file mode 100644 index 0000000..b03fc0c --- /dev/null +++ b/admin-web/src/lib/api.ts @@ -0,0 +1,49 @@ +// Typed fetch client skeleton for Mist API. +// TODO: replace `any` returns with generated types once OpenAPI is wired. + +const API_BASE = import.meta.env.VITE_API_BASE ?? ''; + +function token(): string | null { + if (typeof localStorage === 'undefined') return null; + return localStorage.getItem('mist_token'); +} + +async function call(method: string, path: string, body?: unknown): Promise { + const headers: Record = { 'Content-Type': 'application/json' }; + const t = token(); + if (t) headers['Authorization'] = `Bearer ${t}`; + + const res = await fetch(`${API_BASE}${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined + }); + if (!res.ok) throw new Error(`${method} ${path} -> ${res.status}`); + if (res.status === 204) return undefined as T; + return res.json() as Promise; +} + +export const api = { + login: (username: string, password: string) => + call<{ access_token: string }>('POST', '/auth/login', { username, password }), + me: () => call<{ id: number; username: string; is_admin: boolean }>('GET', '/auth/me'), + listGames: () => call('GET', '/catalog/games'), + createGame: (data: { title: string; app_id?: number; is_private?: boolean }) => + call('POST', '/admin/games', data), + updateGame: (id: number, patch: Record) => + call('PATCH', `/admin/games/${id}`, patch), + deleteGame: (id: number) => call('DELETE', `/admin/games/${id}`), + resyncSteam: (id: number) => call('POST', `/admin/games/${id}/resync-steam`), + listUsers: () => call('GET', '/admin/users'), + createUser: (data: { username: string; password: string; is_admin?: boolean }) => + call('POST', '/admin/users', data), + listBuildJobs: () => call('GET', '/admin/build-jobs') +}; + +export function setToken(t: string) { + localStorage.setItem('mist_token', t); +} + +export function clearToken() { + localStorage.removeItem('mist_token'); +} diff --git a/admin-web/src/routes/+layout.svelte b/admin-web/src/routes/+layout.svelte new file mode 100644 index 0000000..d04ffbc --- /dev/null +++ b/admin-web/src/routes/+layout.svelte @@ -0,0 +1,19 @@ + + +
+
+
+ Mist Admin + +
+
v0.1.0 · skeleton
+
+
+ {@render children()} +
+
diff --git a/admin-web/src/routes/+page.svelte b/admin-web/src/routes/+page.svelte new file mode 100644 index 0000000..8918e86 --- /dev/null +++ b/admin-web/src/routes/+page.svelte @@ -0,0 +1,26 @@ + + +
+

Welcome to Mist Admin

+

+ This is the admin portal. From here you manage games, push updates, and provision friend accounts. + Nothing is wired up yet — this is a skeleton. +

+ +
+ +

Games

+

Catalog management, metadata, branches.

+ + +

Users

+

Provision accounts for friends.

+ + +

Login

+

Sign in with your admin account.

+ +
+
diff --git a/admin-web/src/routes/login/+page.svelte b/admin-web/src/routes/login/+page.svelte new file mode 100644 index 0000000..d4c582c --- /dev/null +++ b/admin-web/src/routes/login/+page.svelte @@ -0,0 +1,37 @@ + + +
+

Sign in

+
+ + + {#if error} +

{error}

+ {/if} + +
+
diff --git a/admin-web/src/routes/users/+page.svelte b/admin-web/src/routes/users/+page.svelte new file mode 100644 index 0000000..cd1d931 --- /dev/null +++ b/admin-web/src/routes/users/+page.svelte @@ -0,0 +1,35 @@ + + +
+
+

Users

+ +
+ + {#if error} +

{error}

+ {/if} + + {#if users.length === 0} +

No users provisioned yet. Use the "Create user" form (TODO) or `mist.scripts.create_user` from a container.

+ {:else} +
    + {#each users as u} +
  • {JSON.stringify(u)}
    • + {/each} +
+ {/if} +
diff --git a/admin-web/svelte.config.js b/admin-web/svelte.config.js new file mode 100644 index 0000000..0d03b92 --- /dev/null +++ b/admin-web/svelte.config.js @@ -0,0 +1,21 @@ +import adapter from '@sveltejs/adapter-static'; +import { vitePreprocess } from '@sveltejs/vite-plugin-svelte'; + +/** @type {import('@sveltejs/kit').Config} */ +const config = { + preprocess: vitePreprocess(), + kit: { + adapter: adapter({ + pages: 'build', + assets: 'build', + fallback: 'index.html', + precompress: false, + strict: true + }), + alias: { + $lib: 'src/lib' + } + } +}; + +export default config; diff --git a/admin-web/tsconfig.json b/admin-web/tsconfig.json new file mode 100644 index 0000000..4344710 --- /dev/null +++ b/admin-web/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "./.svelte-kit/tsconfig.json", + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "sourceMap": true, + "strict": true, + "moduleResolution": "bundler" + } +} diff --git a/admin-web/vite.config.ts b/admin-web/vite.config.ts new file mode 100644 index 0000000..f2505ad --- /dev/null +++ b/admin-web/vite.config.ts @@ -0,0 +1,10 @@ +import { sveltekit } from '@sveltejs/kit/vite'; +import { defineConfig } from 'vite'; + +export default defineConfig({ + plugins: [sveltekit()], + server: { + host: '0.0.0.0', + port: 5173 + } +}); diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..d9edcd0 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,35 @@ +# Mist backend image — used by both `api` and `worker` containers. +# The api container runs uvicorn (the default ENTRYPOINT); the worker container +# overrides the command to `celery -A mist.worker worker --loglevel=INFO`. + +FROM python:3.12-slim AS base + +# System deps for py7zr, librsync, hdiff, build tools. +# hdiffz/hpatchz/rdiff binaries are expected to be installed here at /usr/local/bin. +# In dev they can be bind-mounted; in prod they're baked in. Placeholder COPY below. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + librsync-dev \ + rdiff \ + curl \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# TODO: copy hdiffz/hpatchz binaries from a known release tarball (build-time stage) +# For now, the operator is expected to provide them; or build from source. + +WORKDIR /app + +COPY pyproject.toml ./pyproject.toml +COPY src ./src +COPY alembic.ini ./alembic.ini + +RUN pip install --no-cache-dir -e . + +ENV PYTHONPATH=/app/src +ENV PYTHONUNBUFFERED=1 + +EXPOSE 8000 + +# Default entrypoint = api. The worker container overrides this in docker-compose.yml. +CMD ["uvicorn", "mist.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 0000000..af60543 --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,40 @@ +[alembic] +script_location = src/mist/db/migrations +prepend_sys_path = src +version_path_separator = os + +# sqlalchemy.url is set programmatically in env.py from mist.config.settings + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 0000000..056d649 --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,65 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "mist" +version = "0.1.0" +description = "Mist backend — FastAPI app + Celery worker for the Mist private game distribution platform" +readme = "../README.md" +requires-python = ">=3.12" +license = { text = "Proprietary" } +authors = [{ name = "Josh" }] + +dependencies = [ + # Web framework + "fastapi>=0.115", + "uvicorn[standard]>=0.32", + "python-multipart>=0.0.12", # multipart form parsing for builds upload + "pydantic>=2.9", + "pydantic-settings>=2.6", + + # DB / migrations + "sqlalchemy>=2.0", + "alembic>=1.13", + "psycopg[binary]>=3.2", + + # Auth + "argon2-cffi>=23.1", + "python-jose[cryptography]>=3.3", + + # Background work + "celery>=5.4", + "redis>=5.1", + "kombu>=5.4", + + # Content / compression + "zstandard>=0.23", + "py7zr>=0.22", + + # Outbound HTTP + "httpx>=0.27", + "requests>=2.32", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.3", + "pytest-asyncio>=0.24", + "ruff>=0.7", + "mypy>=1.13", +] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 110 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "N"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/backend/src/mist/__init__.py b/backend/src/mist/__init__.py new file mode 100644 index 0000000..47204ed --- /dev/null +++ b/backend/src/mist/__init__.py @@ -0,0 +1,10 @@ +"""Mist — private game distribution platform. + +Package layout: + api/ FastAPI routers + app factory + worker/ Celery app + tasks + core/ Load-bearing domain logic (delta-patching, manifests, etc.) + db/ SQLAlchemy models + Alembic migrations +""" + +__version__ = "0.1.0" diff --git a/backend/src/mist/api/__init__.py b/backend/src/mist/api/__init__.py new file mode 100644 index 0000000..b69c6f7 --- /dev/null +++ b/backend/src/mist/api/__init__.py @@ -0,0 +1 @@ +"""Mist HTTP API — FastAPI routers + app factory.""" diff --git a/backend/src/mist/api/admin.py b/backend/src/mist/api/admin.py new file mode 100644 index 0000000..0b10ec5 --- /dev/null +++ b/backend/src/mist/api/admin.py @@ -0,0 +1,72 @@ +"""Admin routes — game CRUD, user provisioning, build-job inspection. + +Skeleton implementation; all routes require admin scope. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from mist.api.deps import require_admin +from mist.db.models import User + +router = APIRouter(dependencies=[Depends(require_admin)]) + + +class CreateGameRequest(BaseModel): + title: str + app_id: int | None = None + is_private: bool = False + + +class UpdateGameRequest(BaseModel): + description_override: str | None = None + header_image_override: str | None = None + is_private: bool | None = None + + +class CreateUserRequest(BaseModel): + username: str + password: str + is_admin: bool = False + + +@router.post("/games", status_code=201) +def create_game(_body: CreateGameRequest, _admin: User = Depends(require_admin)) -> dict: + # TODO: create Game row, kick off Steam appdetails fetch if app_id present + raise HTTPException(status_code=501, detail="admin.create_game not implemented yet") + + +@router.patch("/games/{game_id}") +def update_game(game_id: int, _body: UpdateGameRequest, _admin: User = Depends(require_admin)) -> dict: + raise HTTPException(status_code=501, detail="admin.update_game not implemented yet") + + +@router.delete("/games/{game_id}", status_code=204) +def delete_game(game_id: int, _admin: User = Depends(require_admin)) -> None: + # TODO: soft-delete (set deleted_at) + raise HTTPException(status_code=501, detail="admin.delete_game not implemented yet") + + +@router.post("/games/{game_id}/resync-steam") +def resync_steam(game_id: int, _admin: User = Depends(require_admin)) -> dict: + # TODO: re-fetch Steam appdetails for this game + raise HTTPException(status_code=501, detail="admin.resync_steam not implemented yet") + + +@router.post("/users", status_code=201) +def create_user(_body: CreateUserRequest, _admin: User = Depends(require_admin)) -> dict: + # TODO: argon2-hash password, insert User row + raise HTTPException(status_code=501, detail="admin.create_user not implemented yet") + + +@router.get("/users") +def list_users(_admin: User = Depends(require_admin)) -> list[dict]: + # TODO: list users + raise HTTPException(status_code=501, detail="admin.list_users not implemented yet") + + +@router.get("/build-jobs") +def list_build_jobs(_admin: User = Depends(require_admin)) -> list[dict]: + raise HTTPException(status_code=501, detail="admin.list_build_jobs not implemented yet") diff --git a/backend/src/mist/api/app.py b/backend/src/mist/api/app.py new file mode 100644 index 0000000..c79767c --- /dev/null +++ b/backend/src/mist/api/app.py @@ -0,0 +1,60 @@ +"""FastAPI app factory. + +Run with: `uvicorn mist.api.app:app --host 0.0.0.0 --port 8000` +""" + +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager +from collections.abc import AsyncIterator + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from mist.api import admin, auth, builds, catalog, downloads +from mist.config import settings +from mist.core import paths as core_paths + +log = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(_: FastAPI) -> AsyncIterator[None]: + logging.basicConfig(level=settings.log_level) + log.info("Mist API starting; environment=%s", settings.environment) + core_paths.ensure_dirs() + yield + log.info("Mist API stopping") + + +def create_app() -> FastAPI: + app = FastAPI(title="Mist API", version="0.1.0", lifespan=lifespan) + + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins or ["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + app.include_router(auth.router, prefix="/auth", tags=["auth"]) + app.include_router(catalog.router, prefix="/catalog", tags=["catalog"]) + app.include_router(admin.router, prefix="/admin", tags=["admin"]) + app.include_router(builds.router, prefix="/builds", tags=["builds"]) + app.include_router(downloads.router, prefix="/download", tags=["downloads"]) + + @app.get("/healthz", tags=["health"]) + def healthz() -> dict[str, bool]: + return {"ok": True} + + @app.get("/readyz", tags=["health"]) + def readyz() -> dict[str, bool]: + # TODO: actually check db / redis / rabbitmq reachability + return {"ok": True} + + return app + + +app = create_app() diff --git a/backend/src/mist/api/auth.py b/backend/src/mist/api/auth.py new file mode 100644 index 0000000..01eab1e --- /dev/null +++ b/backend/src/mist/api/auth.py @@ -0,0 +1,42 @@ +"""Auth routes — login + identity. + +Skeleton implementation; argon2 password verify and JWT issuance to be filled in. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from mist.api.deps import get_current_user +from mist.db.models import User + +router = APIRouter() + + +class LoginRequest(BaseModel): + username: str + password: str + + +class LoginResponse(BaseModel): + access_token: str + token_type: str = "bearer" + + +class MeResponse(BaseModel): + id: int + username: str + is_admin: bool + + +@router.post("/login", response_model=LoginResponse) +def login(_body: LoginRequest) -> LoginResponse: + # TODO: look up user by username, verify password_hash with argon2_cffi, + # then mint a JWT with sub=user.id and scope=admin if user.is_admin. + raise HTTPException(status_code=501, detail="login not implemented yet") + + +@router.get("/me", response_model=MeResponse) +def me(user: User = Depends(get_current_user)) -> MeResponse: + return MeResponse(id=user.id, username=user.username, is_admin=user.is_admin) diff --git a/backend/src/mist/api/builds.py b/backend/src/mist/api/builds.py new file mode 100644 index 0000000..0b9155c --- /dev/null +++ b/backend/src/mist/api/builds.py @@ -0,0 +1,74 @@ +"""Build routes — receive uploads from `mistpipe`, query update status, kick off delta-gen. + +Skeleton implementation. + +NOTE: this is where the fix for the prototype's request_update.py:53 bug applies. +The old code compared dict values to raw hashes: + from_version_manifest[file] != hash ← wrong (dict vs scalar) +The correct comparison is on the `checksum` key: + from_version_manifest[file]['checksum'] != hash['checksum'] +Implement that pattern when filling in update routing. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, UploadFile +from pydantic import BaseModel + +from mist.api.deps import get_current_user, require_admin +from mist.db.models import User + +router = APIRouter() + + +class PushUpdateRequest(BaseModel): + game_title: str + version: str + app_id: int | None = None + patch_notes_url: str | None = None + + +class RequestUpdateResponse(BaseModel): + mode: str # "cache" | "generate_direct" | "generate_indirect" + file: str | None = None + task_id: str | None = None + files_to_signature: list[str] | None = None + + +@router.post("/upload", dependencies=[Depends(require_admin)]) +async def upload(_file: UploadFile, _admin: User = Depends(require_admin)) -> dict: + """Receive a full game-version bundle from `mistpipe push`.""" + # TODO: stream upload to /mnt/nas, queue push_update task, return BuildJob + raise HTTPException(status_code=501, detail="builds.upload not implemented yet") + + +@router.get("/jobs/{job_id}") +def get_job(job_id: int, _admin: User = Depends(require_admin)) -> dict: + raise HTTPException(status_code=501, detail="builds.get_job not implemented yet") + + +@router.get("/request-update/{game_title}/{from_version}/{to_version}", response_model=RequestUpdateResponse) +def request_update( + game_title: str, from_version: str, to_version: str, _user: User = Depends(get_current_user) +) -> RequestUpdateResponse: + """Client asks: how do I get from `from_version` to `to_version`? + + Returns one of: + - cache hit (file ready to download) + - generate direct (server queues a hdiff direct-delta zip task) + - generate indirect (client must POST signatures next) + """ + # TODO: implement direct/indirect decision logic; apply request_update.py:53 fix + raise HTTPException(status_code=501, detail="builds.request_update not implemented yet") + + +@router.post("/generate-indirect/{game_title}/{from_version}/{to_version}") +async def generate_indirect_update( + game_title: str, + from_version: str, + to_version: str, + _signatures: dict, + _user: User = Depends(get_current_user), +) -> dict: + """Client posts a `signatures` dict of {file: hex_signature}; server queues delta-gen.""" + raise HTTPException(status_code=501, detail="builds.generate_indirect_update not implemented yet") diff --git a/backend/src/mist/api/catalog.py b/backend/src/mist/api/catalog.py new file mode 100644 index 0000000..9d58b69 --- /dev/null +++ b/backend/src/mist/api/catalog.py @@ -0,0 +1,46 @@ +"""Catalog routes — game browsing for end users. + +Skeleton implementation; honors `is_private` filtering once wired. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from mist.api.deps import get_current_user +from mist.db.models import User + +router = APIRouter() + + +class GameSummary(BaseModel): + id: int + title: str + app_id: int | None + header_image: str | None + short_description: str | None + is_private: bool + latest_version: str | None + + +class GameDetail(GameSummary): + versions: list[str] + + +@router.get("/games", response_model=list[GameSummary]) +def list_games(_user: User = Depends(get_current_user)) -> list[GameSummary]: + # TODO: list games visible to the current user (public + admin sees private) + raise HTTPException(status_code=501, detail="catalog.list_games not implemented yet") + + +@router.get("/games/{game_id}", response_model=GameDetail) +def get_game(game_id: int, _user: User = Depends(get_current_user)) -> GameDetail: + # TODO: fetch single game with version list + raise HTTPException(status_code=501, detail="catalog.get_game not implemented yet") + + +@router.get("/games/{game_id}/manifests/{version}") +def get_manifest(game_id: int, version: str, _user: User = Depends(get_current_user)) -> dict: + # TODO: serve per-version manifest JSON + raise HTTPException(status_code=501, detail="catalog.get_manifest not implemented yet") diff --git a/backend/src/mist/api/deps.py b/backend/src/mist/api/deps.py new file mode 100644 index 0000000..b4ee18d --- /dev/null +++ b/backend/src/mist/api/deps.py @@ -0,0 +1,61 @@ +"""Shared FastAPI dependencies — DB session, JWT auth, admin gate.""" + +from __future__ import annotations + +from collections.abc import Iterator + +from fastapi import Depends, HTTPException, status +from fastapi.security import OAuth2PasswordBearer +from jose import JWTError, jwt +from sqlalchemy.orm import Session + +from mist.config import settings +from mist.db.base import SessionLocal +from mist.db.models import User + +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/auth/login", auto_error=False) + + +def get_db() -> Iterator[Session]: + session = SessionLocal() + try: + yield session + finally: + session.close() + + +def _decode_token(token: str) -> dict: + try: + return jwt.decode(token, settings.jwt_secret, algorithms=[settings.jwt_alg]) + except JWTError as e: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=f"Invalid token: {e}", + headers={"WWW-Authenticate": "Bearer"}, + ) from e + + +def get_current_user( + token: str | None = Depends(oauth2_scheme), + db: Session = Depends(get_db), +) -> User: + if not token: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not authenticated", + headers={"WWW-Authenticate": "Bearer"}, + ) + payload = _decode_token(token) + user_id = payload.get("sub") + if user_id is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Token missing sub") + user = db.get(User, int(user_id)) + if user is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found") + return user + + +def require_admin(user: User = Depends(get_current_user)) -> User: + if not user.is_admin: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only") + return user diff --git a/backend/src/mist/api/downloads.py b/backend/src/mist/api/downloads.py new file mode 100644 index 0000000..ca1aeaa --- /dev/null +++ b/backend/src/mist/api/downloads.py @@ -0,0 +1,86 @@ +"""Downloads — resumable HTTP `Range` serving of cached artifacts. + +Ported from: JoshSteam CDN/Routes/download.py +Bug fixes during port: + - Original line 2 had a stray backslash continuation breaking the import + - Ported from Flask to FastAPI (StreamingResponse + Request.headers) + - Hardcoded CACHE_DIR replaced with settings.cache_dir + - Auth gate added (any authenticated user can download; private-game checking + happens at /catalog level before the client is told what file to fetch) +""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import FileResponse, StreamingResponse + +from mist.api.deps import get_current_user +from mist.config import settings +from mist.db.models import User + +router = APIRouter() + +CHUNK_SIZE = 8192 + + +def _file_iter(file_path: Path, start: int, end: int) -> Iterator[bytes]: + with open(file_path, "rb") as f: + f.seek(start) + remaining = end - start + 1 + while remaining > 0: + chunk = f.read(min(CHUNK_SIZE, remaining)) + if not chunk: + break + yield chunk + remaining -= len(chunk) + + +@router.get("/{file}") +def download(file: str, request: Request, _user: User = Depends(get_current_user)) -> StreamingResponse | FileResponse: + """Serve a cached file with HTTP `Range` resume support.""" + # Defensive: prevent path traversal + if "/" in file or "\\" in file or ".." in file: + raise HTTPException(status_code=400, detail="Invalid file name") + + file_path = settings.cache_dir / file + if not file_path.exists() or not file_path.is_file(): + raise HTTPException(status_code=404, detail="File not found") + + file_size = file_path.stat().st_size + range_header = request.headers.get("range") + + if not range_header: + return FileResponse( + file_path, + media_type="application/zstd", + filename=file, + ) + + # Parse "bytes=START-END" (END optional) + try: + units, _, rng = range_header.partition("=") + if units.strip().lower() != "bytes": + raise ValueError("only bytes ranges supported") + start_s, _, end_s = rng.strip().partition("-") + start = int(start_s) + end = int(end_s) if end_s else file_size - 1 + end = min(end, file_size - 1) + if start < 0 or start > end: + raise ValueError("bad range") + except ValueError: + raise HTTPException(status_code=416, detail="Invalid Range header") from None + + headers = { + "Content-Range": f"bytes {start}-{end}/{file_size}", + "Accept-Ranges": "bytes", + "Content-Length": str(end - start + 1), + } + return StreamingResponse( + _file_iter(file_path, start, end), + status_code=206, + media_type="application/zstd", + headers=headers, + ) diff --git a/backend/src/mist/config.py b/backend/src/mist/config.py new file mode 100644 index 0000000..67a7ead --- /dev/null +++ b/backend/src/mist/config.py @@ -0,0 +1,69 @@ +"""Mist configuration — every path and secret comes from env vars. + +Loaded once at process start. Importable as `from mist.config import settings`. +""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + # ---- Environment ---- + environment: str = "development" + log_level: str = "INFO" + + # ---- Database ---- + database_url: str = "postgresql+psycopg://mist:changeme@localhost:5432/mist" + + # ---- Redis ---- + redis_url: str = "redis://localhost:6379/0" + + # ---- Celery ---- + celery_broker_url: str = "amqp://mist:changeme@localhost:5672//" + celery_result_backend: str = "redis://localhost:6379/1" + + # ---- API ---- + api_host: str = "0.0.0.0" + api_port: int = 8000 + jwt_secret: str = "change-me" + jwt_alg: str = "HS256" + jwt_ttl_minutes: int = 60 * 12 + cors_allowed_origins: str = "" # comma-separated + + # ---- Paths ---- + games_dir: Path = Field(default=Path("/mnt/nas/mist/games")) + cache_dir: Path = Field(default=Path("/mist/cache")) + temp_dir: Path = Field(default=Path("/mist/tmp")) + + # ---- Patch tool binaries ---- + hdiffz_path: Path = Field(default=Path("/usr/local/bin/hdiffz")) + hpatchz_path: Path = Field(default=Path("/usr/local/bin/hpatchz")) + rdiff_path: Path = Field(default=Path("/usr/local/bin/rdiff")) + unrealpak_path: Path | None = None # optional; only needed on Unreal-game ingest hosts + + # ---- External integrations ---- + discord_webhook_url: str | None = None + discord_bot_username: str = "Mist" + discord_bot_avatar_url: str = ( + "https://www.pcgamesn.com/wp-content/sites/pcgamesn/2018/10/gabe_newell_meme.jpg" + ) + steam_api_key: str | None = None + + @property + def cors_origins(self) -> list[str]: + return [o.strip() for o in self.cors_allowed_origins.split(",") if o.strip()] + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + return Settings() + + +settings = get_settings() diff --git a/backend/src/mist/core/__init__.py b/backend/src/mist/core/__init__.py new file mode 100644 index 0000000..fff50d2 --- /dev/null +++ b/backend/src/mist/core/__init__.py @@ -0,0 +1,13 @@ +"""Mist core — load-bearing domain logic. + +These modules are ported from the original Josh Steam prototypes: + hdiff — direct-delta generation via HDiffPatch (`hdiffz`) + librsync — indirect-delta generation via librsync (`rdiff`) + chain_replay — cold reconstruction of arbitrary historical versions + manifest — SHA-256 per-file manifests + compression — zstd streaming compression, 7z extraction + discord — Discord webhook announcements + steam — Steam appdetails pull-through + unrealpak — Unreal Engine .pak extract/repack helper + paths — centralized path resolution (uses settings) +""" diff --git a/backend/src/mist/core/chain_replay.py b/backend/src/mist/core/chain_replay.py new file mode 100644 index 0000000..e53719f --- /dev/null +++ b/backend/src/mist/core/chain_replay.py @@ -0,0 +1,237 @@ +"""Cold reconstruction of arbitrary historical versions via delta-chain replay. + +Ported from: JoshSteam CDN/Utils/prepare_game_version.py +Bug fixes / changes during port: + - All hardcoded paths (GAMES_DIR/CACHE_DIR/TEMP_DIR/hdiff_path) replaced with + `mist.core.paths` helpers + `settings` + - Type hints + - Behavior preserved: find closest cached version, copy it, walk forward + applying hdiff patches at each step, verify final state, cache result + +The key insight: we never need to store every version's full files. The base +version (.7z) + the delta chain gives us any historical version on demand, +and the result gets cached for next time. +""" + +from __future__ import annotations + +import concurrent.futures +import json +import os +import shutil +import subprocess +from pathlib import Path +from typing import Any + +from mist.config import settings +from mist.core import paths +from mist.core.compression import extract_7z +from mist.core.manifest import verify_files + + +# ---- Version-history helpers (legacy JSON path; callers should prefer the DB) ---- + + +def get_base_version(game_title: str) -> str: + path = paths.game_version_history_path(game_title) + if not path.exists(): + raise FileNotFoundError(f"Version file for {game_title} not found at {path}") + with open(path) as f: + versions: list[str] = json.load(f) + if not versions: + raise ValueError(f"No versions found for {game_title}") + return versions[0] + + +def get_versions_up_to(game_title: str, target_version: str) -> list[str]: + path = paths.game_version_history_path(game_title) + if not path.exists(): + raise FileNotFoundError(f"Version file for {game_title} not found at {path}") + with open(path) as f: + versions: list[str] = json.load(f) + try: + idx = versions.index(target_version) + except ValueError: + return [] + return versions[0 : idx + 1] + + +def get_versions_in_range(game_title: str, from_version: str, to_version: str) -> list[str]: + path = paths.game_version_history_path(game_title) + if not path.exists(): + raise FileNotFoundError(f"Version file for {game_title} not found at {path}") + with open(path) as f: + versions: list[str] = json.load(f) + try: + from_idx = versions.index(from_version) + to_idx = versions.index(to_version) + except ValueError: + return [] + return versions[from_idx + 1 : to_idx + 1] + + +# ---- Cache lookup ---- + + +def find_game_folder_in_cache(game_title: str, version: str) -> bool: + """Is the reconstructed-version folder for (title, version) present in the cache?""" + return paths.cached_game_version_dir(game_title, version).exists() + + +def find_closest_cached_version(game_title: str, version_list: list[str]) -> str | None: + """Of the candidate versions in `version_list`, return the latest one we've already + reconstructed and have sitting in cache (if any).""" + cache_dir = settings.cache_dir + cached_versions: list[str] = [] + if not cache_dir.exists(): + return None + for folder in os.listdir(cache_dir): + if not folder.startswith(game_title): + continue + folder_version = folder[len(game_title) :].strip() + if folder_version in version_list: + cached_versions.append(folder_version) + if not cached_versions: + return None + cached_versions.sort(key=lambda v: version_list.index(v)) + return cached_versions[-1] + + +# ---- Patch application ---- + + +def _copy_folder(src: Path, dest: Path) -> None: + if not src.exists(): + raise FileNotFoundError(f"Source folder not found: {src}") + shutil.copytree(src, dest) + + +def _apply_hdiff_patch( + patch: str, data: dict[str, str], temp_delta_folder: Path, temp_game_dir: Path +) -> bool: + patch_path = temp_delta_folder / patch + relative_path = data.get("relative_path") + if not relative_path: + return False + original_file_path = temp_game_dir / relative_path + if not original_file_path.exists(): + return False + temp_file_path = temp_delta_folder / f"{patch}.tmp" + try: + subprocess.run( + [str(settings.hpatchz_path), "-f", str(original_file_path), str(patch_path), str(temp_file_path)], + check=True, + ) + os.replace(temp_file_path, original_file_path) + return True + except (subprocess.CalledProcessError, OSError): + return False + + +def apply_update(game_title: str, version: str, temp_game_dir: Path) -> None: + """Apply the delta_manifest for `version` to the working folder `temp_game_dir`. + + Copies the delta folder out of the NAS into temp_delta_folder, applies in parallel, + handles new/deleted files, then deletes the temp_delta_folder. + """ + delta_folder = paths.deltas_dir(game_title, version) + temp_delta_folder = settings.temp_dir / f"{game_title} Update" / version + if not delta_folder.exists(): + return + + _copy_folder(delta_folder, temp_delta_folder) + with open(temp_delta_folder / "delta_manifest.json") as f: + delta_manifest: dict[str, Any] = json.load(f) + + with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: + futures = [ + executor.submit(_apply_hdiff_patch, patch, data, temp_delta_folder, temp_game_dir) + for patch, data in delta_manifest["modified_files"].items() + ] + for f in concurrent.futures.as_completed(futures): + f.result() + + new_files_folder = temp_delta_folder / "new_files" + for new_file in delta_manifest["new_files"]: + src = new_files_folder / new_file["file"] + dst = temp_game_dir / new_file["relative_path"] + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(src, dst) + + for deleted in delta_manifest["deleted_files"]: + target = temp_game_dir / deleted["relative_path"] + if target.exists(): + target.unlink() + + shutil.rmtree(temp_delta_folder) + + +# ---- Orchestrator ---- + + +def prepare_game_version(game_title: str, version: str) -> bool: + """Reconstruct (game_title, version) into the cache folder and verify it. + + Strategy: + 1. If version == base, copy/extract the base archive. + 2. Otherwise: find the closest cached version, copy it, walk forward. + 3. If nothing is cached, extract the base and walk forward from there. + 4. Verify against the per-version manifest. On success, move to cache. + """ + base_version = get_base_version(game_title) + update_path: list[str] | None = None + temp_game_dir: Path + + if version == base_version: + if find_game_folder_in_cache(game_title, base_version): + _copy_folder( + paths.cached_game_version_dir(game_title, base_version), + settings.temp_dir / f"{game_title} {version}", + ) + temp_game_dir = settings.temp_dir / f"{game_title} {version}" + else: + base_archive = paths.base_archive_path(game_title) + temp_game_dir = settings.temp_dir / f"{game_title} {base_version}" + if not base_archive.exists(): + return False + temp_game_dir.mkdir(parents=True, exist_ok=True) + extract_7z(base_archive, temp_game_dir) + else: + update_path_from_base = get_versions_up_to(game_title, version) + closest = find_closest_cached_version(game_title, update_path_from_base) + if closest: + _copy_folder( + paths.cached_game_version_dir(game_title, closest), + settings.temp_dir / f"{game_title} {closest}", + ) + temp_game_dir = settings.temp_dir / f"{game_title} {closest}" + update_path = get_versions_in_range(game_title, closest, version) + else: + base_archive = paths.base_archive_path(game_title) + temp_game_dir = settings.temp_dir / f"{game_title} {base_version}" + if not base_archive.exists(): + return False + temp_game_dir.mkdir(parents=True, exist_ok=True) + extract_7z(base_archive, temp_game_dir) + update_path = get_versions_up_to(game_title, version) + if update_path: + update_path.pop(0) # we already have the base extracted + + if update_path: + for next_version in update_path: + apply_update(game_title, next_version, temp_game_dir) + new_folder = settings.temp_dir / f"{game_title} {next_version}" + os.rename(temp_game_dir, new_folder) + temp_game_dir = new_folder + update_workspace = settings.temp_dir / f"{game_title} Update" + if update_workspace.exists(): + shutil.rmtree(update_workspace) + + requested_manifest = paths.manifest_path(game_title, version) + results = verify_files(temp_game_dir, requested_manifest) + if results["all_files_verified"]: + output_dir = paths.cached_game_version_dir(game_title, version) + shutil.move(str(temp_game_dir), str(output_dir)) + return True + shutil.rmtree(temp_game_dir) + return False diff --git a/backend/src/mist/core/compression.py b/backend/src/mist/core/compression.py new file mode 100644 index 0000000..4f6047f --- /dev/null +++ b/backend/src/mist/core/compression.py @@ -0,0 +1,56 @@ +"""zstd streaming compression + 7z extraction. + +Ported from: JoshSteam CDN/Utils/compression.py +No behavior changes; just type hints and path-objects. +""" + +from __future__ import annotations + +import os +import tarfile +from pathlib import Path +from typing import BinaryIO + +import py7zr +import zstandard as zstd + + +def generate_tar_zstd_stream(depot_path: Path | str, output_stream: BinaryIO) -> None: + """Stream tarball of `depot_path` contents into an output binary stream.""" + depot_path = Path(depot_path) + with tarfile.open(fileobj=output_stream, mode="w|") as tar: + for root, _, files in os.walk(depot_path): + for file in files: + file_path = Path(root) / file + relative_path = file_path.relative_to(depot_path) + tar.add(file_path, arcname=str(relative_path)) + + +def compress_and_save_zstd( + depot_path: Path | str, temp_zstd_path: Path | str, final_zstd_path: Path | str +) -> None: + """Tar+zstd `depot_path` to a temp file, then atomically rename to final. + + No intermediate uncompressed .tar is created on disk. + """ + temp_zstd_path = Path(temp_zstd_path) + final_zstd_path = Path(final_zstd_path) + temp_zstd_path.parent.mkdir(parents=True, exist_ok=True) + + with open(temp_zstd_path, "wb") as out_file: + cctx = zstd.ZstdCompressor(level=3) + with cctx.stream_writer(out_file) as compressor: + generate_tar_zstd_stream(depot_path, compressor) + + os.replace(temp_zstd_path, final_zstd_path) + + +def extract_7z(archive_path: Path | str, destination_path: Path | str) -> None: + """Extract a .7z archive into `destination_path`.""" + archive_path = Path(archive_path) + destination_path = Path(destination_path) + if not archive_path.exists(): + raise FileNotFoundError(f"The archive {archive_path} does not exist.") + destination_path.mkdir(parents=True, exist_ok=True) + with py7zr.SevenZipFile(archive_path, mode="r") as archive: + archive.extractall(path=destination_path) diff --git a/backend/src/mist/core/discord.py b/backend/src/mist/core/discord.py new file mode 100644 index 0000000..892a5fe --- /dev/null +++ b/backend/src/mist/core/discord.py @@ -0,0 +1,95 @@ +"""Discord webhook announcements. + +Ported from: JoshSteam CDN/Utils/discord.py +Bug fixes during port: + - Webhook URL moved from hardcoded source to settings.discord_webhook_url + - Bot username changed from "Josh Steam" to settings.discord_bot_username + - Now no-ops gracefully if webhook URL is unset (logs warning) +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +import requests + +from mist.config import settings + +log = logging.getLogger(__name__) + + +def _send(embed: dict[str, Any]) -> None: + """Send a single embed to the Discord webhook. No-ops if webhook not configured.""" + if not settings.discord_webhook_url: + log.warning("Discord webhook not configured; skipping send") + return + + payload = { + "embeds": [embed], + "username": settings.discord_bot_username, + "avatar_url": settings.discord_bot_avatar_url, + } + + try: + requests.post( + settings.discord_webhook_url, + data=json.dumps(payload), + headers={"Content-Type": "application/json"}, + timeout=10, + ) + except requests.RequestException as e: + log.warning("Discord webhook send failed: %s", e) + + +def announce_new_game( + game_title: str, + version: str, + game_description: str | None, + game_image_url: str | None, + game_size: str, +) -> None: + embed: dict[str, Any] = { + "title": f"New Game Added:\n{game_title}", + "description": game_description or "", + "color": 3447003, + "fields": [], + "footer": {"text": f"Version: {version}\nGame Size: {game_size}"}, + } + if game_image_url: + embed["image"] = {"url": game_image_url} + _send(embed) + + +def announce_update( + game_title: str, + version: str, + patch_notes_url: str | None, + game_description: str | None, + game_image_url: str | None, + update_size: str, +) -> None: + embed: dict[str, Any] = { + "title": f"Update Available:\n{game_title}", + "description": game_description or "", + "color": 3447003, + "fields": [], + "footer": {"text": f"Version: {version}\nUpdate Size: {update_size}"}, + } + if game_image_url: + embed["image"] = {"url": game_image_url} + if patch_notes_url: + embed["fields"].append( + {"name": "Patch Notes", "value": f"[Click here to read the patch notes]({patch_notes_url})"} + ) + _send(embed) + + +def announce_update_failure(game_title: str, version: str) -> None: + embed = { + "title": f"Update failed! {game_title} — {version}", + "color": 15158332, # red + "fields": [], + } + _send(embed) diff --git a/backend/src/mist/core/hdiff.py b/backend/src/mist/core/hdiff.py new file mode 100644 index 0000000..319cd3a --- /dev/null +++ b/backend/src/mist/core/hdiff.py @@ -0,0 +1,174 @@ +"""Direct-delta generation via HDiffPatch (`hdiffz`). + +Ported from: JoshSteam CDN/Utils/direct_patching.py +Bug fixes / changes during port: + - All hardcoded paths replaced with `mist.config.settings` references + - Uses `mist.core.paths` helpers for game-tree layout + - Type hints throughout + - Behavior preserved: same hdiffz options, same delta_manifest format + +A "direct" update is between two consecutive versions whose deltas were +pre-generated at push time. This module is what generates them. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from pathlib import Path +from typing import Any + +from mist.config import settings +from mist.core import paths + + +def _generate_hdiff_patch(old_file: Path, new_file: Path, delta_file: Path) -> bool: + """Run hdiffz to produce a single binary patch. Returns True on success.""" + try: + subprocess.run( + [ + str(settings.hdiffz_path), + "-SD-256k", # Single Data, 256KB step + "-p-8", # 8 threads + "-s-64k", # 64KB stream block + "-c-lzma2-9", # LZMA2 level 9 + str(old_file), + str(new_file), + str(delta_file), + ], + check=True, + ) + return True + except subprocess.CalledProcessError: + return False + + +def _generate_single_delta( + file: str, + new_file_info: dict[str, str], + old_manifest: dict[str, dict[str, str]], + depot_path: Path, + import_path: Path, + deltas_path: Path, + new_files_path: Path, +) -> dict[str, Any] | bool | None: + """Decide per-file action (new / modified / unchanged) and produce artifacts. + + Returns: + dict with `type` ∈ {'new', 'modified'} and `file_info` for the delta_manifest, + or None if the file is unchanged, + or False if delta generation failed (caller should abort). + """ + old_info = old_manifest.get(file) + new_checksum = new_file_info["checksum"] + + # Unchanged + if old_info and old_info["checksum"] == new_checksum: + return None + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + file_basename = os.path.basename(new_file_info["path"]) + + # New file + if old_info is None: + src = import_path / new_file_info["path"] + dest = new_files_path / f"{file_basename}_{timestamp}" + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest) + return { + "type": "new", + "file_info": { + "file": dest.name, + "original_filename": file_basename, + "relative_path": new_file_info["path"], + "checksum": new_checksum, + }, + } + + # Modified file + old_file_path = depot_path / old_info["path"] + new_file_path = import_path / new_file_info["path"] + delta_filename = f"{file.replace(os.sep, '_').replace('/', '_')}_{timestamp}.patch" + delta_path = deltas_path / delta_filename + + if _generate_hdiff_patch(old_file_path, new_file_path, delta_path): + return { + "type": "modified", + "file_info": {"file": delta_filename, "relative_path": new_file_info["path"]}, + } + return False + + +def generate_delta_patches( + game_title: str, old_version: str, new_version: str, import_path: Path | str +) -> bool: + """Generate hdiff patches + delta_manifest.json for old_version -> new_version. + + Reads manifests from the NAS, writes patches under deltas//. + """ + import_path = Path(import_path) + depot_path = paths.depot_dir(game_title) + old_manifest_path = paths.manifest_path(game_title, old_version) + new_manifest_path = paths.manifest_path(game_title, new_version) + deltas_path = paths.deltas_dir(game_title, new_version) + new_files_path = paths.new_files_dir(game_title, new_version) + + deltas_path.mkdir(parents=True, exist_ok=True) + new_files_path.mkdir(parents=True, exist_ok=True) + + with open(old_manifest_path) as f: + old_manifest: dict[str, dict[str, str]] = json.load(f) + with open(new_manifest_path) as f: + new_manifest: dict[str, dict[str, str]] = json.load(f) + + delta_manifest: dict[str, Any] = { + "modified_files": {}, + "new_files": [], + "deleted_files": [], + } + + with ThreadPoolExecutor(max_workers=6) as executor: + futures = [ + executor.submit( + _generate_single_delta, + file, + data, + old_manifest, + depot_path, + import_path, + deltas_path, + new_files_path, + ) + for file, data in new_manifest.items() + ] + for future in as_completed(futures): + result = future.result() + if result is False: + executor.shutdown(wait=False, cancel_futures=True) + return False + if result is None: + continue + if result["type"] == "new": + delta_manifest["new_files"].append(result["file_info"]) + else: + delta_manifest["modified_files"][result["file_info"]["file"]] = result["file_info"] + + # Deletions: present in old, absent in new + for file in old_manifest: + if file not in new_manifest: + delta_manifest["deleted_files"].append( + { + "file": os.path.basename(old_manifest[file]["path"]), + "relative_path": old_manifest[file]["path"], + "checksum": old_manifest[file]["checksum"], + } + ) + + with open(paths.delta_manifest_path(game_title, new_version), "w") as f: + json.dump(delta_manifest, f, indent=4) + + return True diff --git a/backend/src/mist/core/librsync.py b/backend/src/mist/core/librsync.py new file mode 100644 index 0000000..1c7fe7c --- /dev/null +++ b/backend/src/mist/core/librsync.py @@ -0,0 +1,88 @@ +"""Indirect-delta generation via librsync (`rdiff`). + +Ported from: JoshSteam CDN/Utils/indirect_patching.py +Bug fixes / changes during port: + - Hardcoded rdiff.exe path replaced with settings.rdiff_path + - Hardcoded TEMP_DIR replaced with settings.temp_dir + - Type hints + - Behavior preserved: hex-encoded signatures over the wire, .dlt deltas + +An "indirect" update is between arbitrary versions where the deltas were +NOT pre-generated. Flow: + 1. Client sends librsync `signature` of each file it has (hex-encoded) + 2. Server `generate_delta(file, signature_hex, ...)` produces a .dlt + 3. Client applies the .dlt with `rdiff patch` +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from datetime import datetime +from pathlib import Path + +from mist.config import settings + + +def copy_new_file(new_file: str, new_files_folder: Path | str, game_dir: Path | str) -> dict[str, str]: + """Copy a new (added) file into the new_files/ side-channel folder.""" + new_files_folder = Path(new_files_folder) + game_dir = Path(game_dir) + + new_file_path = game_dir / new_file + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + file_basename = os.path.basename(new_file) + new_dest_path = new_files_folder / f"{file_basename}_{timestamp}" + + new_dest_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(new_file_path, new_dest_path) + + return { + "file": new_dest_path.name, + "original_filename": file_basename, + "relative_path": new_file, + } + + +def generate_delta( + file: str, + signature_hex: str, + deltas_folder: Path | str, + from_version: str, + game_dir: Path | str, +) -> dict[str, str]: + """Produce a librsync delta for `file` given the client's signature. + + Writes the hex-decoded signature to a temp .sig, runs `rdiff delta`, + deletes the temp .sig, returns the delta metadata. + """ + deltas_folder = Path(deltas_folder) + game_dir = Path(game_dir) + + file_basename = os.path.basename(file) + signature_file = settings.temp_dir / f"{file_basename} {from_version}.sig" + settings.temp_dir.mkdir(parents=True, exist_ok=True) + + local_file = game_dir / file + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + delta_filename = f"{file_basename}_{timestamp}.dlt" + delta_file = deltas_folder / delta_filename + + with open(signature_file, "wb") as f: + f.write(bytes.fromhex(signature_hex)) + + try: + subprocess.run( + [str(settings.rdiff_path), "delta", str(signature_file), str(local_file), str(delta_file)], + check=True, + ) + finally: + if signature_file.exists(): + signature_file.unlink() + + return { + "file": delta_filename, + "original_filename": file_basename, + "relative_path": file, + } diff --git a/backend/src/mist/core/manifest.py b/backend/src/mist/core/manifest.py new file mode 100644 index 0000000..a105da3 --- /dev/null +++ b/backend/src/mist/core/manifest.py @@ -0,0 +1,141 @@ +"""Per-file SHA-256 manifests + per-game version-history helpers. + +Ported from: JoshSteam CDN/Utils/manifest.py +Bug fixes during port: + - Hardcoded GAMES_DIR removed; uses `mist.core.paths` instead. + +NOTE: the version-history-as-JSON functions (`update_game_version_manifest`, +`get_latest_version_from_game_manifest`, `is_direct_update`) are kept as +**legacy fallbacks**. The Postgres `Version` model is the new source of +truth. Callsites are expected to migrate; the JSON path remains for +emergency manual operations on the NAS. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from mist.core import paths + + +def file_checksum(file_path: Path | str) -> str: + """SHA-256 of a file, computed in 4KB chunks.""" + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + + +def generate_manifest(game_title: str, version: str, depot_path: Path | str) -> Path: + """Walk `depot_path`, compute SHA-256 for every file, write to manifests/.json.""" + depot_path = Path(depot_path) + manifest: dict[str, dict[str, str]] = {} + for root, _, files in __import__("os").walk(depot_path): + for file in files: + file_path = Path(root) / file + checksum = file_checksum(file_path) + relative_path = str(file_path.relative_to(depot_path)) + manifest[relative_path] = {"checksum": checksum, "path": relative_path} + + out_path = paths.manifest_path(game_title, version) + out_path.parent.mkdir(parents=True, exist_ok=True) + with open(out_path, "w") as f: + json.dump(manifest, f, indent=4) + return out_path + + +def load_manifest(manifest_file: Path | str) -> dict[str, dict[str, str]]: + """Load a per-version manifest file.""" + with open(manifest_file) as f: + return json.load(f) + + +# ---- Legacy version-history JSON helpers (preserved for manual ops) ---- + + +def update_game_version_manifest(game_title: str, version: str) -> None: + """Append a version to the per-game version-history JSON. Legacy.""" + history_file = paths.game_version_history_path(game_title) + if history_file.exists(): + with open(history_file) as f: + history: list[str] = json.load(f) + else: + history = [] + history.append(version) + history_file.parent.mkdir(parents=True, exist_ok=True) + with open(history_file, "w") as f: + json.dump(history, f, indent=4) + + +def get_latest_version_from_game_manifest(game_title: str) -> str | None: + """Last entry in the legacy version-history JSON.""" + history_file = paths.game_version_history_path(game_title) + if not history_file.exists(): + return None + with open(history_file) as f: + versions: list[str] = json.load(f) + return versions[-1] if versions else None + + +def get_earliest_version_from_game_manifest(game_title: str) -> str | None: + """First entry in the legacy version-history JSON.""" + history_file = paths.game_version_history_path(game_title) + if not history_file.exists(): + return None + with open(history_file) as f: + versions: list[str] = json.load(f) + return versions[0] if versions else None + + +def is_direct_update(game_title: str, version1: str, version2: str) -> bool: + """True iff version1 is the immediate predecessor of version2. Legacy.""" + history_file = paths.game_version_history_path(game_title) + if not history_file.exists(): + return False + with open(history_file) as f: + versions: list[str] = json.load(f) + if version1 not in versions or version2 not in versions: + return False + return versions.index(version1) + 1 == versions.index(version2) + + +# ---- Manifest verification ---- + + +def verify_files(game_dir: Path | str, manifest_file: Path | str) -> dict[str, Any]: + """Verify every file under `game_dir` against the SHA-256s in `manifest_file`. + + Returns a dict with `missing_files`, `mismatched_checksums`, `all_files_verified`. + """ + game_dir = Path(game_dir) + manifest_file = Path(manifest_file) + if not manifest_file.exists(): + raise FileNotFoundError(f"Manifest not found at {manifest_file}") + + with open(manifest_file) as f: + manifest_data: dict[str, dict[str, str]] = json.load(f) + + results: dict[str, Any] = { + "missing_files": [], + "mismatched_checksums": [], + "all_files_verified": True, + } + + for _, file_info in manifest_data.items(): + file_path = game_dir / file_info["path"] + if not file_path.exists(): + results["missing_files"].append(str(file_path)) + results["all_files_verified"] = False + continue + actual = file_checksum(file_path) + if actual != file_info["checksum"]: + results["mismatched_checksums"].append( + {"file": str(file_path), "expected": file_info["checksum"], "actual": actual} + ) + results["all_files_verified"] = False + + return results diff --git a/backend/src/mist/core/paths.py b/backend/src/mist/core/paths.py new file mode 100644 index 0000000..9282c54 --- /dev/null +++ b/backend/src/mist/core/paths.py @@ -0,0 +1,82 @@ +"""Centralized path resolution. + +Every callsite that needs to know where a game file lives goes through here. +No path is hardcoded anywhere else in the backend. +""" + +from __future__ import annotations + +from pathlib import Path + +from mist.config import settings + + +def game_dir(title: str) -> Path: + """Root folder for a single game on the NAS.""" + return settings.games_dir / title + + +def depot_dir(title: str) -> Path: + """Current latest-version files for a game.""" + return game_dir(title) / "depot" + + +def base_archive_path(title: str) -> Path: + """Immutable original .7z that all chain-replay starts from.""" + return game_dir(title) / "base_version.7z" + + +def manifests_dir(title: str) -> Path: + return game_dir(title) / "manifests" + + +def manifest_path(title: str, version: str) -> Path: + """Per-version SHA-256 file manifest.""" + return manifests_dir(title) / f"{version}.json" + + +def game_version_history_path(title: str) -> Path: + """Ordered list of versions for a game (legacy JSON; migrating to DB).""" + return manifests_dir(title) / f"{title}.json" + + +def deltas_dir(title: str, to_version: str) -> Path: + """Pre-generated direct-update delta artifacts for `to_version`.""" + return game_dir(title) / "deltas" / to_version + + +def delta_manifest_path(title: str, to_version: str) -> Path: + return deltas_dir(title, to_version) / "delta_manifest.json" + + +def new_files_dir(title: str, to_version: str) -> Path: + return deltas_dir(title, to_version) / "new_files" + + +def cache_archive_path(title: str, version: str) -> Path: + """Ready-to-serve full-game .tar.zst archive in the hot cache.""" + return settings.cache_dir / f"{title} {version}.tar.zst" + + +def cache_direct_update_path(title: str, from_version: str, to_version: str) -> Path: + return settings.cache_dir / f"{title}_{from_version}_{to_version}_Direct.tar.zst" + + +def cache_indirect_update_path(title: str, from_version: str, to_version: str) -> Path: + return settings.cache_dir / f"{title}_{from_version}_{to_version}_Indirect.tar.zst" + + +def cached_game_version_dir(title: str, version: str) -> Path: + """Reconstructed-historical-version folder living in cache.""" + return settings.cache_dir / f"{title} {version}" + + +def temp_game_dir(title: str, version: str) -> Path: + """Scratch folder for chain-replay work.""" + return settings.temp_dir / f"{title} {version}" + + +def ensure_dirs() -> None: + """Create the standard cache + temp dirs if missing. NAS is assumed pre-mounted.""" + settings.cache_dir.mkdir(parents=True, exist_ok=True) + settings.temp_dir.mkdir(parents=True, exist_ok=True) diff --git a/backend/src/mist/core/steam.py b/backend/src/mist/core/steam.py new file mode 100644 index 0000000..e4944ce --- /dev/null +++ b/backend/src/mist/core/steam.py @@ -0,0 +1,50 @@ +"""Steam appdetails pull-through. + +Ported from: JoshSteam CDN/Utils/steam_api.py +The Steam Storefront API is public and unauthenticated. We optionally pass +`settings.steam_api_key` if present (used elsewhere in the Steam ecosystem +for higher rate limits, harmless to include). +""" + +from __future__ import annotations + +from typing import Any + +import requests + +from mist.config import settings + + +def _get_steam_app_details(app_id: int | str) -> dict[str, Any] | None: + """Fetch raw appdetails for an app_id. Returns None on any failure.""" + url = f"https://store.steampowered.com/api/appdetails?appids={app_id}" + params: dict[str, str] = {} + if settings.steam_api_key: + params["key"] = settings.steam_api_key + + try: + response = requests.get(url, params=params, timeout=10) + except requests.RequestException: + return None + + if response.status_code != 200: + return None + + data = response.json() + payload = data.get(str(app_id)) + if not payload or not payload.get("success"): + return None + return payload.get("data") + + +def get_steam_app_info(app_id: int | str) -> tuple[str | None, str | None]: + """Return (short_description, header_image_url) for an app_id. + + Returns (None, None) if Steam doesn't know about the app or the call failed. + """ + details = _get_steam_app_details(app_id) + if not details: + return None, None + short_description = (details.get("short_description") or "").replace(""", '"') or None + header_image = details.get("header_image") or None + return short_description, header_image diff --git a/backend/src/mist/core/unrealpak.py b/backend/src/mist/core/unrealpak.py new file mode 100644 index 0000000..09ab918 --- /dev/null +++ b/backend/src/mist/core/unrealpak.py @@ -0,0 +1,133 @@ +"""Unreal Engine .pak extract / repack helper. + +Ported from: Pak City/test.py (+ find_pak_file.py, find_dotpack_folders.py) + +Purpose: Unreal Engine ships massive Oodle-compressed .pak files. Running +delta tools against them produces garbage ratios (input already entropy-maxed) +and treats every byte as changed on any modification. The fix is to extract +.pak files larger than a threshold before delta-gen, treat the contents as +loose files, then repack on the client after install/update. + +Flow: + 1. find_large_pak_files(game_dir) — scan for .pak >= 64MB + 2. extract_pak(unrealpak_path, pak_file, extracted_dir) — UnrealPak -Extract + 3. delete the original .pak; folder is renamed *DOTpak so we know to repack + 4. generate_data_to_pack(extracted_dir, data_to_pack_file) — generate dataToPack manifest + 5. (later, on client) repack_pak(...) using Oodle compression +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +from mist.config import settings + +DEFAULT_PAK_SIZE_LIMIT_MB = 64 + + +def _unrealpak() -> Path: + """Return the configured UnrealPak.exe path, raising if not set.""" + if settings.unrealpak_path is None: + raise RuntimeError( + "UNREALPAK_PATH is not set; this host cannot extract/repack Unreal .pak files." + ) + return settings.unrealpak_path + + +def find_large_pak_files(directory: Path | str, size_limit_mb: int = DEFAULT_PAK_SIZE_LIMIT_MB) -> list[Path]: + """Recursively find every .pak file >= size_limit_mb under `directory`.""" + directory = Path(directory) + size_limit_bytes = size_limit_mb * 1024 * 1024 + found: list[Path] = [] + for root, _, files in os.walk(directory): + for file in files: + if not file.lower().endswith(".pak"): + continue + file_path = Path(root) / file + try: + if file_path.stat().st_size > size_limit_bytes: + found.append(file_path) + except OSError: + continue + return found + + +def find_dotpak_folders(directory: Path | str) -> list[Path]: + """Find every folder marked as an extracted .pak (suffix 'DOTpak').""" + directory = Path(directory) + found: list[Path] = [] + for root, dirs, _ in os.walk(directory): + for d in dirs: + if d.endswith("DOTpak"): + found.append(Path(root) / d) + return found + + +def _run(args: list[str]) -> str: + """Run a subprocess command and return its stdout, raising on nonzero exit.""" + result = subprocess.run(args, capture_output=True, text=True, check=False) + if result.returncode != 0: + raise RuntimeError(f"Command failed: {' '.join(args)}\n{result.stderr}") + return result.stdout + + +def extract_pak(pak_file: Path | str, output_dir: Path | str) -> None: + """Extract a .pak file into output_dir using UnrealPak -Extract.""" + pak_file = Path(pak_file) + output_dir = Path(output_dir) + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + _run([str(_unrealpak()), str(pak_file), "-Extract", str(output_dir)]) + + +def generate_data_to_pack(input_dir: Path | str, output_file: Path | str) -> None: + """Write a UnrealPak dataToPack.txt manifest for repacking.""" + input_dir = Path(input_dir) + output_file = Path(output_file) + with open(output_file, "w") as f: + for root, _, files in os.walk(input_dir): + if not files: + continue + for file in files: + if file == output_file.name: + continue + file_path = Path(root) / file + relative_path = file_path.relative_to(input_dir) + target_dir = str(relative_path.parent).replace("\\", "/") + f.write(f'"{file_path}" "../../../{target_dir}/"\n') + + +def repack_pak(new_pak_file: Path | str, data_to_pack_file: Path | str) -> None: + """Repack a folder back into a .pak with Oodle compression.""" + new_pak_file = Path(new_pak_file) + data_to_pack_file = Path(data_to_pack_file) + _run( + [ + str(_unrealpak()), + str(new_pak_file), + f"-Create={data_to_pack_file}", + "-compress", + "-compressionformat=Oodle", + ] + ) + + +def extract_all_large_paks(game_dir: Path | str, size_limit_mb: int = DEFAULT_PAK_SIZE_LIMIT_MB) -> list[Path]: + """Convenience: walk a game dir, extract every large .pak, delete originals. + + Returns the list of `*DOTpak/` folders created. + """ + game_dir = Path(game_dir) + extracted: list[Path] = [] + for pak_file in find_large_pak_files(game_dir, size_limit_mb): + dir_path = pak_file.parent + base_name = pak_file.name.replace(".pak", "DOTpak") + extracted_dir = dir_path / base_name + extract_pak(pak_file, extracted_dir) + pak_file.unlink() + extracted.append(extracted_dir) + return extracted diff --git a/backend/src/mist/db/__init__.py b/backend/src/mist/db/__init__.py new file mode 100644 index 0000000..e8e3c2b --- /dev/null +++ b/backend/src/mist/db/__init__.py @@ -0,0 +1 @@ +"""Mist database layer — SQLAlchemy models, session factory, Alembic migrations.""" diff --git a/backend/src/mist/db/base.py b/backend/src/mist/db/base.py new file mode 100644 index 0000000..55a76e3 --- /dev/null +++ b/backend/src/mist/db/base.py @@ -0,0 +1,27 @@ +"""SQLAlchemy engine, session factory, declarative base.""" + +from __future__ import annotations + +from collections.abc import Iterator + +from sqlalchemy import create_engine +from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker + +from mist.config import settings + + +class Base(DeclarativeBase): + """Shared declarative base for every model in mist.db.models.""" + + +engine = create_engine(settings.database_url, pool_pre_ping=True, future=True) +SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True) + + +def get_session() -> Iterator[Session]: + """FastAPI dep — yields a Session, closes on request teardown.""" + session = SessionLocal() + try: + yield session + finally: + session.close() diff --git a/backend/src/mist/db/migrations/env.py b/backend/src/mist/db/migrations/env.py new file mode 100644 index 0000000..91bdb20 --- /dev/null +++ b/backend/src/mist/db/migrations/env.py @@ -0,0 +1,48 @@ +"""Alembic environment — pulls the DB URL from mist.config and the metadata from mist.db.models.""" + +from __future__ import annotations + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from mist.config import settings +from mist.db.base import Base +from mist.db import models # noqa: F401 — import side-effect registers tables on Base.metadata + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata +config.set_main_option("sqlalchemy.url", settings.database_url) + + +def run_migrations_offline() -> None: + context.configure( + url=settings.database_url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/src/mist/db/migrations/script.py.mako b/backend/src/mist/db/migrations/script.py.mako new file mode 100644 index 0000000..9e885b8 --- /dev/null +++ b/backend/src/mist/db/migrations/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: str | None = ${repr(down_revision)} +branch_labels: str | Sequence[str] | None = ${repr(branch_labels)} +depends_on: str | Sequence[str] | None = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/backend/src/mist/db/migrations/versions/0001_initial.py b/backend/src/mist/db/migrations/versions/0001_initial.py new file mode 100644 index 0000000..757ccfb --- /dev/null +++ b/backend/src/mist/db/migrations/versions/0001_initial.py @@ -0,0 +1,91 @@ +"""Initial schema — users, games, versions, build_jobs. + +Revision ID: 0001 +Revises: +Create Date: 2026-06-07 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "0001" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "users", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("username", sa.String(64), nullable=False, unique=True), + sa.Column("password_hash", sa.String(255), nullable=False), + sa.Column("is_admin", sa.Boolean(), nullable=False, server_default=sa.text("false")), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("ix_users_username", "users", ["username"], unique=True) + + op.create_table( + "games", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("title", sa.String(255), nullable=False, unique=True), + sa.Column("app_id", sa.Integer(), nullable=True), + sa.Column("description_override", sa.Text(), nullable=True), + sa.Column("header_image_override", sa.String(1024), nullable=True), + sa.Column("is_private", sa.Boolean(), nullable=False, server_default=sa.text("false")), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("ix_games_title", "games", ["title"], unique=True) + op.create_index("ix_games_is_private", "games", ["is_private"]) + op.create_index("ix_games_deleted_at", "games", ["deleted_at"]) + + op.create_table( + "versions", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("game_id", sa.Integer(), sa.ForeignKey("games.id", ondelete="CASCADE"), nullable=False), + sa.Column("version_string", sa.String(64), nullable=False), + sa.Column("ordinal", sa.Integer(), nullable=False), + sa.Column("manifest_hash", sa.String(64), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.UniqueConstraint("game_id", "version_string", name="uq_versions_game_id_version_string"), + ) + op.create_index("ix_versions_game_id_ordinal", "versions", ["game_id", "ordinal"]) + + build_job_kind = sa.Enum( + "import_new_game", + "push_update", + "generate_direct_update", + "generate_indirect_update", + "prepare_full_game", + name="buildjobkind", + ) + build_job_state = sa.Enum("pending", "running", "success", "failure", name="buildjobstate") + + op.create_table( + "build_jobs", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("celery_task_id", sa.String(128), nullable=True), + sa.Column("game_id", sa.Integer(), sa.ForeignKey("games.id", ondelete="SET NULL"), nullable=True), + sa.Column("kind", build_job_kind, nullable=False), + sa.Column("state", build_job_state, nullable=False, server_default="pending"), + sa.Column("detail", sa.Text(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("ix_build_jobs_celery_task_id", "build_jobs", ["celery_task_id"]) + + +def downgrade() -> None: + op.drop_table("build_jobs") + op.execute("DROP TYPE IF EXISTS buildjobstate") + op.execute("DROP TYPE IF EXISTS buildjobkind") + op.drop_table("versions") + op.drop_table("games") + op.drop_table("users") diff --git a/backend/src/mist/db/models.py b/backend/src/mist/db/models.py new file mode 100644 index 0000000..60f7e83 --- /dev/null +++ b/backend/src/mist/db/models.py @@ -0,0 +1,113 @@ +"""Mist data model. + +Four tables for MVP: + users — accounts (provisioned by admin) + games — catalog entries, one per game + versions — per-game ordered version history (replaces legacy .json) + build_jobs — async work tracking for delta-gen, archive prep, etc. +""" + +from __future__ import annotations + +import enum +from datetime import datetime + +from sqlalchemy import ( + Boolean, + DateTime, + Enum, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, + func, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mist.db.base import Base + + +class User(Base): + __tablename__ = "users" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + username: Mapped[str] = mapped_column(String(64), unique=True, nullable=False, index=True) + password_hash: Mapped[str] = mapped_column(String(255), nullable=False) + is_admin: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + +class Game(Base): + __tablename__ = "games" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + title: Mapped[str] = mapped_column(String(255), unique=True, nullable=False, index=True) + app_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + description_override: Mapped[str | None] = mapped_column(Text, nullable=True) + header_image_override: Mapped[str | None] = mapped_column(String(1024), nullable=True) + is_private: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False, index=True) + deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + versions: Mapped[list[Version]] = relationship( + back_populates="game", cascade="all, delete-orphan", order_by="Version.ordinal" + ) + + +class Version(Base): + __tablename__ = "versions" + __table_args__ = ( + UniqueConstraint("game_id", "version_string", name="uq_versions_game_id_version_string"), + Index("ix_versions_game_id_ordinal", "game_id", "ordinal"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + game_id: Mapped[int] = mapped_column(ForeignKey("games.id", ondelete="CASCADE"), nullable=False) + version_string: Mapped[str] = mapped_column(String(64), nullable=False) + ordinal: Mapped[int] = mapped_column(Integer, nullable=False) + manifest_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + game: Mapped[Game] = relationship(back_populates="versions") + + +class BuildJobKind(str, enum.Enum): + IMPORT_NEW_GAME = "import_new_game" + PUSH_UPDATE = "push_update" + GENERATE_DIRECT_UPDATE = "generate_direct_update" + GENERATE_INDIRECT_UPDATE = "generate_indirect_update" + PREPARE_FULL_GAME = "prepare_full_game" + + +class BuildJobState(str, enum.Enum): + PENDING = "pending" + RUNNING = "running" + SUCCESS = "success" + FAILURE = "failure" + + +class BuildJob(Base): + __tablename__ = "build_jobs" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + celery_task_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) + game_id: Mapped[int | None] = mapped_column(ForeignKey("games.id", ondelete="SET NULL"), nullable=True) + kind: Mapped[BuildJobKind] = mapped_column(Enum(BuildJobKind), nullable=False) + state: Mapped[BuildJobState] = mapped_column( + Enum(BuildJobState), nullable=False, default=BuildJobState.PENDING + ) + detail: Mapped[str | None] = mapped_column(Text, nullable=True) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) diff --git a/backend/src/mist/worker/__init__.py b/backend/src/mist/worker/__init__.py new file mode 100644 index 0000000..95cc9b2 --- /dev/null +++ b/backend/src/mist/worker/__init__.py @@ -0,0 +1,56 @@ +"""Celery app factory. + +Reconstructed from the missing JoshSteam CDN/Tasks/__init__.py source. +Bug fixes during port: + - The prototype's `check_if_task_exists` had `celery.control.inspect.reserved()` + (missing parens) and assigned `inspect().active()` to both `active_tasks` and + `reserved_tasks`. Both fixed in `task_already_queued` below. + +Run the worker with: + celery -A mist.worker worker --loglevel=INFO +""" + +from __future__ import annotations + +from celery import Celery + +from mist.config import settings + + +def make_celery() -> Celery: + app = Celery( + "mist", + broker=settings.celery_broker_url, + backend=settings.celery_result_backend, + include=["mist.worker.tasks", "mist.worker.notifications"], + ) + app.conf.update( + task_track_started=True, + task_acks_late=True, + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + ) + return app + + +celery_app = make_celery() + + +def task_already_queued(task_id: str) -> bool: + """Is there already an active or reserved task with this `task_id`? + + Fixed version of the prototype's check_if_task_exists. + """ + inspector = celery_app.control.inspect() + active = inspector.active() or {} + reserved = inspector.reserved() or {} + + for _, tasks in active.items(): + for t in tasks: + if t.get("id") == task_id: + return True + for _, tasks in reserved.items(): + for t in tasks: + if t.get("id") == task_id: + return True + return False diff --git a/backend/src/mist/worker/notifications.py b/backend/src/mist/worker/notifications.py new file mode 100644 index 0000000..987f4e8 --- /dev/null +++ b/backend/src/mist/worker/notifications.py @@ -0,0 +1,16 @@ +"""Notification fan-out — consumes events, dispatches to Discord (and future channels). + +Skeleton: real implementation will subscribe to a topic on RabbitMQ and call +mist.core.discord.* for each event type. For MVP, push_update / import_new_game +can call core.discord directly without going through this layer. +""" + +from __future__ import annotations + +from mist.worker import celery_app + + +@celery_app.task(name="mist.notifications.dispatch") +def dispatch(_event_type: str, _payload: dict) -> None: + """Route a domain event to its notification channel(s).""" + raise NotImplementedError("notifications.dispatch not implemented yet") diff --git a/backend/src/mist/worker/tasks.py b/backend/src/mist/worker/tasks.py new file mode 100644 index 0000000..3fe74de --- /dev/null +++ b/backend/src/mist/worker/tasks.py @@ -0,0 +1,63 @@ +"""Celery task definitions. + +Each task is a thin wrapper around `mist.core.*` functions. The patching IP +lives in `core/`; this file is just plumbing. +""" + +from __future__ import annotations + +from mist.worker import celery_app + + +@celery_app.task(name="mist.tasks.import_new_game") +def import_new_game(_data: dict) -> dict: + """Initial import of a game's base version from an uploaded archive. + + Expected flow once implemented: + 1. Move uploaded archive to NAS as base_version.7z + 2. Extract to depot/ + 3. Generate per-version manifest via core.manifest.generate_manifest + 4. Pre-cache the full-game .tar.zst via core.compression.compress_and_save_zstd + 5. Insert Game + Version rows; call core.discord.announce_new_game + """ + raise NotImplementedError("import_new_game not implemented yet") + + +@celery_app.task(name="mist.tasks.push_update") +def push_update(_data: dict) -> dict: + """Receive a new version's full files, generate per-version manifest + + hdiff direct deltas vs previous version, swap depot. + + Replaces the prototype's push_update task. The chain is: + core.manifest.generate_manifest + core.hdiff.generate_delta_patches + (move new files into depot/) + Version row insert + core.discord.announce_update + """ + raise NotImplementedError("push_update not implemented yet") + + +@celery_app.task(name="mist.tasks.generate_direct_update") +def generate_direct_update(_game_title: str, _from_version: str, _to_version: str) -> dict: + """Pack the pre-built deltas for a direct (consecutive) update into a .tar.zst.""" + raise NotImplementedError("generate_direct_update not implemented yet") + + +@celery_app.task(name="mist.tasks.generate_indirect_update") +def generate_indirect_update( + _game_title: str, _from_version: str, _to_version: str, _signatures_path: str +) -> dict: + """Generate librsync .dlt deltas based on client-provided signatures, pack into .tar.zst. + + Replaces the prototype's Tasks/generate_indirect_update.py. Calls + core.librsync.generate_delta per file. May call core.chain_replay.prepare_game_version + if `to_version` is not the latest and not cached. + """ + raise NotImplementedError("generate_indirect_update not implemented yet") + + +@celery_app.task(name="mist.tasks.prepare_full_game_archive") +def prepare_full_game_archive(_game_title: str, _version: str) -> dict: + """Reconstruct (if needed) and pack `(game_title, version)` as a .tar.zst into cache.""" + raise NotImplementedError("prepare_full_game_archive not implemented yet") diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..5240709 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1 @@ +"""Test configuration. Empty for now — present so pytest discovers the tests/ dir.""" diff --git a/backend/tests/test_smoke.py b/backend/tests/test_smoke.py new file mode 100644 index 0000000..9e2823e --- /dev/null +++ b/backend/tests/test_smoke.py @@ -0,0 +1,26 @@ +"""Smoke test: every module imports cleanly. + +If a dep is missing, a syntax error slipped through, or a circular import got +introduced, this test fails. Cheap insurance. +""" + + +def test_core_modules_import() -> None: + from mist.core import ( # noqa: F401 + chain_replay, + compression, + discord, + hdiff, + librsync, + manifest, + paths, + steam, + unrealpak, + ) + + +def test_config_loads() -> None: + from mist.config import settings + + assert settings.environment in {"development", "staging", "production"} or settings.environment + # don't assert specific values; just that it loaded diff --git a/client/README.md b/client/README.md new file mode 100644 index 0000000..9bf11e4 --- /dev/null +++ b/client/README.md @@ -0,0 +1,26 @@ +# Mist client + +The Tauri-based desktop client for Mist. Rust shell, Svelte UI inside. + +## Dev quickstart + +```sh +pnpm install +pnpm tauri dev +``` + +Tauri builds a native installer per platform via `pnpm tauri build`. Initial target is Windows (friends are on Windows); macOS/Linux straightforward to add. + +## What goes here + +- **`src/`** — Svelte UI (browse store, install, update, launch screens) +- **`src-tauri/`** — Rust shell, native commands, file-system integration + +## Patch application + +The two patch tools (`hpatchz` for direct updates, `rdiff` for indirect) are external binaries. The plan is to either: + +1. Port `apply_hdiff_patch` / `apply_librsync_patch` to Rust using `tauri::command` so the UI invokes them natively, or +2. Bundle a small Python sidecar binary built with PyInstaller that the Tauri shell spawns for each patch op (uses the same `mist.core.hdiff` / `mist.core.librsync` modules as the backend worker, except this side only *applies*, not *generates*). + +Decision deferred to the implementation phase. diff --git a/client/index.html b/client/index.html new file mode 100644 index 0000000..e270ba9 --- /dev/null +++ b/client/index.html @@ -0,0 +1,12 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <title>Mist + + +
+ + + diff --git a/client/package.json b/client/package.json new file mode 100644 index 0000000..6ce5569 --- /dev/null +++ b/client/package.json @@ -0,0 +1,25 @@ +{ + "name": "mist-client", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "tauri": "tauri", + "check": "svelte-check --tsconfig ./tsconfig.json" + }, + "dependencies": { + "@tauri-apps/api": "^2.1.0" + }, + "devDependencies": { + "@sveltejs/vite-plugin-svelte": "^4.0.0", + "@tauri-apps/cli": "^2.1.0", + "svelte": "^5.0.0", + "svelte-check": "^4.0.0", + "tslib": "^2.7.0", + "typescript": "^5.6.0", + "vite": "^5.4.0" + } +} diff --git a/client/src-tauri/Cargo.toml b/client/src-tauri/Cargo.toml new file mode 100644 index 0000000..3b80c7d --- /dev/null +++ b/client/src-tauri/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "mist-client" +version = "0.1.0" +description = "Mist desktop client" +authors = ["Josh"] +edition = "2021" + +[build-dependencies] +tauri-build = { version = "2", features = [] } + +[dependencies] +tauri = { version = "2", features = [] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[features] +default = ["custom-protocol"] +custom-protocol = ["tauri/custom-protocol"] diff --git a/client/src-tauri/build.rs b/client/src-tauri/build.rs new file mode 100644 index 0000000..d860e1e --- /dev/null +++ b/client/src-tauri/build.rs @@ -0,0 +1,3 @@ +fn main() { + tauri_build::build() +} diff --git a/client/src-tauri/src/main.rs b/client/src-tauri/src/main.rs new file mode 100644 index 0000000..8d5f2d2 --- /dev/null +++ b/client/src-tauri/src/main.rs @@ -0,0 +1,14 @@ +// Prevents an extra console window on Windows in release. +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] + +fn main() { + tauri::Builder::default() + // TODO: register #[tauri::command] handlers for: + // - apply_hdiff_patch(old, patch, new) -> bool + // - apply_librsync_patch(old, delta, new) -> bool + // - generate_librsync_signature(file) -> Vec + // - install_game(game_id, version) -> JobHandle + // - launch_game(game_id) -> () + .run(tauri::generate_context!()) + .expect("error while running tauri application"); +} diff --git a/client/src-tauri/tauri.conf.json b/client/src-tauri/tauri.conf.json new file mode 100644 index 0000000..70c844b --- /dev/null +++ b/client/src-tauri/tauri.conf.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "productName": "Mist", + "version": "0.1.0", + "identifier": "app.mist.client", + "build": { + "beforeDevCommand": "npm run dev", + "beforeBuildCommand": "npm run build", + "devUrl": "http://localhost:1420", + "frontendDist": "../dist" + }, + "app": { + "windows": [ + { + "title": "Mist", + "width": 1200, + "height": 800, + "minWidth": 900, + "minHeight": 600 + } + ], + "security": { + "csp": null + } + }, + "bundle": { + "active": true, + "targets": "all", + "icon": [] + } +} diff --git a/client/src/App.svelte b/client/src/App.svelte new file mode 100644 index 0000000..7244241 --- /dev/null +++ b/client/src/App.svelte @@ -0,0 +1,78 @@ + + +
+
+

Mist

+ +
+
+ {#if view === 'store'} +

Store

+

Browse games here. Skeleton — backend wiring TBD.

+ {:else if view === 'library'} +

Library

+

Your installed games will show up here.

+ {:else} +

Downloads

+

In-flight downloads and updates.

+ {/if} +
+
+ + diff --git a/client/src/main.ts b/client/src/main.ts new file mode 100644 index 0000000..c4afabc --- /dev/null +++ b/client/src/main.ts @@ -0,0 +1,6 @@ +import { mount } from 'svelte'; +import App from './App.svelte'; + +const app = mount(App, { target: document.getElementById('app')! }); + +export default app; diff --git a/client/svelte.config.js b/client/svelte.config.js new file mode 100644 index 0000000..21b9399 --- /dev/null +++ b/client/svelte.config.js @@ -0,0 +1,5 @@ +import { vitePreprocess } from '@sveltejs/vite-plugin-svelte'; + +export default { + preprocess: vitePreprocess() +}; diff --git a/client/tsconfig.json b/client/tsconfig.json new file mode 100644 index 0000000..7a6ac92 --- /dev/null +++ b/client/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ESNext", + "useDefineForClassFields": true, + "module": "ESNext", + "resolveJsonModule": true, + "allowJs": true, + "checkJs": true, + "isolatedModules": true, + "moduleResolution": "bundler", + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "types": ["svelte", "vite/client"] + }, + "include": ["src/**/*.ts", "src/**/*.svelte"] +} diff --git a/client/vite.config.ts b/client/vite.config.ts new file mode 100644 index 0000000..88d0b4e --- /dev/null +++ b/client/vite.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vite'; +import { svelte } from '@sveltejs/vite-plugin-svelte'; + +// https://vitejs.dev/config/ +export default defineConfig({ + plugins: [svelte()], + clearScreen: false, + server: { + port: 1420, + strictPort: true + }, + envPrefix: ['VITE_', 'TAURI_'], + build: { + target: 'esnext' + } +}); diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..57ee815 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,54 @@ +# Mist — development overlay. +# Use: `docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build` +# +# Differences from prod: +# - Builds images locally instead of pulling from GHCR +# - Bind-mounts backend src for hot reload (uvicorn --reload) +# - Exposes infra ports so you can connect from your host + +services: + postgres: + ports: + - "5432:5432" + + redis: + ports: + - "6379:6379" + + rabbitmq: + ports: + - "5672:5672" + - "15672:15672" # management UI + + api: + image: mist-backend:dev + build: + context: ./backend + command: + [ + "uvicorn", + "mist.api.app:app", + "--host", + "0.0.0.0", + "--port", + "8000", + "--reload", + ] + volumes: + - ./backend/src:/app/src + + worker: + image: mist-backend:dev + build: + context: ./backend + # Watch the source tree and restart on changes (celery doesn't auto-reload). + # For dev, just restart the container manually after big changes. + volumes: + - ./backend/src:/app/src + + admin-web: + image: mist-admin-web:dev + build: + context: ./admin-web + volumes: + - ./admin-web/src:/app/src diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8f18084 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,97 @@ +# Mist — production-ish Docker Compose stack. +# Brings up: postgres, redis, rabbitmq, api, worker, admin-web. +# All env config comes from .env (copy from .env.example). + +services: + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - postgres-vol:/var/lib/postgresql/data + networks: + - mist-net + + redis: + image: redis:7-alpine + restart: unless-stopped + volumes: + - redis-vol:/data + networks: + - mist-net + + rabbitmq: + image: rabbitmq:3.13-management-alpine + restart: unless-stopped + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS} + volumes: + - rabbitmq-vol:/var/lib/rabbitmq + networks: + - mist-net + + api: + image: ghcr.io/REPLACE_ME/mist-backend:latest + build: + context: ./backend + restart: unless-stopped + depends_on: + - postgres + - redis + - rabbitmq + env_file: + - .env + volumes: + - cache-vol:/mist/cache + - tmp-vol:/mist/tmp + # TODO: bind-mount NAS NFS export here. Configure host-side NFS mount, + # then uncomment the line below and point it at the host mount point. + # - /mnt/nas:/mnt/nas:rw + ports: + - "8000:8000" + networks: + - mist-net + + worker: + image: ghcr.io/REPLACE_ME/mist-backend:latest + build: + context: ./backend + restart: unless-stopped + command: ["celery", "-A", "mist.worker", "worker", "--loglevel=INFO"] + depends_on: + - postgres + - redis + - rabbitmq + env_file: + - .env + volumes: + - cache-vol:/mist/cache + - tmp-vol:/mist/tmp + # - /mnt/nas:/mnt/nas:rw + networks: + - mist-net + + admin-web: + image: ghcr.io/REPLACE_ME/mist-admin-web:latest + build: + context: ./admin-web + restart: unless-stopped + ports: + - "5173:80" + networks: + - mist-net + +volumes: + postgres-vol: + redis-vol: + rabbitmq-vol: + cache-vol: + tmp-vol: + +networks: + mist-net: + driver: bridge diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..4570927 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,140 @@ +# Mist — Architecture + +## Purpose + +A private Steam-clone for ~5–10 friends. Distributes games and updates with bandwidth-efficient delta patching. Sized to the actual problem — the complexity is in the delta-patching system, not in the deployment. + +## Topology + +Friends use a regular web/desktop client over the open internet. Public DNS resolves to a border server. Nginx Proxy Manager terminates TLS (Let's Encrypt) and reverse-proxies to the backend VM over Tailscale. The backend VM itself has no public exposure. + +``` + ┌──────────────────────────────┐ + Friend │ Tauri client (Svelte UI │ + (open │ inside Rust shell) │ + internet) └──────────────┬───────────────┘ + │ HTTPS, public domain + │ store.mist.example + │ admin.mist.example + │ dl.mist.example + ▼ + ┌─────────────────────────────────┐ + │ Border server (public IP) │ + │ Nginx Proxy Manager + Let's │ + │ Encrypt TLS termination │ + └────────────────┬────────────────┘ + │ Tailscale (private backhaul) + ▼ + ┌─────────────────────────────────────────────┐ + │ Proxmox VM "mist" │ + │ Docker Compose stack: │ + │ │ + │ ┌──────────────┐ ┌──────────────────┐ │ + │ │ api │ │ admin-web │ │ + │ │ FastAPI: │ │ static Svelte/ │ │ + │ │ /auth │ │ TS served by │ │ + │ │ /catalog │ │ tiny nginx │ │ + │ │ /admin │ │ (calls api with │ │ + │ │ /downloads │ │ admin JWT) │ │ + │ │ /builds │ └──────────────────┘ │ + │ └──────┬───────┘ │ + │ │ │ + │ ▼ │ + │ ┌──────────────┐ │ + │ │ worker │ │ + │ │ Celery │ same image, │ + │ │ delta-gen, │ different entrypoint │ + │ │ archive prep,│ │ + │ │ notifications│ │ + │ └──────┬───────┘ │ + │ │ │ + │ ▼ │ + │ ┌──────────┐ ┌────────┐ ┌─────────────┐ │ + │ │ postgres │ │ redis │ │ rabbitmq │ │ + │ └──────────┘ └────────┘ └─────────────┘ │ + │ │ + │ Volumes: │ + │ - hot cache (.tar.zst archives) │ + │ - postgres data │ + │ - redis data │ + │ - rabbitmq data │ + │ - /mnt/nas → NFS to NAS (games) │ + └────────────────────────────────────────────┘ +``` + +## Containers + +| Container | Stack | Owns | +|---|---|---| +| **api** | FastAPI + SQLAlchemy + Postgres + Redis | Single web app with internally-modular code: `auth/`, `catalog/`, `admin/`, `downloads/`, `builds/`. Issues JWTs. Serves resumable downloads. Receives `mistpipe` uploads. Queues background work into RabbitMQ. | +| **worker** | Celery (same image as `api`, different entrypoint) | Consumes RabbitMQ. Runs the heavy stuff: hdiff delta generation, librsync indirect-delta generation, `chain_replay` cold reconstruction, `.tar.zst` archive packing, Discord notifications. | +| **admin-web** | SvelteKit, built static + tiny nginx | Admin UI. Calls `api/admin/*` with admin JWT. | +| **postgres** | postgres:16 | Catalog, users, build job state. | +| **redis** | redis:7 | Celery result backend, cache, ephemeral session data. | +| **rabbitmq** | rabbitmq:3.13-management | Celery broker, event bus for `notification.*` events. | + +## Non-container artifacts + +| Artifact | Stack | Notes | +|---|---|---| +| **client** | Tauri 2 (Rust shell + Svelte UI) | Friend-facing app. Distributed as a per-platform installer. Embeds (or spawns) the patch-application logic. | +| **mistpipe** | Python + click | Admin CLI. `login`, `new-game`, `push`, `ls`, `rm`, `resync-steam`. JWT stored in OS keychain. | + +## Storage + +**NAS** (mounted at `/mnt/nas` inside the VM via NFS) is the **source of truth** for game files: + +``` +/mnt/nas/mist/games// + base_version.7z ← immutable original + depot/ ← current latest version's files + manifests/ + <Title>.json ← ordered linear version list (legacy; will migrate to Postgres) + <version>.json ← per-version SHA-256 manifest + deltas/<version>/ + delta_manifest.json + new_files/... + *.patch ← hdiff direct patches +``` + +**Docker volumes** on the VM hold the hot path: + +- `cache-vol` — `.tar.zst` archives ready to serve, reconstructed historical versions +- `tmp-vol` — in-flight delta-gen working dirs +- `postgres-vol`, `redis-vol`, `rabbitmq-vol` — service data + +## Update modes + +Two delta strategies, decided at request time: + +- **Direct update** = consecutive versions (`1.0.0.2` → `1.0.0.3`). Deltas were pre-generated by `hdiff` at push time. Serve from cache or zip on demand. +- **Indirect update** = arbitrary version jumps (`1.0.0.0` → `1.0.0.3`). Server tells client which files changed. Client generates `librsync` signatures of its local files, POSTs them. Worker generates `rdiff` deltas against the server's copy and packs them into a `.tar.zst`. Client applies with `rdiff patch`. + +For arbitrary historical reconstructions the worker runs `chain_replay` — starts from the base or closest cached version and walks forward applying hdiff patches at each step, caching results for next time. + +## Auth + +- Username + password, you provision accounts via the admin portal +- Passwords hashed with argon2id +- JWT issued on login, scope claim distinguishes `user` from `admin` +- Admin scope required for `/admin/*` and `/builds/*` +- Per-game `is_private` boolean flag; non-admin users only see public games + games they've been explicitly granted (future) + +## Catalog metadata + +Steam appdetails pull-through: when an admin adds a game with an `app_id`, the catalog service fetches `https://store.steampowered.com/api/appdetails` and stores `short_description` + `header_image`. Admin can override either via hand-edited fields. + +## Out of scope (for MVP) + +- Branches (stable/beta/internal) +- Save sync, achievements, friend lists, in-app chat +- Payments +- Multi-tenancy (one store) +- Public self-serve signup +- Per-user entitlements beyond `is_private` +- Client-side delta-gen in `mistpipe` (server does it for MVP) +- High availability — single VM is fine at this scale; backup + restore covers failure + +## Why this shape + +The original "Josh Steam" prototypes proved out the hard parts: two-mode delta-patching, chain-replay for cold reconstruction, resumable downloads. The 2-year-later rebuild focuses on **finishing the product**, not re-exploring the design space. Microservices and k8s were considered and rejected for the actual scale — see `DECISIONS.md`. diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md new file mode 100644 index 0000000..405b6d8 --- /dev/null +++ b/docs/DECISIONS.md @@ -0,0 +1,109 @@ +# Mist — Decisions Log + +This file is an append-only log of significant decisions, in lightweight ADR (Architecture Decision Record) format. The goal is that future-you (or a contributor) can reconstruct *why* a choice was made, not just *what* was chosen. + +## Format + +Each entry: + +``` +## NNNN — Short title (YYYY-MM-DD) + +**Status:** Accepted | Superseded | Deprecated +**Context:** What problem were we solving? What forces were at play? +**Decision:** What did we decide? +**Consequences:** What does this make easier? Harder? +**Alternatives considered:** What else we looked at and why we passed. +``` + +--- + +## 0001 — Project named "Mist" (supersedes "Josh Steam") (2026-06-07) + +**Status:** Accepted +**Context:** The original prototype was named "Josh Steam" because it was a personal project for the author and his friends. The rebuild is a real product (private but real) and benefits from a name that travels. +**Decision:** Project name is **Mist**. CLI is `mistpipe` (homage to Steam's SteamPipe). Docker images namespaced `mist-*`. Domain pattern `*.mist.example` in docs. +**Consequences:** All references to "Josh Steam" or "joshsteamctl" in any new code/docs must use the new name. Existing prototypes at `Josh Steam/` on disk stay untouched as historical reference. +**Alternatives considered:** Keep "Josh Steam". Rejected — uncomfortable to share, and the name doesn't say what it is. + +--- + +## 0002 — Single-VM Docker Compose instead of Kubernetes (2026-06-07) + +**Status:** Accepted +**Context:** Original draft of the architecture proposed 7 microservices on a multi-node k3s/rke2 cluster with ArgoCD GitOps, Longhorn storage, MetalLB load balancer, and the Tailscale operator. The framing was "use this as an excuse to learn k8s and microservices." +**Decision:** Run the backend as Docker Compose on a single Proxmox VM. Six containers total: `api`, `worker`, `admin-web`, `postgres`, `redis`, `rabbitmq`. Stateful services share the same compose stack with named volumes. NAS mounted via NFS. +**Consequences:** Massively less operational complexity. Deploy is `docker compose pull && up -d` over SSH. No service mesh, no ingress controller, no GitOps tooling to learn before the product runs. The project itself (delta-patching, content distribution) is already complex enough; deployment shouldn't compound it. Trade-off: less k8s/microservices résumé padding. +**Alternatives considered:** +- Multi-node k8s + GitOps + 7 microservices. Rejected — adds learning surface unrelated to the actual problem and is wildly oversized for ~10 users. +- Modular monolith on bare metal (no containers). Rejected — losing the reproducibility / portability of containers isn't worth the marginal simplicity. + +--- + +## 0003 — Monorepo across services (2026-06-07) + +**Status:** Accepted +**Context:** Backend, worker, admin-web, client, and CLI all evolve together. Sharing types/contracts is easier when they share a repo. +**Decision:** Single git repo with one top-level folder per deployable. Backend and worker share a Python package (`backend/src/mist/`) and run as different entrypoints of the same Docker image. +**Consequences:** One CI workflow per artifact, but a single source of truth for the system. Refactors that cross boundaries are atomic. +**Alternatives considered:** Polyrepo. Rejected — friend-scale doesn't justify the coordination overhead. + +--- + +## 0004 — Modular monolith for backend (api + worker, same code) (2026-06-07) + +**Status:** Accepted +**Context:** Original plan split the backend into multiple services (identity, catalog, builds, downloads, client-bff, notifications). At ~10 users this is overkill. +**Decision:** Single FastAPI app with internal modules per domain (`api/auth.py`, `api/catalog.py`, etc.). Celery worker shares the same Python package and Docker image; only the entrypoint differs. +**Consequences:** Refactoring boundaries is a code-level concern, not an ops concern. If a domain genuinely outgrows the monolith later, extract it then. +**Alternatives considered:** True microservices. Rejected per ADR 0002. + +--- + +## 0005 — Linear versions only, no branches (2026-06-07) + +**Status:** Accepted +**Context:** Steam supports branches (stable / beta / internal). Useful for a real game publisher; overkill here. +**Decision:** Versions form a linear ordered list per game. No branches in MVP. +**Consequences:** Catalog data model is simpler (just `ordinal` on `Version`). Direct/indirect update routing logic is unchanged from prototype. If we ever want betas, we add a `branch` column and migrate. +**Alternatives considered:** Steam-style branches. Rejected for MVP — no current need. + +--- + +## 0006 — Public-by-default catalog with an `is_private` flag (2026-06-07) + +**Status:** Accepted +**Context:** Real entitlements ("Tim owns Game X, Tom doesn't") add an entitlements service. Friend-scale doesn't justify it. +**Decision:** Single boolean `is_private` on `Game`. Public games are visible to anyone logged in. Private games are admin-only (future: explicit grants). +**Consequences:** No entitlements service. If we want per-user grants later, add a `game_user_grants` table without breaking anything. +**Alternatives considered:** Full Steam-style ownership. Rejected as premature. + +--- + +## 0007 — Tauri client (Rust shell + Svelte UI) (2026-06-07) + +**Status:** Accepted +**Context:** Original prototype client was PyQt5. Tauri is smaller, modern, builds tiny installers, and lets the UI be written in web tech. +**Decision:** Client is a Tauri 2 app with Svelte UI inside. +**Consequences:** Need to either port patch-application logic to Rust or ship a Python sidecar the Tauri shell shells out to. UI is web tech (good ecosystem). Installer is small. +**Alternatives considered:** Keep PyQt5 (familiar but dated), Electron (huge install), web-only (loses native install/launch). + +--- + +## 0008 — Username + password auth, admin-provisioned (2026-06-07) + +**Status:** Accepted +**Context:** Options were Discord OAuth (natural fit since friends use Discord), self-hosted SSO (Authentik/Keycloak), or username/password. +**Decision:** Username + password, argon2id hashing, admin provisions accounts manually via admin portal. +**Consequences:** Simplest. No OAuth integration. No self-serve signup. Adding Discord OAuth later is straightforward. +**Alternatives considered:** Discord OAuth (declined — adds dependency on Discord availability), magic-link email (needs SMTP). + +--- + +## 0009 — Border-server reverse proxy + Tailscale backhaul, not cluster-side ingress (2026-06-07) + +**Status:** Accepted +**Context:** Friends shouldn't need to install Tailscale to use the service. Backend VM shouldn't be on the public internet. +**Decision:** Public DNS → border server (public IP) → Nginx Proxy Manager terminates TLS via Let's Encrypt → Tailscale backhaul → VM. VM has no public exposure. +**Consequences:** Friends use a regular browser/client over HTTPS. TLS lives at the border, not in the cluster. Cluster-side cert-manager not needed. +**Alternatives considered:** Tailscale on every friend's machine (rejected — bad UX), public IP on the VM (rejected — security surface). diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md new file mode 100644 index 0000000..f7812d0 --- /dev/null +++ b/docs/RUNBOOK.md @@ -0,0 +1,151 @@ +# Mist — Operational Runbook + +A short, dense reference for "what do I do when X happens." Fill in as we hit real situations. + +## Backend VM access + +SSH: `ssh mist@<vm-tailnet-name>` (or `<vm-tailnet-ip>`). +Compose lives at: `/opt/mist/` (TODO: confirm during deploy). +All `docker compose` commands run from that directory. + +## Normal operations + +### Deploy a new image + +CI pushes images to GHCR on merge to `main`. To pull and restart: + +```sh +cd /opt/mist +docker compose pull +docker compose up -d +docker compose ps +``` + +### View logs + +```sh +docker compose logs -f api +docker compose logs -f worker +docker compose logs --tail=200 api worker +``` + +### Restart the stack + +```sh +cd /opt/mist +docker compose down +docker compose up -d +``` + +### Restart a single service + +```sh +docker compose restart worker +``` + +### Run a Celery task manually (debugging) + +```sh +docker compose exec api python -c "from mist.worker.tasks import generate_direct_update; generate_direct_update.delay('Satisfactory', '1.0.0.0', '1.0.0.1')" +``` + +## Failure scenarios + +### NAS is unreachable + +**Symptoms:** worker tasks fail with `FileNotFoundError` for `/mnt/nas/...`, API `/downloads/*` returns 404 for non-cached files. + +**Action:** +1. Verify NAS reachability from the VM: `ls /mnt/nas/mist/games/` +2. If empty/error, NFS mount is broken. Check mount: `mount | grep nas` +3. Remount: `sudo mount -a` (assuming `/etc/fstab` has the entry) +4. If still broken, log into NAS, verify it's serving NFS +5. Stack will recover automatically once NAS is back; in-flight jobs will retry per Celery config + +### Postgres won't start + +**Symptoms:** `api` container restarts in a loop, logs show `connection refused` to `postgres`. + +**Action:** +1. `docker compose logs postgres` — look for the actual error +2. Common cause: out of disk space. `df -h` on the VM. +3. If corrupted volume: stop stack, restore from last `pg_dump` (see "Restore from backup") + +### Worker queue is backed up + +**Symptoms:** Builds take forever, RabbitMQ UI (`http://<vm>:15672/`) shows growing queue depth. + +**Action:** +1. Check worker logs for stuck tasks +2. Scale workers: edit `docker-compose.yml`, set `worker.deploy.replicas: 2`, `docker compose up -d` +3. If a specific task is hanging, purge it: `docker compose exec worker celery -A mist.worker purge` + +### Cache disk is full + +**Symptoms:** Build jobs fail with `OSError: no space left on device`. + +**Action:** +1. `df -h` to confirm +2. `docker compose exec api python -m mist.core.paths --clear-cache` (TODO: implement this maintenance task) +3. Or manually: stop stack, `rm -rf /var/lib/docker/volumes/mist_cache-vol/_data/*`, restart + +### Stack won't come back up after VM reboot + +**Symptoms:** SSH in after reboot, `docker compose ps` shows nothing or services are Exited. + +**Action:** +1. Verify Docker daemon: `systemctl status docker` +2. `cd /opt/mist && docker compose up -d` +3. If still failing, check `restart: unless-stopped` is set on all services in `docker-compose.yml` + +## Backups + +### What we back up + +- Postgres (full dump) — daily +- `Mist/.env` (passwords, secrets) — versioned outside this repo +- `docker-compose.yml` and any host-level config — in git + +### What we DON'T back up here + +- Game files on NAS — NAS has its own backup story (assumed RAID + remote replication) +- Hot cache — regenerable from NAS + +### Take a Postgres backup + +```sh +docker compose exec -T postgres pg_dump -U mist mist | zstd > /mnt/nas/mist/backups/pg-$(date +%F).sql.zst +``` + +### Restore from a Postgres backup + +```sh +docker compose stop api worker +zstd -d < /mnt/nas/mist/backups/pg-YYYY-MM-DD.sql.zst | docker compose exec -T postgres psql -U mist mist +docker compose start api worker +``` + +## Provisioning a new friend account + +(Until the admin portal supports this end-to-end.) + +```sh +docker compose exec api python -m mist.scripts.create_user <username> <password> [--admin] +``` + +(TODO: implement that script.) + +## Resetting your admin password + +```sh +docker compose exec api python -m mist.scripts.reset_password <username> <new-password> +``` + +(TODO: implement that script.) + +## Health checks (manual) + +```sh +curl -s https://api.mist.example/healthz # expect {"ok": true} +curl -s https://api.mist.example/readyz # expect 200 if DB/Redis/RabbitMQ all reachable +``` diff --git a/mistpipe/README.md b/mistpipe/README.md new file mode 100644 index 0000000..02c6e65 --- /dev/null +++ b/mistpipe/README.md @@ -0,0 +1,32 @@ +# mistpipe + +Admin CLI for [Mist](../README.md). Inspired by Steam's SteamPipe and itch.io's butler. + +`mistpipe` is what you run on your laptop to upload new games and push updates to the Mist backend. It handles authentication, local archive prep, and the HTTP upload to the `api` service. + +## Install + +```sh +pip install -e . +``` + +## Usage + +```sh +mistpipe login # interactive; stores JWT in OS keychain +mistpipe new-game --title "Satisfactory" \ + --app-id 526870 \ + --version 1.0.0.0 \ + --path ./satisfactory +mistpipe push satisfactory 1.0.0.3 \ + --path ./satisfactory \ + --notes-url https://example.com/notes +mistpipe ls +mistpipe resync-steam satisfactory +mistpipe rm satisfactory # soft delete +mistpipe logout +``` + +## Status + +Subcommands are scaffolded; bodies print "TODO" and exit cleanly. Real implementations land alongside the corresponding backend routes. diff --git a/mistpipe/pyproject.toml b/mistpipe/pyproject.toml new file mode 100644 index 0000000..0b0685f --- /dev/null +++ b/mistpipe/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "mistpipe" +version = "0.1.0" +description = "Mist admin upload CLI — SteamPipe but tiny." +readme = "README.md" +requires-python = ">=3.11" +license = { text = "Proprietary" } +authors = [{ name = "Josh" }] + +dependencies = [ + "click>=8.1", + "httpx>=0.27", + "keyring>=25.0", + "rich>=13.9", +] + +[project.scripts] +mistpipe = "mistpipe.cli:cli" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/mistpipe/src/mistpipe/__init__.py b/mistpipe/src/mistpipe/__init__.py new file mode 100644 index 0000000..5223e33 --- /dev/null +++ b/mistpipe/src/mistpipe/__init__.py @@ -0,0 +1,3 @@ +"""mistpipe — admin CLI for the Mist game distribution platform.""" + +__version__ = "0.1.0" diff --git a/mistpipe/src/mistpipe/__main__.py b/mistpipe/src/mistpipe/__main__.py new file mode 100644 index 0000000..68559de --- /dev/null +++ b/mistpipe/src/mistpipe/__main__.py @@ -0,0 +1,4 @@ +from mistpipe.cli import cli + +if __name__ == "__main__": + cli() diff --git a/mistpipe/src/mistpipe/cli.py b/mistpipe/src/mistpipe/cli.py new file mode 100644 index 0000000..96651e8 --- /dev/null +++ b/mistpipe/src/mistpipe/cli.py @@ -0,0 +1,134 @@ +"""mistpipe — click-based CLI. + +Subcommand bodies are skeletons that print TODO and exit. The HTTP wiring +in `mistpipe.client.MistClient` is real where it can be; backend routes +those calls hit are skeletons themselves (501 Not Implemented) until the +product roadmap fills them in. +""" + +from __future__ import annotations + +from pathlib import Path + +import click +from rich.console import Console +from rich.table import Table + +from mistpipe import keychain +from mistpipe.client import DEFAULT_BASE_URL, MistClient + +console = Console() + + +@click.group() +@click.option("--api-url", default=DEFAULT_BASE_URL, envvar="MIST_API_URL", show_default=True) +@click.pass_context +def cli(ctx: click.Context, api_url: str) -> None: + """mistpipe — admin CLI for Mist.""" + ctx.ensure_object(dict) + ctx.obj["api_url"] = api_url + + +@cli.command() +@click.option("--username", prompt=True) +@click.password_option(confirmation_prompt=False) +@click.pass_context +def login(ctx: click.Context, username: str, password: str) -> None: + """Authenticate and store a JWT in the OS keychain.""" + client = MistClient(ctx.obj["api_url"]) + try: + client.login(username, password) + console.print(f"[green]Signed in as[/] {username}") + except Exception as e: + console.print(f"[red]Login failed:[/] {e}") + raise SystemExit(1) + + +@cli.command() +def logout() -> None: + """Clear the stored JWT.""" + keychain.clear_token() + console.print("[green]Signed out.[/]") + + +@cli.command("new-game") +@click.option("--title", required=True, help="Display title of the game") +@click.option("--app-id", type=int, default=None, help="Steam app id (optional)") +@click.option("--version", required=True, help="Initial version string, e.g. 1.0.0.0") +@click.option( + "--path", + required=True, + type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path), + help="Folder containing the initial version's files", +) +@click.option("--private", "is_private", is_flag=True, help="Mark game as private (admin-only)") +@click.pass_context +def new_game(ctx: click.Context, title: str, app_id: int | None, version: str, path: Path, is_private: bool) -> None: + """Register a new game and upload its base version.""" + console.print(f"[yellow]TODO[/] would register {title} ({version}) from {path}, app_id={app_id}, private={is_private}") + # TODO: + # 1. POST /admin/games with title/app_id/is_private + # 2. Pack `path` into .7z (matches NAS base_version.7z convention) + # 3. POST /builds/upload with the archive + version metadata + # 4. Poll /builds/jobs/<id> for completion + + +@cli.command() +@click.argument("game") +@click.argument("version") +@click.option( + "--path", + required=True, + type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path), + help="Folder containing the new version's files", +) +@click.option("--notes-url", default=None, help="URL to the patch notes") +@click.pass_context +def push(ctx: click.Context, game: str, version: str, path: Path, notes_url: str | None) -> None: + """Upload a new version of an existing game.""" + console.print(f"[yellow]TODO[/] would push {game}@{version} from {path}, notes={notes_url}") + # TODO: + # 1. Tar+zstd `path` into temp archive + # 2. POST /builds/upload (streaming) + # 3. Poll /builds/jobs/<id> + + +@cli.command("ls") +@click.pass_context +def list_games(ctx: click.Context) -> None: + """List games in the Mist catalog.""" + client = MistClient(ctx.obj["api_url"]) + try: + games = client.list_games() + except NotImplementedError: + console.print("[yellow]TODO[/] backend not implemented") + return + except Exception as e: + console.print(f"[red]Failed:[/] {e}") + raise SystemExit(1) + + table = Table(show_header=True, header_style="bold") + table.add_column("ID") + table.add_column("Title") + table.add_column("Latest version") + table.add_column("Private") + for g in games: + table.add_row(str(g.get("id")), g.get("title", ""), g.get("latest_version", ""), str(g.get("is_private", False))) + console.print(table) + + +@cli.command("resync-steam") +@click.argument("game_id", type=int) +@click.pass_context +def resync_steam(ctx: click.Context, game_id: int) -> None: + """Re-pull Steam appdetails for a game.""" + console.print(f"[yellow]TODO[/] would resync steam metadata for game {game_id}") + + +@cli.command("rm") +@click.argument("game_id", type=int) +@click.confirmation_option(prompt="Soft-delete this game?") +@click.pass_context +def remove(ctx: click.Context, game_id: int) -> None: + """Soft-delete a game (sets deleted_at).""" + console.print(f"[yellow]TODO[/] would soft-delete game {game_id}") diff --git a/mistpipe/src/mistpipe/client.py b/mistpipe/src/mistpipe/client.py new file mode 100644 index 0000000..f49dfb0 --- /dev/null +++ b/mistpipe/src/mistpipe/client.py @@ -0,0 +1,67 @@ +"""HTTP client for the Mist backend API.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import httpx + +from mistpipe import keychain + +DEFAULT_BASE_URL = os.environ.get("MIST_API_URL", "https://api.mist.example") + + +class MistClient: + def __init__(self, base_url: str | None = None) -> None: + self.base_url = base_url or DEFAULT_BASE_URL + self._http = httpx.Client(base_url=self.base_url, timeout=httpx.Timeout(60.0, read=600.0)) + + # ---- Auth ---- + + def login(self, username: str, password: str) -> str: + r = self._http.post("/auth/login", json={"username": username, "password": password}) + r.raise_for_status() + token: str = r.json()["access_token"] + keychain.save_token(token) + return token + + def _headers(self) -> dict[str, str]: + token = keychain.load_token() + if not token: + raise RuntimeError("Not logged in. Run `mistpipe login` first.") + return {"Authorization": f"Bearer {token}"} + + # ---- Catalog ---- + + def list_games(self) -> list[dict]: + r = self._http.get("/catalog/games", headers=self._headers()) + r.raise_for_status() + return r.json() + + # ---- Admin ---- + + def create_game( + self, title: str, app_id: int | None = None, is_private: bool = False + ) -> dict: + body = {"title": title, "app_id": app_id, "is_private": is_private} + r = self._http.post("/admin/games", json=body, headers=self._headers()) + r.raise_for_status() + return r.json() + + def resync_steam(self, game_id: int) -> dict: + r = self._http.post(f"/admin/games/{game_id}/resync-steam", headers=self._headers()) + r.raise_for_status() + return r.json() + + def delete_game(self, game_id: int) -> None: + r = self._http.delete(f"/admin/games/{game_id}", headers=self._headers()) + r.raise_for_status() + + # ---- Build ingest ---- + + def upload_version( + self, archive_path: Path, game_title: str, version: str, notes_url: str | None = None + ) -> dict: + # TODO: stream the archive instead of buffering. For now, raise. + raise NotImplementedError("upload_version not implemented yet") diff --git a/mistpipe/src/mistpipe/keychain.py b/mistpipe/src/mistpipe/keychain.py new file mode 100644 index 0000000..c26572f --- /dev/null +++ b/mistpipe/src/mistpipe/keychain.py @@ -0,0 +1,29 @@ +"""Cross-platform secret storage for the mistpipe JWT. + +Uses the `keyring` library, which talks to OS-native secret stores: + - Windows: Windows Credential Manager + - macOS: Keychain + - Linux: Secret Service (gnome-keyring, KWallet, etc.) +""" + +from __future__ import annotations + +import keyring + +SERVICE = "mistpipe" +USERNAME = "default" + + +def save_token(token: str) -> None: + keyring.set_password(SERVICE, USERNAME, token) + + +def load_token() -> str | None: + return keyring.get_password(SERVICE, USERNAME) + + +def clear_token() -> None: + try: + keyring.delete_password(SERVICE, USERNAME) + except keyring.errors.PasswordDeleteError: + pass