Improve application responsiveness and background processing

.env.example

@@ -31,3 +31,6 @@ PLUTAINER_IMAGE=ghcr.io/ayymoss/plutainer:latest
 # Existing legacy registrations can still use their previous KOMODO_*,
 # SERVERS_ROOT, T6_SERVERS_ROOT, and RCON_HOST settings. Native hosts need none.
 # Fleet concurrency and campaign retention are saved in Settings, not env vars.
+
+# Optional request/report timings and event-loop delay diagnostics.
+EC_PERF_LOG=false

README.md

@@ -148,7 +148,20 @@ npm run test:agent
 npm run build
 ```
 
-Builds regenerate the downloadable agent, standalone updater and checksum manifest. Local controller
+Builds regenerate the downloadable agent, standalone updater and checksum manifest,
+and bundle the metrics/leaderboard workers under `build/reports`. `npm run dev`
+and `npm test` compile those workers automatically. After editing a report
+implementation during development, restart `npm run dev` to rebuild its worker.
+Reports use separate read-only SQLite connections and bounded revision-aware caches;
+metrics snapshots can be reused for up to 30 seconds when their inputs are unchanged.
+
+Set `EC_PERF_LOG=true` to log slow request timings, report computation/cache timings,
+and event-loop delay. Responses include `Server-Timing`, and browser navigations
+record an `ec:navigation` performance measure. `npm run benchmark:leaderboard`
+uses an isolated synthetic database; pass `-- --scenario all --baseline-ref <commit>`
+to compare full response parity and warm query timings with an earlier revision.
+
+Local controller
 metadata defaults to `./data/cherry.db`; use a separate test database when
 exercising enrollment. Production agent transport requires HTTPS.
 

agent/ec_agent.py

@@ -30,7 +30,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.3.0"
+VERSION = "1.4.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -329,10 +329,47 @@ def managed_write(root, value, content, uid=1000, gid=1000):
                 os.unlink(temp_name, dir_fd=parent)
 
 
+class ReadWriteGate:
+    """Let observations run together; global mutations wait for existing readers."""
+    def __init__(self):
+        self.condition = threading.Condition()
+        self.readers = 0
+        self.writers = 0
+
+    @contextlib.contextmanager
+    def read(self):
+        with self.condition:
+            while self.writers:
+                self.condition.wait()
+            self.readers += 1
+        try:
+            yield
+        finally:
+            with self.condition:
+                self.readers -= 1
+                self.condition.notify_all()
+
+    @contextlib.contextmanager
+    def write(self):
+        # The mutation lane is serial, so there is only one writer.
+        with self.condition:
+            self.writers += 1
+            while self.readers:
+                self.condition.wait()
+        try:
+            yield
+        finally:
+            with self.condition:
+                self.writers -= 1
+                self.condition.notify_all()
+
+
 class Journal:
-    def __init__(self, path):
+    def __init__(self, path, recover=True):
         self.db = sqlite3.connect(path)
         self.db.row_factory = sqlite3.Row
+        if not recover:
+            return
         self.db.execute("PRAGMA journal_mode=WAL")
         self.db.execute("PRAGMA synchronous=FULL")
         self.db.execute("CREATE TABLE IF NOT EXISTS operations (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)")
@@ -672,6 +709,10 @@ class Agent:
         self.docker = docker or Docker()
         self.stopping = threading.Event()
         self.current_operation = None
+        self.instance_locks = {}
+        self.read_gate = ReadWriteGate()
+        self.read_claim_lock = threading.Lock()
+        self.read_claims = set()
         self.cancelled_creations = frozenset()
         self.library_progress = {}
         self.library_transfers = {}
@@ -732,17 +773,25 @@ class Agent:
         return Path(self.config_path) == AGENT_CONFIG and (UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink())
 
     @contextlib.contextmanager
-    def activity(self):
+    def activity(self, read_only=False):
         path = self.root / "agent" / "activity.lock"
         descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+        mutation_descriptor = None
         try:
             try:
-                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                # All work excludes the updater. Mutations additionally exclude
+                # other mutation batches while read lanes may continue.
+                fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB)
+                if not read_only:
+                    mutation_descriptor = os.open(self.root / "agent" / "mutation.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+                    fcntl.flock(mutation_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
             except BlockingIOError:
                 yield False
                 return
             yield True
         finally:
+            if mutation_descriptor is not None:
+                os.close(mutation_descriptor)
             os.close(descriptor)
 
     def record_health(self):
@@ -953,7 +1002,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["instance.cancel-create"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1860,23 +1909,89 @@ class Agent:
             except (AgentError, OSError, ValueError):
                 pass
 
+    @staticmethod
+    def read_operation(operation):
+        return operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "logs.read") or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
+
+    def instance_lock(self, operation):
+        key = operation.get("instanceId")
+        return self.instance_locks.setdefault(key, threading.RLock()) if key else contextlib.nullcontext()
+
+    def read_loop(self):
+        # Each lane owns its SQLite connection and never writes executing_id,
+        # checkpoints, or current_operation used by the mutation state machine.
+        journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
+        try:
+            while not self.stopping.is_set():
+                try:
+                    if self.update_pending():
+                        self.stopping.wait(3)
+                        continue
+                    response = self.request("GET", "/api/agent/operations?lane=read&wait=20")
+                    for operation in response.get("operations", []):
+                        if self.stopping.is_set() or not self.read_operation(operation):
+                            continue
+                        with self.read_claim_lock:
+                            if operation["id"] in self.read_claims:
+                                continue
+                            self.read_claims.add(operation["id"])
+                        try:
+                            with self.activity(read_only=True) as acquired:
+                                if not acquired or self.update_pending():
+                                    continue
+                                with self.read_gate.read(), self.instance_lock(operation):
+                                    row, _ = journal.accept(operation)
+                                    if row["state"] in ("accepted", "unknown", "running"):
+                                        self.report(journal.update(operation["id"], "running"))
+                                        try:
+                                            result = self.dispatch(operation)
+                                            row = journal.update(operation["id"], "completed", result)
+                                        except Exception as err:
+                                            row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
+                                    self.report(row)
+                        finally:
+                            with self.read_claim_lock:
+                                self.read_claims.discard(operation["id"])
+                except (AgentError, OSError, ValueError, KeyError, sqlite3.Error):
+                    self.stopping.wait(3)
+        finally:
+            journal.close()
+
     def poll_operations(self):
+        if self.stopping.is_set() or self.update_pending():
+            return
+        # Check both update and mutation locks before delivery, release them
+        # during the long poll, and recheck immediately before accepting work.
+        with self.activity() as acquired:
+            if not acquired:
+                return
+        # Never hold the updater lock while waiting for delivery.
+        response = self.request("GET", "/api/agent/operations?lane=write&wait=20")
         with self.activity() as acquired:
             if not acquired or self.stopping.is_set() or self.update_pending():
                 return
             if self.config.get("pendingCredential"):
                 self.recover_credential()
-            response = self.request("GET", "/api/agent/operations")
+            completed = True
             for operation in response.get("operations", []):
                 if self.stopping.is_set():
                     break
+                if self.read_operation(operation):
+                    continue
                 self.current_operation = operation["id"]
+                # Global changes wait for observations; instance changes only
+                # block observations for their own immutable instance identity.
+                guard = self.read_gate.write() if operation["kind"] in ("host.decommission", "credentials.rotate") else contextlib.nullcontext()
                 try:
-                    accepted, _ = self.journal.accept(operation)
-                    self.report(accepted)
-                    self.report(self.execute(operation, report_running=True))
+                    with guard, self.instance_lock(operation):
+                        accepted, _ = self.journal.accept(operation)
+                        self.report(accepted)
+                        result = self.execute(operation, report_running=True)
+                        self.report(result)
+                        completed = completed and result["state"] in ("completed", "failed")
                 finally:
                     self.current_operation = None
+            return completed
 
     def run(self):
         # Obtain the configured host port range before accepting any deployment.
@@ -1895,13 +2010,19 @@ class Agent:
                 self.stopping.wait(3)
         heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
         heartbeats.start()
+        readers = [threading.Thread(target=self.read_loop, daemon=True) for _ in range(2)]
+        for reader in readers:
+            reader.start()
         while not self.stopping.is_set():
+            completed = False
             try:
-                self.poll_operations()
+                completed = self.poll_operations()
             except (AgentError, OSError, ValueError, KeyError):
                 # Logs deliberately omit URLs, tokens, payloads and raw errors.
                 print("Host controller contact unavailable; existing containers continue running", file=sys.stderr)
-            self.stopping.wait(3)
+            self.stopping.wait(0.25 if completed else 3)
+        for reader in readers:
+            reader.join(timeout=2)
         heartbeats.join(timeout=2)
 
 

agent/tests/test_update_integration.py

@@ -4,6 +4,7 @@ import json
 import os
 from pathlib import Path
 import tempfile
+import threading
 import unittest
 from unittest.mock import patch
 
@@ -43,11 +44,15 @@ class UpdateIntegrationTests(unittest.TestCase):
             with self.assertRaises(BlockingIOError):
                 fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
 
-    def test_operation_delivery_execution_and_reports_exclude_updates(self):
-        operation = {"id": "one", "kind": "host.inspect", "payload": {}}
+    def test_execution_and_reports_exclude_updates_but_long_poll_does_not(self):
+        operation = {"id": "one", "kind": "runtime.pull", "payload": {}}
         calls = []
         def request(method, path, body=None):
-            self.assert_activity_held()
+            if method != "GET":
+                self.assert_activity_held()
+            else:
+                with (self.agent.root / "agent" / "activity.lock").open("a") as stream:
+                    fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
             calls.append((method, path))
             return {"operations": [operation]} if method == "GET" else {}
         def dispatch(value):
@@ -82,7 +87,61 @@ class UpdateIntegrationTests(unittest.TestCase):
         module.UPDATE_JOURNAL.unlink()
         with patch.object(self.agent, "request", return_value={"operations": []}) as request:
             self.agent.poll_operations()
-            request.assert_called_once_with("GET", "/api/agent/operations")
+            request.assert_called_once_with("GET", "/api/agent/operations?lane=write&wait=20")
+
+    def test_read_lane_completes_while_mutation_is_busy_and_excludes_updater(self):
+        read_done = threading.Event()
+        operation = {"id": "read-one", "kind": "host.inspect", "payload": {}}
+        self.agent.request = lambda method, path, body=None: {"operations": [operation]} if method == "GET" else {}
+        def dispatch(value):
+            self.assert_activity_held()
+            read_done.set()
+            self.agent.stopping.set()
+            return {"ok": True}
+        self.agent.dispatch = dispatch
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+            reader = threading.Thread(target=self.agent.read_loop)
+            reader.start()
+            self.assertTrue(read_done.wait(2), "Read waited behind unrelated mutation activity")
+            reader.join(2)
+        self.assertFalse(reader.is_alive())
+        self.assertEqual(self.agent.journal.get("read-one")["state"], "completed")
+        self.assertIsNone(self.agent.executing_id)
+
+    def test_read_lane_never_dispatches_mutations(self):
+        def request(method, path, body=None):
+            self.agent.stopping.set()
+            return {"operations": [{"id": "bad-read", "kind": "instance.remove", "payload": {}}]}
+        self.agent.request = request
+        with patch.object(self.agent, "dispatch") as dispatch:
+            self.agent.read_loop()
+            dispatch.assert_not_called()
+
+    def test_instance_mutations_and_global_mutations_exclude_observations(self):
+        operation = {"instanceId": "one"}
+        entered = threading.Event()
+        release = threading.Event()
+        def read():
+            with self.agent.read_gate.read(), self.agent.instance_lock(operation):
+                entered.set()
+                release.wait(2)
+        with self.agent.instance_lock(operation):
+            reader = threading.Thread(target=read)
+            reader.start()
+            self.assertFalse(entered.wait(0.05))
+        self.assertTrue(entered.wait(2))
+        writer_entered = threading.Event()
+        def write():
+            with self.agent.read_gate.write():
+                writer_entered.set()
+        writer = threading.Thread(target=write)
+        writer.start()
+        self.assertFalse(writer_entered.wait(0.05))
+        release.set()
+        reader.join(2)
+        writer.join(2)
+        self.assertTrue(writer_entered.is_set())
 
     def test_failed_controller_contact_never_writes_healthy_receipt(self):
         with patch.object(self.agent, "request", side_effect=module.AgentError("offline")):

docs/native-hosts.md

@@ -443,3 +443,23 @@ check can install the release once the batch has drained. When a heartbeat
 advertises cancellation support, delivery resumes with cancellation cleanup
 first. This controller-side hold also recovers cancellations already waiting
 on older agents. Hosts without automatic updates need the current installer.
+
+## Responsive observations (agent 1.4.0)
+
+Agent 1.4.0 separates read-only inspections, file/log reads, and the exact RCON
+commands `status`, `getstatus`, and `version` from the serial mutation queue.
+Two bounded read lanes can run during runtime or library downloads. Instance
+mutations exclude reads of that instance; credential rotation and host cleanup
+wait for active reads. Both lanes retain durable operation IDs, generation checks,
+and the updater activity lock. Older agents continue using the combined queue.
+
+Agents long-poll `GET /api/agent/operations?lane=read|write&wait=20`; requests
+without those parameters retain immediate combined delivery. The controller wakes
+pending polls when work is queued and rechecks authentication after waiting.
+Long polls release the updater lock; execution reacquires it and rechecks update
+state before accepting work.
+
+Browser game-folder uploads transfer up to three files concurrently. Chunks of
+one file remain serial and checksum-verified. Publication waits for admitted file
+transfers and prevents subsequent writes; pause/resume still uses acknowledged
+per-file offsets. Host downloads retain their existing bounded concurrency.

package.json

@@ -6,7 +6,7 @@
   "type": "module",
   "scripts": {
     "dev": "vite dev",
-    "build": "vite build",
+    "build": "vite build && npm run build:reports -- --outDir build/reports",
     "preview": "vite preview",
     "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
     "test": "vitest run",
@@ -15,7 +15,11 @@
     "seed:fixture": "tsx scripts/seed-fixture.ts",
     "agent:package": "python3 agent/package_release.py",
     "prebuild": "npm run agent:package",
-    "test:agent": "python3 -m unittest discover -s agent/tests -v"
+    "test:agent": "python3 -m unittest discover -s agent/tests -v",
+    "build:reports": "vite build --config vite.reports.config.ts",
+    "predev": "npm run build:reports",
+    "pretest": "npm run build:reports",
+    "pretest:watch": "npm run build:reports"
   },
   "dependencies": {
     "@fontsource/alfa-slab-one": "^5.3.0",

scripts/benchmark-leaderboard.mjs

@@ -45,6 +45,7 @@ const percentile = (values, fraction) => [...values].sort((a, b) => a - b)[Math.
 
 function seed(handle, definition) {
     handle.exec(source('src/lib/server/db/schema.sql'));
+    handle.exec("CREATE TABLE report_revisions(topic TEXT PRIMARY KEY, revision INTEGER NOT NULL); INSERT INTO report_revisions VALUES ('leaderboard',0)");
     const match = handle.prepare(`INSERT INTO leaderboard_matches
         (id, server, game, map, map_key, map_name, started_at, ended_at, last_received_at,
             status, start_observed, start_players, highest_round, last_event, last_fingerprint)

src/hooks.server.ts

@@ -3,6 +3,8 @@ import { migrate } from '$lib/server/db/migrate.js';
 import { startWorker } from '$lib/server/pipeline/worker.js';
 import { interruptDeployments } from '$lib/server/provisioning/history.js';
 import { authenticationHandle } from '$lib/server/auth/guard.js';
+import { emit } from '$lib/server/pipeline/bus.js';
+import { startPerformanceMonitor } from '$lib/server/performance.js';
 
 let booted = false;
 function boot(): void {
@@ -11,6 +13,7 @@ function boot(): void {
     migrate();
     interruptDeployments();
     startWorker();
+    startPerformanceMonitor();
 }
 
 // SvelteKit calls handle() for every request; using the first request as the
@@ -18,5 +21,13 @@ function boot(): void {
 // out-of-band init hook.
 export const handle: Handle = async (input) => {
     boot();
-    return authenticationHandle(input);
+    const started = performance.now();
+    const response = await authenticationHandle(input);
+    const elapsed = performance.now() - started;
+    response.headers.append('Server-Timing', `app;dur=${elapsed.toFixed(1)}`);
+    if (process.env.EC_PERF_LOG === 'true' && elapsed > 250 && !input.event.route.id?.startsWith('/api/agent/')) {
+        console.info('[request]', { route: input.event.route.id, method: input.event.request.method, status: response.status, elapsedMs: Math.round(elapsed) });
+    }
+    if (input.event.locals.user && !['GET','HEAD','OPTIONS'].includes(input.event.request.method) && response.status < 400) emit({ type: 'fleet-invalidated' });
+    return response;
 };

src/lib/server/db/index.ts

@@ -1,4 +1,5 @@
 import Database from 'better-sqlite3';
+import { installReportRevisions } from '../reports/revisions.js';
 import { mkdirSync } from 'node:fs';
 import { dirname, resolve } from 'node:path';
 import { config } from '../config.js';
@@ -368,6 +369,7 @@ export function db(): Database.Database {
         backfillLeaderboard(handle);
         initializeFleetSettings(handle);
         migrateNativeInstances(handle);
+        installReportRevisions(handle);
     } catch (err) {
         handle.close();
         throw err;

src/lib/server/fleet/leaderboard.ts

@@ -10,6 +10,11 @@ const metrics = new Set<string>(leaderboardMetrics.map(metric => metric.key));
 export const MAX_LEADERBOARD_OFFSET = 1_000_000;
 type RankedEntry = Omit<LeaderboardEntry, 'records'>;
 
+interface RankedScope { revision: number; entries: RankedEntry[]; maps: LeaderboardMap[]; summary: LeaderboardResponse['summary'] }
+const scopeCaches = new WeakMap<object, Map<string, RankedScope>>();
+// Match SQLite's built-in lower(), which folds ASCII rather than Unicode.
+const searchText = (value: string) => value.replace(/[A-Z]/g, ch => ch.toLowerCase());
+
 interface LeaderboardRow extends Omit<RankedEntry, 'guid'> {
     // A left join keeps metadata available when the requested page has no players.
     guid: string | null;
@@ -91,21 +96,15 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
         SELECT measures.*, p.name, RANK() OVER (ORDER BY measures.${metric} DESC) AS rank
         FROM measures JOIN identities p ON p.guid = measures.guid AND p.recency = 1 ${qualification}
     )`;
-    if (exactGuid !== undefined) {
-        const entries = handle.prepare(`${aggregated} SELECT * FROM ranked WHERE guid = ?`)
-            .all(...parameters, exactGuid) as RankedEntry[];
-        return addRecordSources(handle, entries, options)[0] ?? null;
-    }
-
-    const query = options.q?.trim() ?? '';
-    const search = query ? 'WHERE instr(lower(name), lower(?)) > 0 OR instr(lower(guid), lower(?)) > 0' : '';
-    // Reuse one ranked set for the page and matching count. Summaries deliberately
-    // cover the full game/map scope, including players below an efficiency threshold.
-    const rows = handle.prepare(`${aggregated}, searched AS MATERIALIZED (
-        SELECT * FROM ranked ${search}
-    ), page AS (
-        SELECT * FROM searched ORDER BY ${metric} DESC, name COLLATE NOCASE, name, guid LIMIT ? OFFSET ?
-    ), playerSummary AS (
+    let revision = -1;
+    try { revision = (handle.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'").get() as { revision: number }).revision; }
+    catch { /* Standalone history tools can operate without cache metadata. */ }
+    let cache = scopeCaches.get(handle);
+    if (!cache) { cache = new Map(); scopeCaches.set(handle, cache); }
+    const cacheKey = JSON.stringify([options.game ?? '', options.map ?? '', metric]);
+    let saved = cache.get(cacheKey);
+    if (!saved || revision < 0 || saved.revision !== revision) {
+    const rows = handle.prepare(`${aggregated}, playerSummary AS (
         SELECT COUNT(*) AS players, COALESCE(SUM(kills), 0) AS kills,
             COALESCE(SUM(roundsSurvived), 0) AS roundsSurvived,
             COALESCE(MAX(highestRound), 0) AS highestRound FROM totals
@@ -113,21 +112,18 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
         SELECT COUNT(DISTINCT match_id) AS matches, MIN(matchStartedAt) AS firstTrackedAt,
             MAX(matchUpdatedAt) AS updatedAt FROM observations
     ), metadata AS (
-        SELECT (SELECT COUNT(*) FROM searched) AS total, playerSummary.*, matchSummary.*
-        FROM playerSummary CROSS JOIN matchSummary
-    ) SELECT page.*, metadata.total AS matchingTotal, metadata.players AS summaryPlayers,
+        SELECT playerSummary.*, matchSummary.* FROM playerSummary CROSS JOIN matchSummary
+    ) SELECT ranked.*, metadata.players AS summaryPlayers,
         metadata.matches AS summaryMatches, metadata.kills AS summaryKills,
         metadata.roundsSurvived AS summaryRounds, metadata.highestRound AS summaryHighestRound,
         metadata.firstTrackedAt AS summaryFirstTrackedAt, metadata.updatedAt AS summaryUpdatedAt
-        FROM metadata LEFT JOIN page ON 1
-        ORDER BY page.${metric} DESC, page.name COLLATE NOCASE, page.name, page.guid`)
-        .all(...parameters, ...(query ? [query, query] : []), limit, offset) as LeaderboardRow[];
+        FROM metadata LEFT JOIN ranked ON 1
+        ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`)
+        .all(...parameters) as LeaderboardRow[];
     const rankedEntries = rows.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
         .map(({ matchingTotal, summaryPlayers, summaryMatches, summaryKills, summaryRounds,
             summaryHighestRound, summaryFirstTrackedAt, summaryUpdatedAt, ...entry }) => entry);
-    const entries = addRecordSources(handle, rankedEntries, options);
     const metadata = rows[0];
-    const total = metadata.matchingTotal;
     const summary: LeaderboardResponse['summary'] = {
         players: metadata.summaryPlayers, matches: metadata.summaryMatches, kills: metadata.summaryKills,
         roundsSurvived: metadata.summaryRounds, highestRound: metadata.summaryHighestRound,
@@ -140,5 +136,18 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
         GROUP BY game, map_key HAVING id = MAX(id)
         ORDER BY game, name COLLATE NOCASE, map_key`)
         .all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
-    return { entries, maps, total, limit, offset, metric, summary };
+    saved = { revision, entries: rankedEntries, maps, summary };
+    cache.delete(cacheKey);
+    if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
+    // Bound total retained players as well as the number of scopes.
+    while (cache.size > 8 || [...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000) cache.delete(cache.keys().next().value!);
+    }
+    if (exactGuid !== undefined) {
+        const entry = saved.entries.find(entry => entry.guid === exactGuid);
+        return entry ? addRecordSources(handle, [entry], options)[0] : null;
+    }
+    const query = searchText(options.q?.trim() ?? '');
+    const matching = query ? saved.entries.filter(entry => searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)) : saved.entries;
+    const entries = addRecordSources(handle, matching.slice(offset, offset + limit), options);
+    return { entries, maps: saved.maps, total: matching.length, limit, offset, metric, summary: saved.summary };
 }

src/lib/server/fleet/servers.ts

@@ -179,6 +179,10 @@ export function setServerField(
     field: keyof ServerRow,
     value: string | number | null
 ): void {
+    setServerFields(name, { [field]: value });
+}
+
+export function setServerFields(name: string, fields: Partial<Record<keyof ServerRow, string | number | null>>): void {
     const allowed: Array<keyof ServerRow> = [
         'komodo_type',
         'komodo_name',
@@ -211,8 +215,12 @@ export function setServerField(
         'game_ended_at',
         'game_cycle_key'
     ];
-    if (!allowed.includes(field)) throw new Error(`refusing to set ${field}`);
-    db().prepare(`UPDATE servers SET ${field} = ? WHERE name = ?`).run(value, name);
+    const entries = Object.entries(fields);
+    if (!entries.length) return;
+    for (const [field] of entries) if (!allowed.includes(field as keyof ServerRow)) throw new Error(`refusing to set ${field}`);
+    db().prepare(`UPDATE servers SET ${entries.map(([field]) => `${field} = ?`).join(', ')} WHERE name = ?
+        AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})`)
+        .run(...entries.map(([,value]) => value), name, ...entries.map(([,value]) => value));
 }
 
 /**

src/lib/server/fleet/snapshot.ts

@@ -0,0 +1,51 @@
+import { db } from '../db/index.js';
+import { listServers } from './servers.js';
+import { observationReader } from './status.js';
+import { config, komodoConfigured, rconConfigured } from '../config.js';
+import { runningVersions } from '../integrations/plutonium.js';
+import { listHosts } from '../hosts/index.js';
+import type { NativeInstance } from '../backends/native.js';
+
+export function fleetSnapshot() {
+    const rconHost = config.rcon.host;
+    const hosts = listHosts();
+    const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host]));
+    const versions = new Map(runningVersions().map(version => [version.server, version]));
+    const instances = new Map((db().prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[]).map(instance => [instance.server, instance]));
+    const holds = new Map<string, { target: string; betaId: number }[]>();
+    for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
+        FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id JOIN beta_scripts bs ON bs.beta_id=b.id
+        JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).all() as
+        { server: string; target: string; betaId: number }[]) {
+        const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries);
+    }
+    const observe = observationReader();
+    const servers = listServers().map((s) => {
+        const instance = s.native_instance_id ? instances.get(s.name) : null;
+        return {
+            ...observe(s),
+            host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
+            network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
+            plutonium: versions.get(s.name),
+            // Derived endpoint — same logic as endpointFor() but without asking
+            // Komodo. If we don't have a port yet, endpoint is null.
+            endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
+            held: holds.get(s.name) ?? []
+        };
+    });
+    return {
+        servers,
+        serverNow: Date.now(),
+        reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
+        integrations: {
+            native: hosts.length > 0 || !komodoConfigured(),
+            hostsReady: hosts.filter(host => host.readiness === 'ready').length,
+            hostsTotal: hosts.length,
+            rcon: rconConfigured(),
+            komodo: komodoConfigured(),
+            rconTimeoutMs: config.rcon.timeoutMs,
+            defaultRconHost: rconHost
+        }
+    };
+}
+

src/lib/server/fleet/status.ts

@@ -4,21 +4,57 @@ import { isProvisioning } from './provisioning.js';
 import { restartSettling } from '../provisioning/restarts.js';
 import { type ServerRow } from './servers.js';
 import { telemetryReport, latestGameSubmission } from './status-evidence.js';
+import { db } from '../db/index.js';
+import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
+import { restartRequestInFlight } from '../provisioning/restarts.js';
+
+interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null }
+
+/** One set of evidence reads per snapshot, rather than four queries per server. */
+export function observationReader() {
+    const handle = db();
+    const operations = new Map((handle.prepare(`SELECT server, action, state FROM server_deployments
+        WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as
+        { server: string; action: string; state: string }[]).map(row => [row.server, row]));
+    const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as
+        { server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row]));
+    const submissions = new Map((handle.prepare(`SELECT server, MAX(at) AS at FROM (
+        SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
+        UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as
+        { server: string; at: string | null }[]).map(row => [row.server, row.at]));
+    const reports = new Map<string, GameTelemetry>();
+    for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
+        { server: string; received_at: string; payload: string }[]) {
+        try {
+            const saved = JSON.parse(row.payload) as GameTelemetry;
+            const event = parseGameEvent(saved);
+            if (event.server === row.server) reports.set(row.server, { ...event, receivedAt: row.received_at, timing: saved.timing });
+        } catch { /* Invalid retained telemetry is unavailable. */ }
+    }
+    return (server: ServerRow) => {
+        const operation = operations.get(server.name), restart = restarts.get(server.name);
+        return serverObservation(server, {
+            provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
+            settling: restartRequestInFlight(server.name) || !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
+            telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
+        });
+    };
+}
 
 /** Shared snapshot for REST, status events, metrics, and other server-side readers. */
-export function serverObservation(server: ServerRow) {
+export function serverObservation(server: ServerRow, evidence?: Evidence) {
     let roster = null;
     try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
     const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
         last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
-    const provisioning = isProvisioning(server.name);
-    const state = restartSettling(server.name) ? 'restarting' : server.container_state;
+    const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
+    const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
     const ready = !provisioning && (!state || state === 'running');
     const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
     return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
         status_source: null as 'telemetry' | 'rcon' | null,
-        telemetry: telemetryReport(server.name) ?? undefined,
-        telemetry_after: latestGameSubmission(server.name),
+        telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,
+        telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name),
         last_error: ready ? server.last_error : null,
         replySnippet: ready ? server.last_reply_snippet : null,
         active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null,

src/lib/server/game-files/index.ts

@@ -18,14 +18,14 @@ type CacheRow = { host_id: string; version_id: string; status: LibraryCacheState
 const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id, libraryVersionId: row.version_id, status: row.status, bytesDownloaded: row.bytes_downloaded, bytesTotal: row.bytes_total, manifestSha256: row.manifest_sha256, error: row.error, updatedAt: row.updated_at, operationId: row.operation_id,
     ...(row.bytes_verified == null ? {} : { bytesVerified: row.bytes_verified }),
     ...(row.transfer_progress ? { transfer: JSON.parse(row.transfer_progress) as LibraryTransferProgress } : {}) });
-function versionFromRow(row: VersionRow): GameLibraryVersion {
-    const files = JSON.parse(row.manifest) as GameFileEntry[];
+function versionFromRow(row: VersionRow & { file_count?: number }, summaryCaches?: Map<string, HostLibraryCache[]>): GameLibraryVersion {
+    const files = summaryCaches ? [] : JSON.parse(row.manifest) as GameFileEntry[];
     return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
         description: row.description, state: row.state, sizeBytes: row.size_bytes,
-        files, fileCount: files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
+        files, fileCount: row.file_count ?? (files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count), manifestSha256: row.manifest_sha256, createdAt: row.created_at,
         publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null,
         sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id,
-        caches: (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
+        caches: summaryCaches ? summaryCaches.get(row.id) ?? [] : (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
 }
 export function getLibraryVersion(id: string): GameLibraryVersion | null {
     const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined;
@@ -52,12 +52,19 @@ export function latestPublishedLibraries() {
         return version ? [{ ...version, name: game.name }] : [];
     });
 }
-export function listLibraries(): GameLibrary[] {
+export function listLibraries(summary = false): GameLibrary[] {
     reconcileLibraryOperations();
+    let summaryCaches: Map<string, HostLibraryCache[]> | undefined;
+    if (summary) {
+        summaryCaches = new Map();
+        for (const row of database().prepare('SELECT * FROM host_library_caches').all() as CacheRow[]) {
+            const items = summaryCaches.get(row.version_id) ?? []; items.push(cacheFromRow(row)); summaryCaches.set(row.version_id, items);
+        }
+    }
     const latest = new Map(latestPublishedLibraries().map(version => [version.game, version.id]));
     return GAME_FILE_GAMES.map(game => {
-        const versions = (database().prepare(`SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
-            WHERE l.game=? ORDER BY v.created_at DESC,v.rowid DESC`).all(game.id) as VersionRow[]).map(versionFromRow);
+        const versions = (database().prepare(`SELECT ${summary ? `v.id, v.library_id, v.version, v.description, v.dlc, v.state, v.size_bytes, v.manifest_sha256, v.created_at, v.published_at, v.storage_location, v.source_host_id, v.source_volume, v.error, v.operation_id, CASE WHEN json_array_length(v.manifest)>0 THEN json_array_length(v.manifest) ELSE (SELECT COUNT(*) FROM game_file_upload_entries e WHERE e.version_id=v.id) END AS file_count` : 'v.*'}, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
+            WHERE l.game=? ORDER BY v.created_at DESC,v.rowid DESC`).all(game.id) as VersionRow[]).map(row => versionFromRow(row, summaryCaches));
         const firstLibrary = database().prepare('SELECT created_at FROM game_file_libraries WHERE game=? ORDER BY created_at,rowid LIMIT 1').get(game.id) as { created_at: string } | undefined;
         return { id: game.id, name: game.name, game: game.id, description: '', createdAt: firstLibrary?.created_at ?? '', latestVersionId: latest.get(game.id) ?? null, versions };
     });
@@ -392,9 +399,9 @@ export function reconcileLibraryOperations(): void {
 /** Lightweight list used in launch forms; large manifests stay on the detail endpoint. */
 export function libraryOverview() {
     const hosts = listHosts();
-    return { libraries: listLibraries().map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({
+    return { libraries: listLibraries(true).map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({
         ...version, caches: hosts.map(host => {
-            const cached = getHostLibraryCache(host.id, version.id);
+            const cached = version.caches.find(cache => cache.hostId === host.id);
             if (cached) return { ...cached, status: cached.status === 'ready' && library.latestVersionId !== version.id ? 'outdated' as const : cached.status };
             return {
             hostId: host.id, libraryVersionId: version.id, status: 'missing' as const, bytesDownloaded: 0,

src/lib/server/game-files/uploads.ts

@@ -44,20 +44,50 @@ export function validateUploadPlan(input: unknown): UploadFilePlan[] {
     return manifest.map(file => ({ path: file.path, size: file.size, lastModified: modified.get(file.path)! }));
 }
 
-const uploadLocks = new Map<string, Promise<void>>();
+interface UploadLocks { barrier: Promise<void>; writers: number; files: Map<string, Promise<void>>; active: Set<Promise<void>> }
+const uploadLocks = new Map<string, UploadLocks>();
+function locksFor(key: string): UploadLocks {
+    let locks = uploadLocks.get(key);
+    if (!locks) { locks = { barrier: Promise.resolve(), writers: 0, files: new Map(), active: new Set() }; uploadLocks.set(key, locks); }
+    return locks;
+}
+function releaseLocks(key: string, locks: UploadLocks) {
+    if (!locks.writers && !locks.active.size && uploadLocks.get(key) === locks) uploadLocks.delete(key);
+}
 
 /** Serialize transfers and final publication for one import; a rejected operation never poisons the lock. */
 export async function withUploadLock<T>(root: string, callback: () => Promise<T> | T): Promise<T> {
     const key = resolve(root);
-    const previous = uploadLocks.get(key) ?? Promise.resolve();
+    const locks = locksFor(key);
+    // Capture readers admitted before publication, then prevent later readers
+    // from starting until publication finishes (including failed publication).
+    const previous = Promise.all([locks.barrier, ...locks.active]);
     let release!: () => void;
     const current = new Promise<void>(done => { release = done; });
-    uploadLocks.set(key, current);
+    locks.barrier = current;
+    locks.writers++;
     await previous;
     try { return await callback(); }
     finally {
         release();
-        if (uploadLocks.get(key) === current) uploadLocks.delete(key);
+        locks.writers--;
+        releaseLocks(key, locks);
+    }
+}
+
+/** Different files may transfer together; offsets for the same file remain serial. */
+export async function withUploadFileLock<T>(root: string, file: string, callback: () => Promise<T>): Promise<T> {
+    const key = resolve(root), locks = locksFor(key);
+    const previous = Promise.all([locks.barrier, locks.files.get(file)]);
+    let release!: () => void;
+    const current = new Promise<void>(done => { release = done; });
+    locks.files.set(file, current); locks.active.add(current);
+    await previous;
+    try { return await callback(); }
+    finally {
+        release(); locks.active.delete(current);
+        if (locks.files.get(file) === current) locks.files.delete(file);
+        releaseLocks(key, locks);
     }
 }
 
@@ -95,7 +125,7 @@ export async function receiveUploadChunk(
     }
     if (!body && chunk.size !== 0) throw new Error('A file chunk body is required.');
 
-    return withUploadLock(root, async () => {
+    return withUploadFileLock(root, file.path, async () => {
         await beforeWrite?.();
         const progress = await getUploadFileProgress(root, file);
         if (chunk.offset !== progress.receivedBytes) throw new UploadOffsetError(progress.receivedBytes);

src/lib/server/hosts/index.ts

@@ -1,4 +1,5 @@
 import { recordAgentReceipt } from '../provisioning/agent-events.js';
+import { EventEmitter } from 'node:events';
 import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
 import { isIP } from 'node:net';
 import { db } from '../db/index.js';
@@ -8,6 +9,16 @@ import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, Hos
 export const AGENT_PROTOCOL_VERSION = 1;
 export const HEARTBEAT_MAX_AGE_MS = 90_000;
 const now = () => new Date().toISOString();
+const workEvents = new EventEmitter();
+workEvents.setMaxListeners(0);
+export function waitForHostWork(hostId: string, signal: AbortSignal, timeoutMs = 20_000): Promise<void> {
+    return new Promise(resolve => {
+        const done = () => { clearTimeout(timer); workEvents.off(hostId, done); signal.removeEventListener('abort', done); resolve(); };
+        const timer = setTimeout(done, timeoutMs);
+        workEvents.on(hostId, done); signal.addEventListener('abort', done, { once: true });
+        if (signal.aborted) done();
+    });
+}
 const digest = (token: string) => createHash('sha256').update(token).digest('hex');
 function database() { const handle = db(); ensureHostSchema(handle); return handle; }
 type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
@@ -113,6 +124,7 @@ export function deleteHost(id: string): { deleted: boolean; operationId?: string
             ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
             .run(id, operationId, digest(token), timestamp, timestamp);
         handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
+        queueMicrotask(() => workEvents.emit(id));
         return { deleted: false, operationId };
     }).immediate();
 }
@@ -287,7 +299,7 @@ export function requireReadyHost(hostId: string, options: { libraryId?: string;
 type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string };
 function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
 export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; }
-export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
+export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] { return (database().prepare(`SELECT ${summary ? `id,host_id,kind,instance_id,generation,state,error,created_at,updated_at,'{}' AS payload,NULL AS result` : '*'} FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?`).all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
 /** Heartbeats can tell an executing create to stop at its next safe command boundary. */
 export function pendingCreationCancellations(hostId: string): string[] {
     return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
@@ -300,7 +312,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
     if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
     if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
     if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
-    return database().transaction(() => {
+    const operation = database().transaction(() => {
         if (options.idempotencyKey) {
             const existing = database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? AND idempotency_key=?').get(hostId, options.idempotencyKey) as OperationRow | undefined;
             if (existing) {
@@ -315,8 +327,10 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
         database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
         return getHostOperation(id)!;
     })();
+    queueMicrotask(() => workEvents.emit(hostId));
+    return operation;
 }
-export function pendingHostOperations(hostId: string): HostOperation[] {
+export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'write' = 'all'): HostOperation[] {
     const host = requireHost(hostId);
     if (host.mode === 'disabled' || !host.connected) return [];
     // Older agents hold activity.lock across the entire delivered batch, including
@@ -328,8 +342,12 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
         WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
     // Accepted/running/unknown commands keep the same immutable ID. The agent
     // consults its durable journal before acknowledging or executing a replay.
+    const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
+        OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
+    const laneFilter = lane === 'all' ? '' : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}`;
     return database().transaction(() => {
         const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
+            ${laneFilter}
             AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
             AND (kind!='instance.cancel-create' OR ?)
             AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
@@ -342,7 +360,7 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
     })();
 }
 export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
-    return database().transaction(() => {
+    const operation = database().transaction(() => {
         requireHost(hostId);
         const operation = getHostOperation(id);
         if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
@@ -367,6 +385,8 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
         }
         return getHostOperation(id)!;
     })();
+    queueMicrotask(() => workEvents.emit(`operation:${id}`));
+    return operation;
 }
 export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
     const deadline = Date.now() + timeoutMs;
@@ -375,7 +395,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise
         if (!operation) throw new Error('Host operation was not found.');
         if (['completed','failed','unknown'].includes(operation.state)) return operation;
         if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`);
-        await new Promise(resolve => setTimeout(resolve, 200));
+        await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now())));
     }
 }
 

src/lib/server/integrations/plutonium.ts

@@ -22,13 +22,18 @@ export function freshRelease(): number | null {
 export function runningVersions(): PlutoniumVersion[] {
     const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
     const target = freshRelease();
+    const byServer = new Map(versions.map(version => [version.server, version]));
+    const epochs = new Map((db().prepare(`SELECT server, MAX(at) AS at FROM (
+        SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
+        UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as { server: string; at: string | null }[]).map(row => [row.server, row.at]));
+    const recovering = new Set((db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all() as { server: string }[]).map(row => row.server));
     return listServers().map(server => {
-        const value = versions.find(v => v.server === server.name)
+        const value = byServer.get(server.name)
             ?? { server: server.name, revision: null, checked_at: null, error: null };
         const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN;
-        const epoch = latestGameSubmission(server.name);
+        const epoch = epochs.get(server.name);
         const stopped = server.desired_state === 'stopped';
-        const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !restartRecoveryPending(server.name);
+        const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !recovering.has(server.name);
         const revision = valid && !stopped ? value.revision : null;
         return { ...value, revision, stopped,
             update_status: stopped ? 'stopped' : revision === null || target === null ? 'unknown'

src/lib/server/performance.ts

@@ -0,0 +1,13 @@
+import { monitorEventLoopDelay } from 'node:perf_hooks';
+
+let started = false;
+export function startPerformanceMonitor() {
+    if (started || process.env.EC_PERF_LOG !== 'true') return;
+    started = true;
+    const delay = monitorEventLoopDelay({ resolution: 20 }); delay.enable();
+    setInterval(() => {
+        console.info('[performance]', { eventLoopP95Ms: Math.round(delay.percentile(95) / 1e6),
+            eventLoopMaxMs: Math.round(delay.max / 1e6), rssMiB: Math.round(process.memoryUsage().rss / 1048576) });
+        delay.reset();
+    }, 60_000).unref();
+}

src/lib/server/pipeline/bus.ts

@@ -46,6 +46,7 @@ export type BetaStage =
 export type BetaState = 'running' | 'ready-promote' | 'promoted' | 'rolled-back';
 
 export type PipelineEvent =
+    | { type: 'fleet-invalidated' }
     | { type: 'server-telemetry'; entries: GameTelemetry[] }
     | { type: 'deployment-opened'; deploymentId: string; action: string; servers: string[] }
     | { type: 'server-changed'; deploymentId: string; server: string; files: number }
@@ -58,7 +59,7 @@ export type PipelineEvent =
           error: string | null;
       }
     | { type: 'sweep-tick'; frozen: boolean; reason?: string }
-    | { type: 'server-status'; entries: ServerStatusEntry[] }
+    | { type: 'server-status'; entries: ServerStatusEntry[]; delta?: boolean }
     | { type: 'beta-opened'; betaId: number; scriptIds: number[]; servers: string[] }
     | {
           type: 'beta-server-stage';
@@ -78,9 +79,17 @@ class Bus extends EventEmitter {
 }
 
 const bus = new Bus();
+let invalidationPending = false;
+export function invalidateFleet(): void {
+    if (invalidationPending) return;
+    invalidationPending = true;
+    // Flush after nested database transactions have committed.
+    queueMicrotask(() => { invalidationPending = false; bus.emit('event', { type: 'fleet-invalidated' }); });
+}
 
 export function emit(event: PipelineEvent): void {
     bus.emit('event', event);
+    if (event.type === 'beta-opened' || event.type === 'beta-state' || event.type === 'beta-server-stage') invalidateFleet();
 }
 
 export function subscribe(listener: (event: PipelineEvent) => void): () => void {

src/lib/server/pipeline/worker.ts

@@ -6,10 +6,10 @@ import { emit } from './bus.js';
 import type { ServerStatusEntry } from './bus.js';
 import { rconStatusResult } from '../integrations/rcon.js';
 import type { RconFailure } from '../integrations/rcon.js';
-import { getServer, listServers, setServerField, type ServerRow } from '../fleet/servers.js';
+import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
 import { komodoInspectHealthy } from '../integrations/komodo.js';
 import { restartSettling } from '../provisioning/index.js';
-import { serverObservation } from '../fleet/status.js';
+import { serverObservation, observationReader } from '../fleet/status.js';
 import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
 import { isProvisioning } from '../fleet/provisioning.js';
 import { STATUS_REFRESH_MS } from '../../server-freshness.js';
@@ -76,6 +76,15 @@ function encodeError(reason: RconFailure, detail: string): string {
 const pendingStatus = new Map<string, Promise<number | null>>();
 const statusQueue: Array<() => Promise<void>> = [];
 let activeStatus = 0;
+const changedStatus = new Map<string, ServerStatusEntry>();
+function publishStatus(entry: ServerStatusEntry) {
+    const scheduled = changedStatus.size > 0;
+    changedStatus.set(entry.server, entry);
+    if (!scheduled) queueMicrotask(() => {
+        const entries = [...changedStatus.values()]; changedStatus.clear();
+        emit({ type: 'server-status', delta: true, entries });
+    });
+}
 
 function drainStatusQueue(): void {
     while (activeStatus < STATUS_CONCURRENCY && statusQueue.length) {
@@ -95,9 +104,9 @@ function queueStatus(server: string): Promise<number | null> {
         let capacity: number | null = null;
         try {
             capacity = await refreshServerStatus(server);
-            // Consumers expect a full snapshot. Re-read it now so neither game
-            // data nor container evidence waits for the slowest fleet member.
-            emit({ type: 'server-status', entries: listServers().map(statusEntry) });
+            // Publish this server's latest evidence without waiting for the fleet.
+            const current = getServer(server);
+            if (current) publishStatus(statusEntry(current));
         } catch (err) { console.error(`[worker] status error for ${server}:`, err); }
         finally { pendingStatus.delete(server); finish(capacity); }
     });
@@ -113,8 +122,9 @@ export async function fanoutStatus(): Promise<void> {
     try {
         // Metrics remain complete fleet collections, using freshness at capture
         // time rather than the status saved at the beginning of a long scan.
+        const observe = observationReader();
         recordMetrics(listServers().map(srv => {
-            const current = serverObservation(srv);
+            const current = observe(srv);
             return { server: srv.name, game: srv.game ?? 'unknown',
                 map: selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug)?.friendly ?? current.last_map ?? srv.map ?? 'Unknown map',
                 players: current.health === 'online' ? current.last_players : null,
@@ -127,6 +137,8 @@ async function refreshServerStatus(server: string): Promise<number | null> {
     const srv = getServer(server);
     if (!srv) return null;
     let capacity: number | null = null;
+    let updates: Partial<Record<keyof ServerRow, string | number | null>> = {};
+    const flush = () => { setServerFields(srv.name, updates); updates = {}; };
     if (!isProvisioning(srv.name)) {
         const epoch = latestGameSubmission(srv.name);
         const unchanged = () => epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
@@ -136,53 +148,56 @@ async function refreshServerStatus(server: string): Promise<number | null> {
             : await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
         // A lifecycle request may be submitted while an inspection is in flight.
         if (unchanged()) {
-            setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown');
-            setServerField(srv.name, 'container_checked_at', new Date().toISOString());
+            updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown';
+            updates.container_checked_at = new Date().toISOString();
             if (!container.ok) {
-                setServerField(srv.name, 'last_error', null);
-                setServerField(srv.name, 'last_reply_snippet', null);
+                updates.last_error = null;
+                updates.last_reply_snippet = null;
             } else if (!telemetryEvidence(srv.name)) {
+                flush();
                 const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
                     reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
                 if (unchanged()) {
                     const now = new Date().toISOString();
-                    setServerField(srv.name, 'last_checked_at', now);
-                    setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null);
+                    updates.last_checked_at = now;
+                    updates.last_reply_snippet = result.rawReply ?? null;
                     if (result.ok) {
                         capacity = result.status.capacity ?? null;
-                        setServerField(srv.name, 'last_players', result.status.players);
-                        setServerField(srv.name, 'last_map', result.status.map);
-                        setServerField(srv.name, 'last_gametype', result.status.gametype);
-                        setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
-                        setServerField(srv.name, 'last_seen_at', now);
-                        setServerField(srv.name, 'last_error', null);
-                    } else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail));
+                        updates.last_players = result.status.players;
+                        updates.last_map = result.status.map;
+                        updates.last_gametype = result.status.gametype;
+                        updates.last_roster = JSON.stringify(result.status.roster);
+                        updates.last_seen_at = now;
+                        updates.last_error = null;
+                    } else updates.last_error = encodeError(result.reason, result.detail);
                 }
             }
         }
     } else {
-        setServerField(srv.name, 'last_error', null);
-        setServerField(srv.name, 'last_reply_snippet', null);
+        updates.last_error = null;
+        updates.last_reply_snippet = null;
     }
+    flush();
     const current = getServer(srv.name) ?? srv;
     const status = serverObservation(current);
     // Occupancy timers and match clearing follow the same source as the UI.
     if (status.health === 'online' && status.last_players !== null) {
         if (status.last_players > 0) {
-            setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at);
-            setServerField(srv.name, 'inactive_since', null);
+            updates.active_since = current.active_since ?? status.last_seen_at;
+            updates.inactive_since = null;
         } else {
-            setServerField(srv.name, 'active_since', null);
-            setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at);
-            setServerField(srv.name, 'game_started_at', null);
-            setServerField(srv.name, 'game_ended_at', null);
+            updates.active_since = null;
+            updates.inactive_since = current.inactive_since ?? status.last_seen_at;
+            updates.game_started_at = null;
+            updates.game_ended_at = null;
         }
     }
+    flush();
     return capacity;
 }
 
-function statusEntry(srv: ServerRow): ServerStatusEntry {
-    const final = serverObservation(srv);
+export function statusEntry(srv: ServerRow, observe = serverObservation): ServerStatusEntry {
+    const final = observe(srv);
     return { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map,
         gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at,
         lastError: final.last_error, replySnippet: final.replySnippet,

src/lib/server/provisioning/progress.ts

@@ -1,4 +1,5 @@
 import { db } from '../db/index.js';
+import { invalidateFleet } from '../pipeline/bus.js';
 import { provisioningPhases, type ProvisioningPhase, type ProvisioningProgress, type ProvisioningCheck,
     type ProvisioningCheckpoint, type ProvisioningBlocker, type RecoveryObservation } from '../../provisioning-detail.js';
 
@@ -116,5 +117,6 @@ export function recordProgress(id: string, update: ProgressUpdate): void {
         const insert = handle.prepare(`INSERT INTO server_deployment_events (deployment_id, created_at, message, kind, phase_key, details_json)
             VALUES (?, ?, ?, ?, ?, ?)`);
         for (const event of events) insert.run(id, now, event.message, event.kind, next.phase, event.details ? JSON.stringify(event.details) : null);
+        if (events.length) invalidateFleet();
     })();
 }

src/lib/server/reports/database.ts

@@ -0,0 +1,5 @@
+import Database from 'better-sqlite3';
+import { workerData } from 'node:worker_threads';
+
+const handle = new Database(workerData.dbPath, { readonly: true, fileMustExist: true });
+export function db() { return handle; }

src/lib/server/reports/index.ts

@@ -0,0 +1,64 @@
+import { Worker } from 'node:worker_threads';
+import { resolve } from 'node:path';
+import { config } from '../config.js';
+import { db } from '../db/index.js';
+import type { MetricsQuery, MetricsData } from '../../metrics.js';
+import type { LeaderboardOptions, LeaderboardResponse, PlayerProfile, LeaderboardEntry } from '../../leaderboard.js';
+
+class ReportWorker {
+    private worker: Worker | null = null;
+    private sequence = 0;
+    private pending = new Map<number, { resolve: (value: any) => void; reject: (error: Error) => void; timer: NodeJS.Timeout }>();
+    private flights = new Map<string, Promise<any>>();
+    async close() {
+        const worker = this.worker; this.worker = null;
+        for (const pending of this.pending.values()) { clearTimeout(pending.timer); pending.reject(new Error('Report service stopped')); }
+        this.pending.clear(); this.flights.clear();
+        await worker?.terminate();
+    }
+    run<T>(kind: string, input: unknown): Promise<T> {
+        const requestInput = input as { query?: MetricsQuery; now?: number };
+        const key = JSON.stringify(kind === 'metrics' ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]);
+        const flight = this.flights.get(key);
+        if (flight) return flight;
+        if (this.pending.size >= 64) return Promise.reject(new Error('Reports are busy. Please retry shortly.'));
+        db(); // Finish migrations before opening the independent reader.
+        if (!this.worker) {
+            const worker = this.worker = new Worker(resolve(process.env.NODE_ENV === 'production' ? 'build/reports/worker.mjs' : '.svelte-kit/reports/worker.mjs'), { workerData: { dbPath: resolve(config.dbPath) } });
+            worker.on('message', ({ id, value, error, elapsedMs, cached }) => {
+                const pending = this.pending.get(id); if (!pending) return;
+                clearTimeout(pending.timer); this.pending.delete(id);
+                if (process.env.EC_PERF_LOG === 'true') console.info('[report]', { kind, elapsedMs, cached });
+                if (error) pending.reject(new Error(error)); else pending.resolve(value);
+                if (!this.pending.size) worker.unref();
+            });
+            const fail = (error: Error) => {
+                if (this.worker !== worker) return;
+                this.worker = null;
+                for (const pending of this.pending.values()) { clearTimeout(pending.timer); pending.reject(error); }
+                this.pending.clear(); void worker.terminate();
+            };
+            worker.on('error', fail);
+            worker.on('exit', () => fail(new Error('Report worker stopped. Please retry.')));
+        }
+        this.worker.ref();
+        const id = ++this.sequence;
+        const promise = new Promise<T>((resolve, reject) => {
+            const timer = setTimeout(() => {
+                // Reset the worker rather than leaving an expensive abandoned query queued.
+                const worker = this.worker; this.worker = null;
+                for (const item of this.pending.values()) { clearTimeout(item.timer); item.reject(new Error('Report timed out. Please retry.')); }
+                this.pending.clear(); void worker?.terminate();
+            }, 60_000);
+            this.pending.set(id, { resolve, reject, timer }); this.worker!.postMessage({ id, kind, input });
+        }).finally(() => { if (this.flights.get(key) === promise) this.flights.delete(key); });
+        this.flights.set(key, promise);
+        return promise;
+    }
+}
+const metrics = new ReportWorker();
+const leaderboard = new ReportWorker();
+export const metricsReport = (query: MetricsQuery) => metrics.run<MetricsData>('metrics', { query, now: Date.now() });
+export const leaderboardReport = (query: LeaderboardOptions) => leaderboard.run<LeaderboardResponse>('leaderboard', query);
+export const playerReport = (guid: string, offset: number, options: LeaderboardOptions) => leaderboard.run<{ profile: PlayerProfile | null; entry: LeaderboardEntry | null }>('player', { guid, offset, options });
+export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close()]); }

src/lib/server/reports/revisions.ts

@@ -0,0 +1,14 @@
+import type Database from 'better-sqlite3';
+
+export function installReportRevisions(handle: Database.Database) {
+    handle.exec(`CREATE TABLE IF NOT EXISTS report_revisions (topic TEXT PRIMARY KEY, revision INTEGER NOT NULL DEFAULT 0);
+        INSERT OR IGNORE INTO report_revisions(topic) VALUES ('metrics'), ('leaderboard');`);
+    for (const [topic, tables] of Object.entries({ metrics: ['metric_samples', 'servers'],
+        leaderboard: ['leaderboard_matches', 'leaderboard_match_players', 'leaderboard_players'] })) {
+        for (const table of tables) for (const action of ['INSERT', 'UPDATE', 'DELETE']) {
+            const condition = table === 'servers' && action === 'UPDATE' ? 'WHEN OLD.destroyed_at IS NOT NEW.destroyed_at OR OLD.name IS NOT NEW.name' : '';
+            handle.exec(`CREATE TRIGGER IF NOT EXISTS report_${table}_${action} AFTER ${action} ON ${table} ${condition}
+                BEGIN UPDATE report_revisions SET revision=revision+1 WHERE topic='${topic}'; END;`);
+        }
+    }
+}

src/lib/server/reports/worker.ts

@@ -0,0 +1,33 @@
+import { parentPort } from 'node:worker_threads';
+import { db } from '../db/index.js';
+import { readMetrics } from '../fleet/metrics.js';
+import { getLeaderboard, getPlayerLeaderboardEntry } from '../fleet/leaderboard.js';
+import { getPlayerProfile } from '../fleet/player-profile.js';
+
+const cache = new Map<string, { value: unknown; bytes: number }>();
+let bytes = 0;
+parentPort!.on('message', ({ id, kind, input }) => {
+    const started = performance.now();
+    try {
+        const result = db().transaction(() => {
+            const topic = kind === 'metrics' ? 'metrics' : 'leaderboard';
+            const { revision } = db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number };
+            const now = kind === 'metrics' ? input.now : Date.now();
+            const key = JSON.stringify([kind, revision, kind === 'metrics' ? Math.floor(now / 30_000) : 0, kind === 'metrics' ? input.query : input]);
+            const saved = cache.get(key);
+            if (saved) { cache.delete(key); cache.set(key, saved); return { value: saved.value, cached: true }; }
+            const value = kind === 'metrics' ? readMetrics(input.query, now)
+                : kind === 'leaderboard' ? getLeaderboard(input)
+                : { profile: getPlayerProfile(input.guid, input.offset, input.options), entry: getPlayerLeaderboardEntry(input.guid, input.options) };
+            const size = Buffer.byteLength(JSON.stringify(value));
+            if (size < 16 * 1024 * 1024) {
+                cache.set(key, { value, bytes: size }); bytes += size;
+                while (cache.size > 64 || bytes > 32 * 1024 * 1024) {
+                    const oldest = cache.keys().next().value!; bytes -= cache.get(oldest)!.bytes; cache.delete(oldest);
+                }
+            }
+            return { value, cached: false };
+        })();
+        parentPort!.postMessage({ id, ...result, elapsedMs: performance.now() - started });
+    } catch (cause) { parentPort!.postMessage({ id, error: cause instanceof Error ? cause.message : 'Report failed' }); }
+});

src/lib/ui/campaigns.ts

@@ -0,0 +1,15 @@
+/** Load only the requested history, respecting the API's per-request page bound. */
+export async function fetchCampaigns<T>(filters: Record<string, string>, count: number, signal?: AbortSignal): Promise<{ deployments: T[]; total: number }> {
+    const deployments: T[] = [];
+    let total = Infinity;
+    while (deployments.length < Math.min(count, total)) {
+        const params = new URLSearchParams({ ...filters, limit: String(Math.min(500, count - deployments.length)), offset: String(deployments.length) });
+        const response = await fetch(`/api/campaigns?${params}`, { signal });
+        if (!response.ok) throw new Error('Could not refresh campaigns.');
+        const page = await response.json() as { deployments: T[]; total: number };
+        total = page.total;
+        deployments.push(...page.deployments);
+        if (!page.deployments.length) break;
+    }
+    return { deployments, total: Number.isFinite(total) ? total : 0 };
+}

src/lib/ui/poll.ts

@@ -1,25 +1,32 @@
 /** Poll visible page content without overlapping requests. Fleet watches use their own monitor. */
-export function pollVisible(task: () => void | Promise<unknown>, interval: number, immediate = true): () => void {
+export function pollVisible(task: (signal: AbortSignal) => unknown | Promise<unknown>, interval: number, immediate = true): () => void {
     let stopped = false;
     let running = false;
     let timer: ReturnType<typeof setTimeout> | undefined;
+    let controller: AbortController | undefined;
+    let failures = 0;
     const visible = () => document.visibilityState !== 'hidden';
     const run = async () => {
         if (stopped || running || !visible()) return;
         clearTimeout(timer);
         running = true;
-        try { await task(); }
-        catch (error) { console.error('Page refresh failed', error); }
+        controller = new AbortController();
+        const timeout = setTimeout(() => controller?.abort(), 30_000);
+        try { failures = await task(controller.signal) === false ? failures + 1 : 0; }
+        catch (error) { if (!controller.signal.aborted) { failures++; console.error('Page refresh failed', error); } }
         finally {
+            clearTimeout(timeout);
+            controller = undefined;
             running = false;
-            if (!stopped && visible()) timer = setTimeout(run, interval);
+            if (!stopped && visible()) timer = setTimeout(run, interval * Math.min(8, 2 ** failures));
         }
     };
     const visibility = () => {
         clearTimeout(timer);
+        if (!visible()) controller?.abort();
         if (document.visibilityState !== 'hidden') void run();
     };
     document.addEventListener('visibilitychange', visibility);
     if (immediate) void run(); else timer = setTimeout(run, interval);
-    return () => { stopped = true; clearTimeout(timer); document.removeEventListener('visibilitychange', visibility); };
+    return () => { stopped = true; clearTimeout(timer); controller?.abort(); document.removeEventListener('visibilitychange', visibility); };
 }

src/lib/ui/stores/fleet.ts

@@ -1,7 +1,9 @@
+import { freshStatusTimestamp } from '$lib/server-freshness.js';
+import { pollVisible } from '../poll.js';
 import { selectServerObservation } from '$lib/server-observation.js';
-import { derived, writable } from 'svelte/store';
+import { derived, writable, get } from 'svelte/store';
 import { browser } from '$app/environment';
-import { statusByServer, statusReceivedAt, syncServerTime, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
+import { statusByServer, statusReceivedAt, syncServerTime, serverTimeNow, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
 import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js';
 
 export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
@@ -11,9 +13,30 @@ export const fleetError = writable('');
 export const fleetRefreshing = writable(false);
 export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
 
-export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, uiNow], ([snapshot, live, received, matches, reports, now]) =>
-    snapshot.servers.map(server => {
-        const status = received > snapshot.receivedAt ? live.get(server.name) : undefined;
+const observed = new Map<string, { server: FleetServer; status: unknown; match: unknown; report: unknown; phase: string; value: FleetServer }>();
+const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => {
+    const times = snapshot.servers.flatMap(server => {
+        const status = live.get(server.name);
+        return [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, status?.telemetry?.receivedAt,
+            server.last_seen_at, server.container_checked_at, server.last_checked_at, server.rcon_status?.last_seen_at,
+            server.telemetry?.receivedAt, reports.get(server.name)?.receivedAt];
+    });
+    return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
+});
+export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => {
+    const now = get(uiNow);
+    const next = snapshot.servers.map(server => {
+        const candidate = live.get(server.name);
+        const status = (candidate?.receivedAt ?? received) > snapshot.receivedAt ? candidate : undefined;
+        const match = matches.get(server.name);
+        const report = reports.get(server.name);
+        const times = [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at,
+            status?.telemetry?.receivedAt, server.last_seen_at, server.container_checked_at, server.last_checked_at,
+            server.rcon_status?.last_seen_at, server.telemetry?.receivedAt, report?.receivedAt];
+        const phase = times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
+        const saved = observed.get(server.name);
+        if (saved && saved.server === server && saved.status === status && saved.match === match && saved.report === report && saved.phase === phase) return saved.value;
+        const remember = (value: FleetServer) => { observed.set(server.name, { server, status, match, report, phase, value }); return value; };
         const base = status ? { ...server, last_players: status.players, last_map: status.map,
             last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
             roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
@@ -23,38 +46,65 @@ export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, ma
             telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus,
             telemetry: status.telemetry ?? server.telemetry } : server;
         const current = selectServerObservation(base, reports.get(server.name), now);
-        const match = matches.get(server.name);
         // Apply lifecycle events immediately, until a newer status observes them
         // or confirms that the lobby has emptied.
         if (!match || current.health !== 'online'
             || (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after))
             || Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
             || match.receivedAt < (current.game_started_at ?? '')
-            || match.receivedAt < (current.game_ended_at ?? '')) return current;
-        return { ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
-            game_ended_at: match.event === 'game_end' ? match.receivedAt : null };
-    }));
+            || match.receivedAt < (current.game_ended_at ?? '')) return remember(current);
+        return remember({ ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
+            game_ended_at: match.event === 'game_end' ? match.receivedAt : null });
+    });
+    const names = new Set(snapshot.servers.map(server => server.name));
+    for (const name of observed.keys()) if (!names.has(name)) observed.delete(name);
+    return next;
+});
+
+
+let snapshotServerNow = 0;
+export function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
+    if (!browser || snapshot.serverNow < snapshotServerNow) return;
+    snapshotServerNow = snapshot.serverNow;
+    syncServerTime(snapshot.serverNow);
+    fleetIntegrations.set(snapshot.integrations);
+    fleet.set({ servers: snapshot.servers, operations: snapshot.operations, receivedAt: Date.now() - Math.max(0, serverTimeNow() - snapshot.serverNow) });
+}
+function getIntegrations() { return get(fleetIntegrations); }
+
+
 
 let inFlight: Promise<void> | null = null;
-export function refreshFleet(): Promise<void> {
+let refreshRequested = false;
+export function refreshFleet(afterCurrent = false): Promise<void> {
     if (!browser) return Promise.resolve();
-    if (inFlight) return inFlight;
+    if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; }
     inFlight = (async () => {
         fleetRefreshing.set(true);
         const controller = new AbortController();
         const timeout = setTimeout(() => controller.abort(), 12_000);
         const requestedAt = Date.now();
         try {
-            const responses = await Promise.all([
-                fetch('/api/servers', { signal: controller.signal }),
-                fetch('/api/fleet', { signal: controller.signal })
+            const results = await Promise.allSettled([
+                (async () => {
+                    const response = await fetch('/api/servers', { signal: controller.signal });
+                    if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.');
+                    const servers = await response.json();
+                    if (servers.serverNow < snapshotServerNow) return;
+                    snapshotServerNow = servers.serverNow;
+                    syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations);
+                    fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt }));
+                })(),
+                (async () => {
+                    const response = await fetch('/api/fleet', { signal: controller.signal });
+                    if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.');
+                    const operations = await response.json();
+                    fleet.update(value => ({ ...value, operations: operations.operations }));
+                })()
             ]);
-            if (responses.some(response => !response.ok)) throw new Error('Could not refresh Fleet. Retrying automatically.');
-            const [servers, operations] = await Promise.all(responses.map(response => response.json()));
-            syncServerTime(servers.serverNow);
+            const failure = results.find(result => result.status === 'rejected');
+            if (failure?.status === 'rejected') throw failure.reason;
             fleetError.set('');
-            fleetIntegrations.set(servers.integrations);
-            fleet.set({ servers: servers.servers, operations: operations.operations, receivedAt: requestedAt });
         } catch (error) {
             fleetError.set(error instanceof Error && error.name !== 'AbortError'
                 ? error.message : 'Fleet refresh timed out. Retrying automatically.');
@@ -62,23 +112,25 @@ export function refreshFleet(): Promise<void> {
             clearTimeout(timeout);
             fleetRefreshing.set(false);
         }
-    })().finally(() => { inFlight = null; });
+    })().finally(() => {
+        inFlight = null;
+        if (refreshRequested) { refreshRequested = false; void refreshFleet(); }
+    });
     return inFlight;
 }
 
 let stop: (() => void) | null = null;
 export function startFleetMonitor(): () => void {
     if (!browser || stop) return () => {};
-    void refreshFleet();
-    const interval = setInterval(() => {
-        if (document.visibilityState !== 'hidden') void refreshFleet();
-    }, 10_000);
-    const onFocus = () => { if (document.visibilityState === 'visible') void refreshFleet(); };
-    document.addEventListener('visibilitychange', onFocus);
+    const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000);
+    let refreshTimer: ReturnType<typeof setTimeout> | undefined;
+    const changed = () => {
+        if (refreshTimer || document.visibilityState === 'hidden') return;
+        refreshTimer = setTimeout(() => { refreshTimer = undefined; void refreshFleet(true); }, 250);
+    };
+    window.addEventListener('ec:fleet-changed', changed);
     stop = () => {
-        clearInterval(interval);
-        document.removeEventListener('visibilitychange', onFocus);
-        stop = null;
+        stopPolling(); clearTimeout(refreshTimer); window.removeEventListener('ec:fleet-changed', changed); stop = null;
     };
     return stop;
 }

src/lib/ui/stores/pipeline.ts

@@ -26,6 +26,9 @@ function updateDeployment(id: string, mutate: (d: PipelineDeployment) => void):
 export function startPipelineStream(): (() => void) | undefined {
     if (!browser || source) return;
     source = new EventSource('/api/events/pipeline');
+    for (const event of ['server-changed', 'server-state', 'deployment-opened', 'fleet-invalidated']) {
+        source.addEventListener(event, () => window.dispatchEvent(new Event('ec:fleet-changed')));
+    }
 
     source.addEventListener('deployment-opened', (evt) => {
         const data = JSON.parse((evt as MessageEvent).data) as { deploymentId: string; action: string; servers: string[] };

src/lib/ui/stores/servers.ts

@@ -17,6 +17,7 @@ export interface RconEndpoint {
 }
 
 export interface ServerStatus {
+    receivedAt?: number;
     server: string;
     containerState?: string | null;
     containerCheckedAt?: string | null;
@@ -91,17 +92,17 @@ export function startServerStream(): (() => void) | undefined {
         });
     });
     source.addEventListener('server-status', (evt) => {
-        const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[]; serverNow?: number };
+        const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[]; serverNow?: number; delta?: boolean };
         syncServerTime(data.serverNow);
-        statusByServer.update(() => {
-            const m = new Map<string, ServerStatus>();
-            for (const e of data.entries) m.set(e.server, e);
+        statusByServer.update(previous => {
+            const m = data.delta ? new Map(previous) : new Map<string, ServerStatus>();
+            for (const e of data.entries) m.set(e.server, { ...e, receivedAt: Date.now() });
             return m;
         });
         statusReceivedAt.set(Date.now());
     });
     if (tickInterval === null) {
-        tickInterval = setInterval(() => _uiNow.set(serverTimeNow()), 1000);
+        tickInterval = setInterval(() => { if (typeof document === 'undefined' || document.visibilityState !== 'hidden') _uiNow.set(serverTimeNow()); }, 1000);
     }
     return () => { source?.close(); source = null; if (tickInterval) clearInterval(tickInterval); tickInterval = null; };
 }

src/routes/+layout.svelte

@@ -1,10 +1,10 @@
 <script lang="ts">
     import { onMount } from 'svelte';
     import { startTimeZonePreference } from '$ui/stores/time-zone.js';
-    import { afterNavigate, onNavigate, goto } from '$app/navigation';
+    import { afterNavigate, beforeNavigate, onNavigate, goto } from '$app/navigation';
     import { startPipelineStream } from '$ui/stores/pipeline.js';
     import { startServerStream } from '$ui/stores/servers.js';
-    import { startFleetMonitor } from '$ui/stores/fleet.js';
+    import { startFleetMonitor, hydrateFleet } from '$ui/stores/fleet.js';
     import { page } from '$app/stores';
     import BrandMark from '$ui/components/BrandMark.svelte';
     import Toaster from '$ui/components/Toaster.svelte';
@@ -40,6 +40,8 @@
         else helpOpen = true;
     }
 
+    beforeNavigate(() => performance.mark('ec:navigation-start'));
+    afterNavigate(() => { if (performance.getEntriesByName('ec:navigation-start').length) { performance.measure('ec:navigation', 'ec:navigation-start'); performance.clearMarks('ec:navigation-start'); } });
     afterNavigate(() => { menuOpen = false; accountOpen = false; resolveConfirmation(false); });
 
     function closeMenu(event: KeyboardEvent) {
@@ -67,8 +69,15 @@
         const stopNotifications = startNotificationHistory();
         const stopPipeline = startPipelineStream();
         const stopServers = startServerStream();
-        const stopFleet = startFleetMonitor();
-        return () => { stopNotifications(); stopFleet(); stopPipeline?.(); stopServers?.(); };
+        return () => { stopNotifications(); stopPipeline?.(); stopServers?.(); };
+    });
+
+    $effect(() => {
+        if (!sessionReady) return;
+        const path = $page.url.pathname;
+        if (!(path === '/' || /^\/(servers|scripts|configs|plugins)\//.test(path) || path === '/betas/new')) return;
+        if ('fleetSnapshot' in $page.data && $page.data.fleetSnapshot) hydrateFleet($page.data.fleetSnapshot);
+        return startFleetMonitor();
     });
 
     onMount(() => {

src/routes/+page.server.ts

@@ -0,0 +1,4 @@
+import { fleetSnapshot } from '$lib/server/fleet/snapshot.js';
+import { fleetOperations } from '$lib/server/fleet/overview.js';
+
+export function load() { return { fleetSnapshot: { ...fleetSnapshot(), operations: fleetOperations() } }; }

src/routes/+page.svelte

@@ -1,4 +1,7 @@
 <script lang="ts">
+    let { data }: { data: import('./$types').PageData } = $props();
+    const snapshot = $derived($fleet.receivedAt ? $fleet : data.fleetSnapshot);
+    const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.fleetSnapshot.servers);
     import { onMount, untrack } from 'svelte';
     import { afterNavigate, replaceState } from '$app/navigation';
     import { page } from '$app/stores';
@@ -60,7 +63,7 @@
         } catch { /* Preferences also work when storage is disabled. */ }
         readViewUrl();
         preferencesReady = true;
-        void refreshFleet();
+
     });
 
     $effect(() => {
@@ -75,25 +78,26 @@
             if (url.href !== window.location.href) replaceState(url, $page.state);
         });
     });
-    const snapshotAge = $derived.by(() => { $uiNow; return Math.max(0, Math.floor((Date.now() - $fleet.receivedAt) / 1000)); });
+    const snapshotAge = $derived.by(() => { $uiNow; return Math.max(0, Math.floor((Date.now() - ($fleet.receivedAt || data.fleetSnapshot.serverNow)) / 1000)); });
+    const integrations = $derived($fleetIntegrations ?? data.fleetSnapshot.integrations);
 
-    let servers = $derived($fleetServers.map(server => server.health === 'online' && !freshServer(server, $uiNow)
-        ? { ...server, health: 'stale' as const } : server));
-    let activeOperations = $derived($fleet.operations.filter(op => op.state !== 'done'));
+    let servers = $derived(visibleFleet);
+    const observationNow = () => untrack(() => $fleet.receivedAt ? $uiNow : data.fleetSnapshot.serverNow);
+    let activeOperations = $derived(snapshot.operations.filter(op => op.state !== 'done'));
     let attentionOperations = $derived(activeOperations.filter(op => op.state === 'attention'));
     let campaigns = $derived(new Set(activeOperations.filter(op => op.kind === 'campaign').map(op => op.group)).size);
-    let healthy = $derived(servers.filter(server => freshServer(server, $uiNow)));
+    let healthy = $derived(servers.filter(server => freshServer(server, observationNow())));
     let players = $derived(healthy.reduce((sum, server) => sum + (server.last_players ?? 0), 0));
     let empty = $derived(healthy.filter(server => server.last_players === 0).length);
     let problems = $derived(servers.filter(server => server.health === 'offline' || server.health === 'stale'));
     let starting = $derived(servers.filter(server => server.health === 'starting').length);
     let provisioning = $derived(servers.filter(server => server.health === 'provisioning').length);
-    let filtered = $derived(servers.filter(server => matchesFleet(server, query, filter, game, pinned, activeOperations, $uiNow))
+    let filtered = $derived(servers.filter(server => matchesFleet(server, query, filter, game, pinned, activeOperations, observationNow()))
         .sort((a, b) => {
             const pinOrder = Number(pinned.includes(b.name)) - Number(pinned.includes(a.name));
             if (pinOrder) return pinOrder;
-            const rank = (server: FleetServer) => sort === 'players' ? (freshServer(server, $uiNow) ? server.last_players ?? -1 : -1)
-                : sort === 'round' ? currentRound(server, $telemetryByServer.get(server.name), $uiNow) ?? -1
+            const rank = (server: FleetServer) => sort === 'players' ? (freshServer(server, observationNow()) ? server.last_players ?? -1 : -1)
+                : sort === 'round' ? currentRound(server, $telemetryByServer.get(server.name), observationNow()) ?? -1
                 : sort === 'duration' ? matchDuration(server) : 0;
             return rank(b) - rank(a) || serverTitle(a.name).localeCompare(serverTitle(b.name), undefined, { numeric: true });
         }));
@@ -109,12 +113,12 @@
         }
         return [...result.entries()].sort(([, a], [, b]) => a.title.localeCompare(b.title));
     });
-    let spotlight = $derived(servers.filter(server => currentRound(server, $telemetryByServer.get(server.name), $uiNow) !== null)
+    let spotlight = $derived(servers.filter(server => currentRound(server, $telemetryByServer.get(server.name), observationNow()) !== null)
         .sort((a, b) => matchDuration(b) - matchDuration(a))[0]);
 
     function matchDuration(server: FleetServer): number {
-        return freshServer(server, $uiNow) && server.active_since && server.game_started_at && !server.game_ended_at
-            && (server.last_players ?? 0) > 0 ? Math.max(0, $uiNow - Date.parse(server.game_started_at)) : -1;
+        return freshServer(server, observationNow()) && server.active_since && server.game_started_at && !server.game_ended_at
+            && (server.last_players ?? 0) > 0 ? Math.max(0, observationNow() - Date.parse(server.game_started_at)) : -1;
     }
     function togglePin(name: string) { pinned = pinned.includes(name) ? pinned.filter(item => item !== name) : [...pinned, name]; }
     function launch(selection: { game: ZombieGame; slug: string } | null = null) {
@@ -139,7 +143,7 @@
             <button onclick={() => void refreshFleet()}>Retry now</button></div>
     {/if}
 
-    {#if !$fleet.receivedAt}
+    {#if !$fleet.receivedAt && !data.fleetSnapshot}
         {#if !$fleetError}<div class="loading" role="status">Loading your fleet…</div>{/if}
     {:else}
         {#if servers.length}<div class="overview" aria-label="Fleet summary">
@@ -149,10 +153,10 @@
             <a class="metric-link" class:no-campaigns={!campaigns} href="/campaigns"><strong>{campaigns}</strong> active campaigns</a>
             <span class="freshness">{$fleetError ? 'Reconnecting' : 'Updated'} · {snapshotAge < 60 ? `${snapshotAge}s` : `${Math.floor(snapshotAge / 60)}m`} ago</span>
         </div>{/if}
-        {#if $fleetIntegrations?.native && !$fleetIntegrations.hostsReady}
-            <div class="setup-note"><span><strong>Prepare a game host</strong> · {$fleetIntegrations.hostsTotal ? 'Hosts are connected or enrolled but need setup before new deployments.' : 'Add a host, prepare its runtime, and download game files.'}</span><a href="/hosts">Review hosts →</a></div>
-        {:else if $page.data.user?.role === 'admin' && $fleetIntegrations && !$fleetIntegrations.native && (!$fleetIntegrations.komodo || !$fleetIntegrations.rcon)}
-            <div class="setup-note"><span><strong>Finish system setup</strong> · {!$fleetIntegrations.komodo ? 'Komodo' : ''}{!$fleetIntegrations.komodo && !$fleetIntegrations.rcon ? ' and ' : ''}{!$fleetIntegrations.rcon ? 'RCON' : ''} not configured.</span><a href="/settings">Review setup →</a></div>
+        {#if integrations?.native && !integrations.hostsReady}
+            <div class="setup-note"><span><strong>Prepare a game host</strong> · {integrations.hostsTotal ? 'Hosts are connected or enrolled but need setup before new deployments.' : 'Add a host, prepare its runtime, and download game files.'}</span><a href="/hosts">Review hosts →</a></div>
+        {:else if $page.data.user?.role === 'admin' && $fleetIntegrations && !integrations.native && (!integrations.komodo || !integrations.rcon)}
+            <div class="setup-note"><span><strong>Finish system setup</strong> · {!integrations.komodo ? 'Komodo' : ''}{!integrations.komodo && !integrations.rcon ? ' and ' : ''}{!integrations.rcon ? 'RCON' : ''} not configured.</span><a href="/settings">Review setup →</a></div>
         {/if}
 
         {#if problems.length || attentionOperations.length || starting || provisioning}

src/routes/api/agent/operations/+server.ts

@@ -1,9 +1,17 @@
 import { json } from '@sveltejs/kit';
-import { pendingHostOperations } from '$lib/server/hosts/index.js';
+import { pendingHostOperations, waitForHostWork } from '$lib/server/hosts/index.js';
 import { authenticateAgent } from '$lib/server/hosts/http.js';
 import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js';
-export function GET({ request }: { request: Request }) {
+export async function GET({ request }: { request: Request }) {
     const host = authenticateAgent(request);
     reconcileHostPreparation(host.id);
-    return json({ operations: pendingHostOperations(host.id), serverTime: new Date().toISOString() }, { headers: { 'Cache-Control': 'no-store' } });
+    const params = new URL(request.url).searchParams;
+    const lane = params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all';
+    let operations = pendingHostOperations(host.id, lane);
+    if (!operations.length && params.get('wait') === '20') {
+        await waitForHostWork(host.id, request.signal);
+        authenticateAgent(request); // Revocation still applies after waiting.
+        operations = pendingHostOperations(host.id, lane);
+    }
+    return json({ operations, serverTime: new Date().toISOString() }, { headers: { 'Cache-Control': 'no-store' } });
 }

src/routes/api/campaigns/+server.ts

@@ -1,9 +1,22 @@
 import { json } from '@sveltejs/kit';
+import { serverTitle } from '$lib/ui/server-labels.js';
 import { db } from '$lib/server/db/index.js';
 
 export async function GET({ url }: { url: URL }) {
     const activeOnly = url.searchParams.get('active') === 'true';
+    const historyOnly = url.searchParams.get('view') === 'history';
     const server = url.searchParams.get('server');
+    const query = (url.searchParams.get('q') ?? '').slice(0, 100).toLowerCase();
+    const limit = Math.min(500, Math.max(1, Math.floor(Number(url.searchParams.get('limit')) || 25)));
+    const offset = Math.min(1_000_000, Math.max(0, Math.floor(Number(url.searchParams.get('offset')) || 0)));
+    const active = `EXISTS (SELECT 1 FROM deployment_servers a WHERE a.deployment_id=d.id AND a.server_deleted_at IS NULL AND a.restart_state IN ('pending','failed'))`;
+    const names = query ? (db().prepare('SELECT DISTINCT server FROM deployment_servers').all() as { server: string }[])
+        .map(row => row.server).filter(name => `${name} ${serverTitle(name)}`.toLowerCase().includes(query)) : [];
+    const where = `(? IS NULL OR EXISTS (SELECT 1 FROM deployment_servers target WHERE target.deployment_id=d.id AND target.server=?))
+        ${activeOnly ? `AND ${active}` : historyOnly ? `AND NOT ${active}` : ''}
+        ${query ? `AND EXISTS (SELECT 1 FROM deployment_servers target WHERE target.deployment_id=d.id AND target.server IN (${names.map(() => '?').join(',') || 'NULL'}))` : ''}`;
+    const parameters = [server, server, ...names];
+    const { total } = db().prepare(`SELECT COUNT(*) AS total FROM deployments d WHERE ${where}`).get(...parameters) as { total: number };
 
     const deployments = db()
         .prepare(
@@ -15,11 +28,11 @@ export async function GET({ url }: { url: URL }) {
              LEFT JOIN scripts s ON s.id = d.script_id
              LEFT JOIN script_versions sv ON sv.id = d.script_version_id
              LEFT JOIN script_versions pv ON pv.id = d.prior_version_id
-             WHERE (? IS NULL OR EXISTS (SELECT 1 FROM deployment_servers target WHERE target.deployment_id = d.id AND target.server = ?))
-             ORDER BY d.started_at DESC
-             LIMIT 500`
+             WHERE ${where}
+             ORDER BY d.started_at DESC, d.rowid DESC
+             LIMIT ? OFFSET ?`
         )
-        .all(server, server) as Array<{
+        .all(...parameters, limit, offset) as Array<{
         id: string;
         action: string;
         started_at: string;
@@ -36,7 +49,7 @@ export async function GET({ url }: { url: URL }) {
     }>;
 
     const ids = deployments.map(deployment => deployment.id);
-    if (!ids.length) return json({ deployments: [] });
+    if (!ids.length) return json({ deployments: [], total, limit, offset });
     const placeholders = ids.map(() => '?').join(',');
     const serversByDep = db()
         .prepare(
@@ -95,5 +108,5 @@ export async function GET({ url }: { url: URL }) {
         }))
         .filter((d) => (activeOnly ? d.servers.some((s) => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed')) : true));
 
-    return json({ deployments: rows });
+    return json({ deployments: rows, total, limit, offset });
 }

src/routes/api/events/pipeline/+server.ts

@@ -21,7 +21,7 @@ export async function GET() {
                     event.type === 'server-state' ||
                     event.type === 'deployment-opened' ||
                     event.type === 'server-changed' ||
-                    event.type === 'sweep-tick'
+                    event.type === 'sweep-tick' || event.type === 'fleet-invalidated'
                 ) {
                     send(controller, event);
                 }

src/routes/api/events/servers/+server.ts

@@ -1,3 +1,6 @@
+import { observationReader } from '$lib/server/fleet/status.js';
+import { listServers } from '$lib/server/fleet/servers.js';
+import { statusEntry } from '$lib/server/pipeline/worker.js';
 import { subscribe } from '$lib/server/pipeline/bus.js';
 import { listTelemetry } from '$lib/server/fleet/telemetry.js';
 
@@ -15,7 +18,10 @@ export async function GET() {
                     controller.enqueue(encoder.encode(`event: ${event.type}\ndata: ${JSON.stringify({ ...event, serverNow: Date.now() })}\n\n`));
                 }
             });
+            const observation = observationReader();
+            const observe = (server: ReturnType<typeof listServers>[number]) => statusEntry(server, observation);
             controller.enqueue(encoder.encode(': hello\n\n'));
+            controller.enqueue(encoder.encode(`event: server-status\ndata: ${JSON.stringify({ entries: listServers().map(server => observe(server)), snapshot: true, serverNow: Date.now() })}\n\n`));
             controller.enqueue(encoder.encode(`event: server-telemetry\ndata: ${JSON.stringify({ entries: listTelemetry(), snapshot: true, serverNow: Date.now() })}\n\n`));
             heartbeat = setInterval(() => controller.enqueue(encoder.encode(': ping\n\n')), HEARTBEAT_MS);
         },

src/routes/api/hosts/[id]/+server.ts

@@ -8,7 +8,7 @@ export function GET({ request, params }: Event) {
     requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.');
     // Payloads and results can contain RCON keys or staged credentials. History
     // exposes metadata only; agent APIs remain the only command delivery path.
-    return json({ host: presentHostPreparation(host), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id).map(({ payload: _payload, result: _result, ...operation }) => operation) });
+    return json({ host: presentHostPreparation(host), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id, 100, true).map(({ payload: _payload, result: _result, ...operation }) => operation) });
 }
 export async function PATCH({ request, params }: Event) {
     requireManagementOrigin(request);

src/routes/api/leaderboard/+server.ts

@@ -1,7 +1,8 @@
+import { leaderboardReport } from '$lib/server/reports/index.js';
 import { json } from '@sveltejs/kit';
-import { getLeaderboard, isLeaderboardMetric, MAX_LEADERBOARD_OFFSET } from '$lib/server/fleet/leaderboard.js';
+import { isLeaderboardMetric, MAX_LEADERBOARD_OFFSET } from '$lib/server/fleet/leaderboard.js';
 
-export function GET({ url }: { url: URL }) {
+export async function GET({ url }: { url: URL }) {
     const params = url.searchParams;
     const game = params.get('game');
     const map = params.get('map');
@@ -20,7 +21,7 @@ export function GET({ url }: { url: URL }) {
         || !Number.isFinite(offset) || offset < 0 || offset > MAX_LEADERBOARD_OFFSET) {
         return json({ ok: false, kind: 'bad-request', error: 'Use game t5 or t6, a supported metric, map key up to 1024 characters, q up to 100 characters, limit 1–100, and offset 0–1000000.' }, { status: 400 });
     }
-    return json(getLeaderboard({ game: game ?? undefined, map: map || undefined,
+    return json(await leaderboardReport({ game: game ?? undefined, map: map || undefined,
         metric: metric !== null && isLeaderboardMetric(metric) ? metric : undefined,
         q: q ?? undefined, limit, offset }));
 }

src/routes/api/metrics/+server.ts

@@ -1,9 +1,9 @@
 import { json } from '@sveltejs/kit';
 import { parseMetricsQuery } from '$lib/metrics.js';
-import { readMetrics } from '$lib/server/fleet/metrics.js';
+import { metricsReport } from '$lib/server/reports/index.js';
 
-export function GET({ url }: { url: URL }) {
-    return json(readMetrics(parseMetricsQuery(url.searchParams)), {
+export async function GET({ url }: { url: URL }) {
+    return json(await metricsReport(parseMetricsQuery(url.searchParams)), {
         headers: { 'cache-control': 'no-store' }
     });
 }

src/routes/api/servers/+server.ts

@@ -1,48 +1,10 @@
-import { db } from '$lib/server/db/index.js';
+import { fleetSnapshot } from '$lib/server/fleet/snapshot.js';
 import { kick } from '$lib/server/pipeline/worker.js';
 import { json } from '@sveltejs/kit';
-import { listServers, heldByBeta } from '$lib/server/fleet/servers.js';
-import { serverObservation } from '$lib/server/fleet/status.js';
 import { queueCreateServer } from '$lib/server/provisioning/index.js';
-import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
 import type { GameMode, ZombieGame } from '$lib/server/fleet/map-aliases.js';
-import { runningVersions } from '$lib/server/integrations/plutonium.js';
-import { listHosts } from '$lib/server/hosts/index.js';
-import { nativeInstance } from '$lib/server/backends/native.js';
 
-export async function GET() {
-    const rconHost = config.rcon.host;
-    const hosts = listHosts();
-    const hostsById = new Map(hosts.map(host => [host.id, host]));
-    const versions = new Map(runningVersions().map(version => [version.server, version]));
-    const servers = listServers().map((s) => {
-        const instance = s.native_instance_id ? nativeInstance(s.name) : null;
-        return {
-            ...serverObservation(s),
-            host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
-            network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
-            plutonium: versions.get(s.name),
-            // Derived endpoint — same logic as endpointFor() but without asking
-            // Komodo. If we don't have a port yet, endpoint is null.
-            endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
-            held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId }))
-        };
-    });
-    return json({
-        servers,
-        serverNow: Date.now(),
-        reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
-        integrations: {
-            native: hosts.length > 0 || !komodoConfigured(),
-            hostsReady: hosts.filter(host => host.readiness === 'ready').length,
-            hostsTotal: hosts.length,
-            rcon: rconConfigured(),
-            komodo: komodoConfigured(),
-            rconTimeoutMs: config.rcon.timeoutMs,
-            defaultRconHost: rconHost
-        }
-    });
-}
+export function GET() { return json(fleetSnapshot()); }
 
 export async function POST({ request }: { request: Request }) {
     const body = (await request.json().catch(() => ({}))) as {

src/routes/betas/new/+page.svelte

@@ -176,7 +176,7 @@
 <div class="page">
     <SectionHeader title="New beta" subtitle="Choose exact file versions and test servers, then review before starting.">{#snippet actions()}<Button href="/betas" variant="ghost">Back to betas</Button>{/snippet}</SectionHeader>
     {#if flash}<div class="ui-error" role="alert">{flash}{#if !scripts.length}<Button size="sm" onclick={()=>initialize()}>Retry</Button>{/if}</div>{/if}
-    {#if $fleetError}<div class="ui-error" role="alert">{$fleetError}<Button size="sm" onclick={refreshFleet}>Retry servers</Button></div>{/if}
+    {#if $fleetError}<div class="ui-error" role="alert">{$fleetError}<Button size="sm" onclick={() => refreshFleet(true)}>Retry servers</Button></div>{/if}
     {#if notice}<p role="status" class="notice">{notice}</p>{/if}
     {#if loading}<EmptyState title="Loading beta setup…"/>
     {:else if availableDraft}<section class="ui-panel draft"><h2>Resume unfinished beta?</h2><p>Saved {$timeFormat.dateTime(availableDraft.savedAt)} · {availableDraft.picks.filter(p=>p.scriptId!==null).length} files · {availableDraft.servers.length} selected servers. Versions and eligibility will be rechecked.</p><div class="actions"><Button variant="primary" onclick={resumeDraft}>Resume draft</Button><Button onclick={discardDraft}>{hasLinkedSetup ? 'Discard draft and use linked files' : 'Discard draft'}</Button></div></section>

src/routes/campaigns/+page.svelte

@@ -4,6 +4,7 @@
     import { page } from '$app/stores';
     import { goto } from '$app/navigation';
     import { pollVisible } from '$ui/poll.js';
+    import { fetchCampaigns } from '$ui/campaigns.js';
     import { confirmAction } from '$ui/confirm.js';
     import { serverTitle } from '$ui/server-labels.js';
     import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
@@ -48,6 +49,8 @@
     }
     let loadError = $state('');
     let limit = $state(25);
+    let historyTotal = $state(0);
+    let loadSequence = 0;
     let deployments = $state<Deployment[]>([]);
     let loading = $state(true);
     let rollingBack = $state<string | null>(null);
@@ -67,15 +70,25 @@
         void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
     }
 
-    async function loadCampaigns(): Promise<void> {
+    async function loadCampaigns(signal?: AbortSignal): Promise<boolean> {
+        const sequence = ++loadSequence;
         try {
-            const res = await fetch('/api/campaigns');
-            if (!res.ok) throw new Error('Could not refresh campaigns.');
-            deployments = (await res.json()).deployments; loadError = '';
-        } catch (error) { loadError = (error as Error).message; }
-        finally { loading = false; }
+            const [history, active] = await Promise.all([
+                fetchCampaigns<Deployment>({ q: serverFilter, view: 'history' }, limit, signal),
+                fetchCampaigns<Deployment>({ q: serverFilter, active: 'true' }, Infinity, signal)
+            ]);
+            if (sequence !== loadSequence || signal?.aborted) return true;
+            deployments = [...active.deployments, ...history.deployments]; historyTotal = history.total; loadError = '';
+            return true;
+        } catch (error) {
+            if (sequence === loadSequence && !signal?.aborted) loadError = (error as Error).message;
+            return false;
+        } finally { if (sequence === loadSequence && !signal?.aborted) loading = false; }
     }
-    onMount(() => pollVisible(loadCampaigns, 5000));
+    $effect(() => {
+        void limit; void serverFilter;
+        return pollVisible(loadCampaigns, 15000);
+    });
 
     function isActive(d: Deployment): boolean {
         return d.servers.some((s) => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed'));
@@ -145,7 +158,7 @@
         subtitle="Track file campaigns, server activation, and rollbacks."
     />
 
-    {#if loadError}<div class="ui-error" role="alert">{loadError}<Button size="sm" onclick={loadCampaigns}>Retry</Button></div>{/if}
+    {#if loadError}<div class="ui-error" role="alert">{loadError}<Button size="sm" onclick={() => loadCampaigns()}>Retry</Button></div>{/if}
     {#if loading}
         <div class="loading">Loading…</div>
     {:else}
@@ -195,7 +208,7 @@
                 {/each}
             {/if}
         </section>
-        <ListFooter total={history.length} bind:limit onchange={() => saveContext()} label="campaigns" />
+        <ListFooter total={historyTotal} bind:limit onchange={() => saveContext()} label="campaigns" />
     {/if}
 </div>
 

src/routes/game-files/+page.svelte

@@ -26,7 +26,9 @@
     let phase = $state<Phase>('idle'); let currentPath = $state(''); let pauseRequested = $state(false);
     let acceptedBytes = $state(0); let sendingBytes = $state(0); let completedFiles = $state(0);
     let folderInput = $state<HTMLInputElement>();
-    let activeRequest: XMLHttpRequest | null = null; let disposed = false;
+    const activeRequests = new Map<XMLHttpRequest, number>(); let disposed = false; let failedTransfer = false;
+    const updateSending = () => { sendingBytes = [...activeRequests.values()].reduce((a, b) => a + b, 0); };
+    const abortUploads = () => { for (const request of activeRequests.keys()) request.abort(); };
     const canManage = $derived($page.data.user?.role === 'admin');
     const onlineHosts = $derived(hosts.filter(host => host.connected && host.credentialActive && host.mode !== 'disabled'));
     const totalBytes = $derived(folder?.size ?? uploadStatus?.files.reduce((total, file) => total + file.size, 0) ?? 0);
@@ -122,21 +124,21 @@
         finally { uploadBusy = false; }
         await tick(); document.getElementById('game-upload')?.scrollIntoView({ behavior: 'smooth' });
     }
-    function pauseUpload() { pauseRequested = true; activeRequest?.abort(); }
+    function pauseUpload() { pauseRequested = true; abortUploads(); }
     async function sendChunk(path: string, file: File, offset: number, length: number): Promise<number> {
         const chunk = file.slice(offset, offset + length);
         const hash = await crypto.subtle.digest('SHA-256', await chunk.arrayBuffer());
-        if (pauseRequested || disposed) throw new Error('Upload paused.');
+        if (pauseRequested || disposed || failedTransfer) throw new Error('Upload paused.');
         const sha256 = [...new Uint8Array(hash)].map(value => value.toString(16).padStart(2, '0')).join('');
         return new Promise((resolve, reject) => {
-            const request = new XMLHttpRequest(); activeRequest = request;
+            const request = new XMLHttpRequest(); activeRequests.set(request, 0);
             request.open('PUT', `/api/game-files/${uploadId}/upload/files/${path.split('/').map(encodeURIComponent).join('/')}`);
             request.setRequestHeader('Content-Type', 'application/octet-stream');
             request.setRequestHeader('X-Upload-Offset', String(offset)); request.setRequestHeader('X-Upload-Length', String(length));
             request.setRequestHeader('X-Content-SHA256', sha256); request.timeout = 300_000;
-            request.upload.onprogress = event => { sendingBytes = Math.min(length, event.loaded); };
+            request.upload.onprogress = event => { activeRequests.set(request, Math.min(length, event.loaded)); updateSending(); };
             request.onload = () => {
-                activeRequest = null;
+                activeRequests.delete(request); updateSending();
                 try {
                     const body = JSON.parse(request.responseText);
                     if (request.status < 200 || request.status >= 300) throw new Error(body.error ?? `Upload failed (${request.status}).`);
@@ -144,16 +146,16 @@
                     resolve(body.receivedBytes);
                 } catch (cause) { reject(cause); }
             };
-            request.onerror = () => { activeRequest = null; reject(new Error('The connection was interrupted. Resume to continue from the last saved chunk.')); };
-            request.ontimeout = () => { activeRequest = null; reject(new Error('This chunk timed out. Resume to check saved progress and retry.')); };
-            request.onabort = () => { activeRequest = null; reject(new Error('Upload paused.')); };
+            request.onerror = () => { activeRequests.delete(request); updateSending(); reject(new Error('The connection was interrupted. Resume to continue from the last saved chunk.')); };
+            request.ontimeout = () => { activeRequests.delete(request); updateSending(); reject(new Error('This chunk timed out. Resume to check saved progress and retry.')); };
+            request.onabort = () => { activeRequests.delete(request); updateSending(); reject(new Error('Upload paused.')); };
             request.send(chunk);
         });
     }
     async function startUpload(event: SubmitEvent) {
         event.preventDefault(); if (uploadBusy || !folder) return;
         if (!globalThis.crypto?.subtle) { uploadError = 'Folder uploads require a secure connection. Open this controller over HTTPS or localhost.'; return; }
-        uploadBusy = true; pauseRequested = false; uploadError = ''; notice = ''; phase = 'preparing';
+        uploadBusy = true; pauseRequested = false; failedTransfer = false; uploadError = ''; notice = ''; phase = 'preparing';
         try {
             if (!uploadId) {
                 const result = await responseBody(await fetch('/api/game-files', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ game, files: folder.files.map(({ path, file }) => ({ path, size: file.size, lastModified: file.lastModified })) }) }));
@@ -165,19 +167,28 @@
             const localFiles = new Map(folder.files.map(item => [item.path, item.file]));
             const chunkSize = Math.min(8_388_608, Math.max(1, status.chunkSize));
             phase = 'uploading';
-            for (const remote of status.files) {
-                if (pauseRequested || disposed) break;
-                if (remote.complete) continue;
-                currentPath = remote.path;
-                const file = localFiles.get(remote.path)!;
-                do {
-                    if (pauseRequested || disposed) break;
-                    const length = Math.min(chunkSize, remote.size - remote.receivedBytes);
-                    remote.receivedBytes = await sendChunk(remote.path, file, remote.receivedBytes, length);
-                    acceptedBytes += length; sendingBytes = 0;
-                    if (remote.receivedBytes === remote.size) { remote.complete = true; completedFiles++; }
-                } while (!remote.complete);
-            }
+            let nextFile = 0;
+            let failure: unknown;
+            const transfer = async () => {
+                while (!pauseRequested && !disposed && !failure) {
+                    const remote = status.files[nextFile++];
+                    if (!remote) return;
+                    if (remote.complete) continue;
+                    currentPath = remote.path;
+                    const file = localFiles.get(remote.path)!;
+                    try {
+                        do {
+                            if (pauseRequested || disposed || failure) return;
+                            const length = Math.min(chunkSize, remote.size - remote.receivedBytes);
+                            remote.receivedBytes = await sendChunk(remote.path, file, remote.receivedBytes, length);
+                            acceptedBytes += length;
+                            if (remote.receivedBytes === remote.size) { remote.complete = true; completedFiles++; }
+                        } while (!remote.complete);
+                    } catch (cause) { failure ??= cause; failedTransfer = true; abortUploads(); }
+                }
+            };
+            await Promise.all(Array.from({ length: Math.min(3, status.files.length) }, transfer));
+            if (failure) throw failure;
             if (pauseRequested || disposed) { phase = 'paused'; return; }
             phase = 'verifying'; currentPath = '';
             const verification = await responseBody(await fetch(`/api/game-files/${uploadId}/upload/complete`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })) as UploadStatus;
@@ -192,7 +203,7 @@
     });
     onMount(() => {
         const stop = pollVisible(load, 5000);
-        return () => { disposed = true; pauseRequested = true; activeRequest?.abort(); stop(); };
+        return () => { disposed = true; pauseRequested = true; abortUploads(); stop(); };
     });
 </script>
 

src/routes/hosts/+page.server.ts

@@ -0,0 +1,4 @@
+import { listHosts } from '$lib/server/hosts/index.js';
+import { presentHostPreparation } from '$lib/server/hosts/preparation.js';
+
+export function load() { return { hosts: listHosts().map(presentHostPreparation) }; }

src/routes/hosts/+page.svelte

@@ -1,22 +1,24 @@
 <script lang="ts">
-    import { onMount } from 'svelte';
+    import { onMount, untrack } from 'svelte';
+    import { pollVisible } from '$ui/poll.js';
+    let { data }: { data: import('./$types').PageData } = $props();
     import { page } from '$app/stores';
     import type { Host } from '$lib/hosts.js';
     import { hostAgentUpdateLabel, hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
     import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
-    let hosts = $state<Host[]>([]);
-    let loading = $state(true); let busy = $state(false); let error = $state('');
+    let hosts = $state<Host[]>(untrack(() => data.hosts));
+    let loading = $state(false); let busy = $state(false); let error = $state('');
     let notice = $state(''); let deleteOpen = $state(false); let deletingHost = $state<Host | null>(null);
     let loadSequence = 0;
     let name = $state(''); let portMin = $state(28960); let portMax = $state(28980);
     let enrollment = $state<{ command: string; expiresAt: string; hostId: string } | null>(null);
     const canManage = $derived($page.data.user?.role === 'admin');
-    async function load() {
+    async function load(signal?: AbortSignal) {
         const sequence = ++loadSequence;
-        try { const response = await fetch('/api/hosts'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); if (sequence === loadSequence) { hosts = body.hosts; error = ''; } }
-        catch (cause) { if (sequence === loadSequence) error = (cause as Error).message; } finally { if (sequence === loadSequence) loading = false; }
+        try { const response = await fetch('/api/hosts', { signal: signal ?? AbortSignal.timeout(15000) }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); if (sequence === loadSequence) { hosts = body.hosts; error = ''; } }
+        catch (cause) { if (!signal?.aborted && sequence === loadSequence) error = (cause as Error).message; return false; } finally { if (sequence === loadSequence) loading = false; }
     }
     async function deleted(result: { deleted: boolean }) {
         notice = result.deleted ? `${deletingHost?.name ?? 'Host'} deleted.` : `Cleanup requested for ${deletingHost?.name ?? 'host'}. It remains listed until its files and agent are removed.`;
@@ -32,7 +34,7 @@
             enrollment = body.enrollment; name = ''; await load();
         } catch (cause) { error = (cause as Error).message; } finally { busy = false; }
     }
-    onMount(() => { void load(); const timer = setInterval(() => { void load(); }, 15000); return () => clearInterval(timer); });
+    onMount(() => pollVisible(load, 30000, false));
 </script>
 
 <svelte:head><title>Hosts · Electric Cherry</title></svelte:head>

src/routes/hosts/[id]/+page.svelte

@@ -1,5 +1,7 @@
 <script lang="ts">
-    import { onMount } from 'svelte';
+    import { untrack } from 'svelte';
+    import { pollVisible } from '$ui/poll.js';
+    let requestSequence = 0;
     import { page } from '$app/stores';
     import { goto } from '$app/navigation';
     import type { Host, HostOperation, HostMode } from '$lib/hosts.js';
@@ -17,11 +19,12 @@
     const canManage = $derived($page.data.user?.role === 'admin');
     const cleaning = $derived(!!host?.decommission);
     function bytes(value: unknown) { return typeof value === 'number' ? `${(value / 1073741824).toFixed(1)} GiB` : 'Unknown'; }
-    async function load(initialize = false) {
+    async function load(initialize = false, signal?: AbortSignal) {
+        const target = endpoint; const sequence = ++requestSequence;
         try {
-            const response = await fetch(endpoint); const body = await response.json(); if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
+            const response = await fetch(target, { signal: signal ?? AbortSignal.timeout(15000) }); const body = await response.json(); if (signal?.aborted || target !== endpoint || sequence !== requestSequence) return; if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations; error = '';
             if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; runtimeImage = body.defaultRuntimeImage ?? runtimeImage; }
-        } catch (cause) { error = (cause as Error).message; } finally { loading = false; }
+        } catch (cause) { if (!signal?.aborted && sequence === requestSequence) error = (cause as Error).message; return false; } finally { if (sequence === requestSequence) loading = false; }
     }
     async function action(path: string, method: string, body?: unknown, message?: string) {
         busy = true; error = ''; notice = '';
@@ -30,7 +33,13 @@
     }
     async function save(event: SubmitEvent) { event.preventDefault(); await action('', 'PATCH', { name, portMin, portMax, bindAddress, publicAddress, minimumFreeBytes: Math.round(minimumFreeGB * 1073741824) }, 'Host settings saved.'); }
     async function mode(value: HostMode) { await action('', 'PATCH', { mode: value }, value === 'draining' ? 'Drain enabled. Existing games continue until safe retirement.' : `Host ${value === 'active' ? 'enabled' : 'disabled'}.`); }
-    onMount(() => { void load(true); const timer = setInterval(() => { void load(); }, 10000); return () => clearInterval(timer); });
+    $effect(() => {
+        void endpoint;
+        return untrack(() => {
+            let initialize = true;
+            return pollVisible(async signal => { const result = await load(initialize, signal); if (result !== false) initialize = false; return result; }, 15000);
+        });
+    });
 </script>
 
 <svelte:head><title>{host?.name ?? 'Host'} · Electric Cherry</title></svelte:head>

src/routes/leaderboard/+page.server.ts

@@ -0,0 +1,10 @@
+import { GET } from '../api/leaderboard/+server.js';
+import { error } from '@sveltejs/kit';
+import type { LeaderboardResponse } from '$lib/leaderboard.js';
+
+export async function load({ url }: { url: URL }) {
+    const response = await GET({ url });
+    const body = await response.json();
+    if (!response.ok) error(response.status, body.error);
+    return { leaderboard: body as LeaderboardResponse };
+}

src/routes/leaderboard/+page.svelte

@@ -1,8 +1,10 @@
 <script lang="ts">
+    let { data: initial }: { data: import('./$types').PageData } = $props();
     import { timeFormat } from '$ui/stores/time-zone.js';
     import { onMount, untrack } from 'svelte';
     import { goto } from '$app/navigation';
-    import { page } from '$app/stores';
+    import { page, navigating } from '$app/stores';
+    const changingFilters = $derived($navigating?.to?.route.id === '/leaderboard');
     import { leaderboardMetrics, type LeaderboardMetric, type PlayerStatMetric, type LeaderboardEntry, type LeaderboardMap, type LeaderboardResponse } from '$lib/leaderboard.js';
     import { playerContextHref } from '$lib/leaderboard-navigation.js';
     import { efficiencyMetrics, isEfficiencyMetric, efficiencyRequirement, efficiencySample, formatEfficiencyValue } from '$lib/leaderboard-efficiency.js';
@@ -33,9 +35,9 @@
     const formatValue = (entry: LeaderboardEntry, key: LeaderboardMetric) => isEfficiencyMetric(key)
         ? formatEfficiencyValue(entry[key], key) : number.format(entry[key]);
 
-    let data = $state<LeaderboardResponse | null>(null);
-    let maps = $state<LeaderboardMap[]>([]);
-    let loading = $state(true);
+    let data = $state<LeaderboardResponse | null>(untrack(() => initial.leaderboard));
+    let maps = $state<LeaderboardMap[]>(untrack(() => initial.leaderboard.maps));
+    let loading = $state(false);
     let error = $state('');
     let search = $state('');
     let mounted = $state(false);
@@ -68,8 +70,8 @@
 
     $effect(() => { search = query; });
     $effect(() => {
-        const url = apiUrl;
-        if (mounted) untrack(() => { void refresh(url); });
+        const report = initial.leaderboard;
+        untrack(() => { ++requestId; controller?.abort(); data = report; maps = report.maps; loadedUrl = apiUrl; loading = false; error = ''; });
     });
 
     onMount(() => {
@@ -91,7 +93,7 @@
         controller?.abort();
         const activeController = new AbortController();
         controller = activeController;
-        if (url !== loadedUrl) { data = null; checkedAt = null; }
+        if (url !== loadedUrl) checkedAt = null;
         loading = true;
         error = '';
         const timeout = setTimeout(() => activeController.abort(), 20_000);
@@ -191,10 +193,10 @@
         </div>
         </details>
 
-        <section class="rankings" class:compact={!fullTable} aria-labelledby="rankings-title" aria-busy={loading}>
+        <section class="rankings" class:compact={!fullTable} aria-labelledby="rankings-title" aria-busy={loading || changingFilters}>
             <div class="rankings-header">
                 <div><h2 id="rankings-title">{metricLabel}</h2><p>{description(metric)}</p>{#if isEfficiencyMetric(metric)}<p>Qualification is measured within the selected game and map. Fleet totals include all recorded players.</p>{/if}</div>
-                <div class="refresh-status" role="status">{loading ? 'Refreshing…' : checkedAt ? `Checked ${$timeFormat.time(checkedAt)}` : ''}<span>Refreshes every 15 seconds</span></div>
+                <div class="refresh-status" role="status">{changingFilters ? 'Updating filters…' : loading ? 'Refreshing…' : checkedAt ? `Checked ${$timeFormat.time(checkedAt)}` : ''}<span>Refreshes every 15 seconds</span></div>
             </div>
             {#if data.entries.length}
                 <div class="mobile-view"><button class="text-button" aria-pressed={fullTable} onclick={() => fullTable = !fullTable}>{fullTable ? 'Show selected stat only' : 'Show all stats and player IDs'}</button></div>
@@ -227,7 +229,7 @@
                 </div>
                 <div class="pagination">
                     <span>{number.format(data.offset + 1)}–{number.format(data.offset + data.entries.length)} of {number.format(data.total)} {isEfficiencyMetric(metric) ? 'qualifying ' : ''}{data.total === 1 ? 'player' : 'players'}{query ? ` matching “${query}”` : ''}</span>
-                    <div><Button size="sm" disabled={data.offset === 0 || loading} onclick={() => updateFilters({ offset: String(Math.max(0, offset - pageSize)) })}>Previous</Button><Button size="sm" disabled={data.offset + data.entries.length >= data.total || loading} onclick={() => updateFilters({ offset: String(offset + pageSize) })}>Next</Button></div>
+                    <div><Button size="sm" disabled={data.offset === 0 || loading || changingFilters} onclick={() => updateFilters({ offset: String(Math.max(0, offset - pageSize)) })}>Previous</Button><Button size="sm" disabled={data.offset + data.entries.length >= data.total || loading || changingFilters} onclick={() => updateFilters({ offset: String(offset + pageSize) })}>Next</Button></div>
                 </div>
             {:else if isEfficiencyMetric(metric)}
                 <EmptyState spacious title="No qualifying players found" description={`${efficiencyRequirement(metric)} in this game/map selection. Try another scope or player search.`}>

src/routes/leaderboard/players/[guid]/+page.server.ts

@@ -1,9 +1,9 @@
 import { error } from '@sveltejs/kit';
-import { getPlayerProfile } from '$lib/server/fleet/player-profile.js';
-import { getPlayerLeaderboardEntry, isLeaderboardMetric } from '$lib/server/fleet/leaderboard.js';
+import { playerReport } from '$lib/server/reports/index.js';
+import { isLeaderboardMetric } from '$lib/server/fleet/leaderboard.js';
 import { leaderboardReturnHref } from '$lib/leaderboard-navigation.js';
 
-export function load({ params, url }: { params: { guid: string }; url: URL }) {
+export async function load({ params, url }: { params: { guid: string }; url: URL }) {
     const game = url.searchParams.get('game');
     const map = url.searchParams.get('map');
     const requestedMetric = url.searchParams.get('metric');
@@ -13,8 +13,8 @@ export function load({ params, url }: { params: { guid: string }; url: URL }) {
     }
     const metric = requestedMetric !== null && isLeaderboardMetric(requestedMetric) ? requestedMetric : 'highestRound';
     const options = { game: game || undefined, map: map || undefined, metric };
-    const profile = getPlayerProfile(params.guid, Number(url.searchParams.get('offset') ?? 0), options);
+    const { profile, entry } = await playerReport(params.guid, Number(url.searchParams.get('offset') ?? 0), options);
     if (!profile) error(404, 'No recorded stats found for this player');
     return { profile, returnTo: leaderboardReturnHref(url.searchParams.get('returnTo')),
-        scope: { game: game ?? '', map: map ?? '', metric, entry: getPlayerLeaderboardEntry(params.guid, options) } };
+        scope: { game: game ?? '', map: map ?? '', metric, entry } };
 }

src/routes/metrics/+page.server.ts

@@ -1,7 +1,7 @@
-import { readMetrics } from '$lib/server/fleet/metrics.js';
+import { metricsReport } from '$lib/server/reports/index.js';
 import { parseMetricsQuery } from '$lib/metrics.js';
 import type { PageServerLoad } from './$types';
 
 export const load: PageServerLoad = ({ url }) => {
-    return readMetrics(parseMetricsQuery(url.searchParams));
+    return metricsReport(parseMetricsQuery(url.searchParams));
 };

src/routes/servers/[name]/+page.server.ts

@@ -0,0 +1,4 @@
+import { fleetSnapshot } from '$lib/server/fleet/snapshot.js';
+import { fleetOperations } from '$lib/server/fleet/overview.js';
+
+export function load() { return { fleetSnapshot: { ...fleetSnapshot(), operations: fleetOperations() } }; }

src/routes/servers/[name]/+page.svelte

@@ -1,9 +1,13 @@
 <script lang="ts">
+    let { data }: { data: import('./$types').PageData } = $props();
+    const snapshot = $derived($fleet.receivedAt ? $fleet : data.fleetSnapshot);
+    const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.fleetSnapshot.servers);
     import RuntimeVersion from '$ui/components/RuntimeVersion.svelte';
     import MatchHistory from '$ui/components/MatchHistory.svelte';
     import RoundTelemetry from '$ui/components/RoundTelemetry.svelte';
     import TelemetryDiagnostics from '$ui/components/TelemetryDiagnostics.svelte';
     import { pollVisible } from '$ui/poll.js';
+    import { fetchCampaigns } from '$ui/campaigns.js';
     import { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } from '$ui/stores/fleet.js';
     import { freshServer, fleetIdentity } from '$lib/fleet-overview.js';
     import { serverTitle } from '$ui/server-labels.js';
@@ -105,9 +109,9 @@
     }
 
     let name = $derived($page.params.name ?? '');
-    let server = $derived(($fleetServers.find(s => s.name === name) as Server | undefined) ?? null);
+    let server = $derived((visibleFleet.find(s => s.name === name) as Server | undefined) ?? null);
     const native = $derived(!!server?.native_instance_id);
-    let integrations = $derived($fleetIntegrations);
+    let integrations = $derived($fleetIntegrations ?? data.fleetSnapshot.integrations);
     let history = $state<DeploymentRow[]>([]);
     let inventory = $state<InventoryEntry[]>([]);
     let komodoProbe = $state<KomodoProbeResult | null>(null);
@@ -127,27 +131,39 @@
     let tabLoading = $state(false);
     let tabError = $state('');
     let historyLimit = $state(25);
+    let historyTotal = $state(0);
     let inventoryLimit = $state(25);
-    const operations = $derived($fleet.operations.filter(op => op.server === name));
+    const operations = $derived(snapshot.operations.filter(op => op.server === name));
 
-    async function loadTab(currentName: string, tab: string) {
+    let tabSequence = 0;
+    async function loadTab(currentName: string, tab: string, signal?: AbortSignal) {
         if (tab !== 'history' && tab !== 'inventory') return;
+        const sequence = ++tabSequence;
         try {
-            const response = await fetch(tab === 'history' ? '/api/campaigns?server=' + encodeURIComponent(currentName) : '/api/servers/' + encodeURIComponent(currentName) + '/scripts');
-            if (!response.ok) throw new Error('Could not refresh ' + tab + '.');
-            const body = await response.json();
-            if (name !== currentName || activeTab !== tab) return;
-            if (tab === 'history') history = body.deployments; else inventory = body.scripts;
+            if (tab === 'history') {
+                const body = await fetchCampaigns<DeploymentRow>({ server: currentName }, historyLimit, signal);
+                if (sequence !== tabSequence || name !== currentName || activeTab !== tab || signal?.aborted) return;
+                history = body.deployments; historyTotal = body.total;
+            } else {
+                const response = await fetch('/api/servers/' + encodeURIComponent(currentName) + '/scripts', { signal });
+                if (!response.ok) throw new Error('Could not refresh inventory.');
+                const body = await response.json();
+                if (sequence !== tabSequence || name !== currentName || activeTab !== tab || signal?.aborted) return;
+                inventory = body.scripts;
+            }
             tabError = '';
-        } catch (error) { if (name === currentName && activeTab === tab) tabError = (error as Error).message; }
-        finally { if (name === currentName && activeTab === tab) tabLoading = false; }
+            return true;
+        } catch (error) {
+            if (sequence === tabSequence && name === currentName && activeTab === tab && !signal?.aborted) tabError = (error as Error).message;
+            return false;
+        } finally { if (sequence === tabSequence && name === currentName && activeTab === tab && !signal?.aborted) tabLoading = false; }
     }
     $effect(() => {
         const currentName = name;
         const tab = activeTab;
         tabError = ''; history = []; inventory = []; historyLimit = 25; inventoryLimit = 25;
         tabLoading = tab === 'history' || tab === 'inventory';
-        return pollVisible(() => loadTab(currentName, tab), 10000);
+        return pollVisible(signal => loadTab(currentName, tab, signal), 10000);
     });
     $effect(() => { name; komodoProbe = null; rconProbe = null; netDiagnosis = null; });
     const fresh = $derived(server ? freshServer(server, $uiNow) : false);
@@ -312,7 +328,7 @@
 
         </Card>
 
-        {#if $fleetError}<div class="ui-error" role="alert">{$fleetError}<Button size="sm" onclick={refreshFleet}>Retry</Button></div>{/if}
+        {#if $fleetError}<div class="ui-error" role="alert">{$fleetError}<Button size="sm" onclick={() => refreshFleet(true)}>Retry</Button></div>{/if}
         {#if hasDiagnosticIssue}
             <section class="connection-issue" aria-label="Server connection issue">
                 <div><strong>{effectiveHealth === 'stale' ? 'Server report is outdated' : 'Server is not responding'}</strong><p>Player count is unknown. {effectiveSeen ? `Last successful report ${ago(effectiveSeen, $uiNow)}.` : 'No successful report has been received.'} {native && !server.host?.connected ? 'The host agent is offline; the game may still be running.' : !native && integrations && !integrations.rcon ? 'RCON is not configured.' : ''}</p></div>
@@ -588,13 +604,13 @@
                         </div>
                     {/if}
                 </Card>
-                <ListFooter total={history.length} bind:limit={historyLimit} label="campaigns" />
+                <ListFooter total={historyTotal} bind:limit={historyLimit} onchange={() => loadTab(name, 'history')} label="campaigns" />
             {/if}
         </div>
     </div>
 {:else if $fleetError}
-    <div class="ui-error" role="alert">{$fleetError}<Button onclick={refreshFleet}>Retry</Button></div>
-{:else if $fleet.receivedAt}
+    <div class="ui-error" role="alert">{$fleetError}<Button onclick={() => refreshFleet(true)}>Retry</Button></div>
+{:else if $fleet.receivedAt || data.fleetSnapshot}
     <EmptyState title="Server not found" description="This server may have been removed from the fleet." />
 {:else}
     <div class="loading">Loading…</div>

static/agent/ec-agent.py

@@ -30,7 +30,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.3.0"
+VERSION = "1.4.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -329,10 +329,47 @@ def managed_write(root, value, content, uid=1000, gid=1000):
                 os.unlink(temp_name, dir_fd=parent)
 
 
+class ReadWriteGate:
+    """Let observations run together; global mutations wait for existing readers."""
+    def __init__(self):
+        self.condition = threading.Condition()
+        self.readers = 0
+        self.writers = 0
+
+    @contextlib.contextmanager
+    def read(self):
+        with self.condition:
+            while self.writers:
+                self.condition.wait()
+            self.readers += 1
+        try:
+            yield
+        finally:
+            with self.condition:
+                self.readers -= 1
+                self.condition.notify_all()
+
+    @contextlib.contextmanager
+    def write(self):
+        # The mutation lane is serial, so there is only one writer.
+        with self.condition:
+            self.writers += 1
+            while self.readers:
+                self.condition.wait()
+        try:
+            yield
+        finally:
+            with self.condition:
+                self.writers -= 1
+                self.condition.notify_all()
+
+
 class Journal:
-    def __init__(self, path):
+    def __init__(self, path, recover=True):
         self.db = sqlite3.connect(path)
         self.db.row_factory = sqlite3.Row
+        if not recover:
+            return
         self.db.execute("PRAGMA journal_mode=WAL")
         self.db.execute("PRAGMA synchronous=FULL")
         self.db.execute("CREATE TABLE IF NOT EXISTS operations (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)")
@@ -672,6 +709,10 @@ class Agent:
         self.docker = docker or Docker()
         self.stopping = threading.Event()
         self.current_operation = None
+        self.instance_locks = {}
+        self.read_gate = ReadWriteGate()
+        self.read_claim_lock = threading.Lock()
+        self.read_claims = set()
         self.cancelled_creations = frozenset()
         self.library_progress = {}
         self.library_transfers = {}
@@ -732,17 +773,25 @@ class Agent:
         return Path(self.config_path) == AGENT_CONFIG and (UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink())
 
     @contextlib.contextmanager
-    def activity(self):
+    def activity(self, read_only=False):
         path = self.root / "agent" / "activity.lock"
         descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+        mutation_descriptor = None
         try:
             try:
-                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                # All work excludes the updater. Mutations additionally exclude
+                # other mutation batches while read lanes may continue.
+                fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB)
+                if not read_only:
+                    mutation_descriptor = os.open(self.root / "agent" / "mutation.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+                    fcntl.flock(mutation_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
             except BlockingIOError:
                 yield False
                 return
             yield True
         finally:
+            if mutation_descriptor is not None:
+                os.close(mutation_descriptor)
             os.close(descriptor)
 
     def record_health(self):
@@ -953,7 +1002,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["instance.cancel-create"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1860,23 +1909,89 @@ class Agent:
             except (AgentError, OSError, ValueError):
                 pass
 
+    @staticmethod
+    def read_operation(operation):
+        return operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "logs.read") or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
+
+    def instance_lock(self, operation):
+        key = operation.get("instanceId")
+        return self.instance_locks.setdefault(key, threading.RLock()) if key else contextlib.nullcontext()
+
+    def read_loop(self):
+        # Each lane owns its SQLite connection and never writes executing_id,
+        # checkpoints, or current_operation used by the mutation state machine.
+        journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
+        try:
+            while not self.stopping.is_set():
+                try:
+                    if self.update_pending():
+                        self.stopping.wait(3)
+                        continue
+                    response = self.request("GET", "/api/agent/operations?lane=read&wait=20")
+                    for operation in response.get("operations", []):
+                        if self.stopping.is_set() or not self.read_operation(operation):
+                            continue
+                        with self.read_claim_lock:
+                            if operation["id"] in self.read_claims:
+                                continue
+                            self.read_claims.add(operation["id"])
+                        try:
+                            with self.activity(read_only=True) as acquired:
+                                if not acquired or self.update_pending():
+                                    continue
+                                with self.read_gate.read(), self.instance_lock(operation):
+                                    row, _ = journal.accept(operation)
+                                    if row["state"] in ("accepted", "unknown", "running"):
+                                        self.report(journal.update(operation["id"], "running"))
+                                        try:
+                                            result = self.dispatch(operation)
+                                            row = journal.update(operation["id"], "completed", result)
+                                        except Exception as err:
+                                            row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
+                                    self.report(row)
+                        finally:
+                            with self.read_claim_lock:
+                                self.read_claims.discard(operation["id"])
+                except (AgentError, OSError, ValueError, KeyError, sqlite3.Error):
+                    self.stopping.wait(3)
+        finally:
+            journal.close()
+
     def poll_operations(self):
+        if self.stopping.is_set() or self.update_pending():
+            return
+        # Check both update and mutation locks before delivery, release them
+        # during the long poll, and recheck immediately before accepting work.
+        with self.activity() as acquired:
+            if not acquired:
+                return
+        # Never hold the updater lock while waiting for delivery.
+        response = self.request("GET", "/api/agent/operations?lane=write&wait=20")
         with self.activity() as acquired:
             if not acquired or self.stopping.is_set() or self.update_pending():
                 return
             if self.config.get("pendingCredential"):
                 self.recover_credential()
-            response = self.request("GET", "/api/agent/operations")
+            completed = True
             for operation in response.get("operations", []):
                 if self.stopping.is_set():
                     break
+                if self.read_operation(operation):
+                    continue
                 self.current_operation = operation["id"]
+                # Global changes wait for observations; instance changes only
+                # block observations for their own immutable instance identity.
+                guard = self.read_gate.write() if operation["kind"] in ("host.decommission", "credentials.rotate") else contextlib.nullcontext()
                 try:
-                    accepted, _ = self.journal.accept(operation)
-                    self.report(accepted)
-                    self.report(self.execute(operation, report_running=True))
+                    with guard, self.instance_lock(operation):
+                        accepted, _ = self.journal.accept(operation)
+                        self.report(accepted)
+                        result = self.execute(operation, report_running=True)
+                        self.report(result)
+                        completed = completed and result["state"] in ("completed", "failed")
                 finally:
                     self.current_operation = None
+            return completed
 
     def run(self):
         # Obtain the configured host port range before accepting any deployment.
@@ -1895,13 +2010,19 @@ class Agent:
                 self.stopping.wait(3)
         heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
         heartbeats.start()
+        readers = [threading.Thread(target=self.read_loop, daemon=True) for _ in range(2)]
+        for reader in readers:
+            reader.start()
         while not self.stopping.is_set():
+            completed = False
             try:
-                self.poll_operations()
+                completed = self.poll_operations()
             except (AgentError, OSError, ValueError, KeyError):
                 # Logs deliberately omit URLs, tokens, payloads and raw errors.
                 print("Host controller contact unavailable; existing containers continue running", file=sys.stderr)
-            self.stopping.wait(3)
+            self.stopping.wait(0.25 if completed else 3)
+        for reader in readers:
+            reader.join(timeout=2)
         heartbeats.join(timeout=2)
 
 

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.3.0",
+  "version": "1.4.0",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "a698bdbba3716726a5a63fbd18392b53bd4c1e0c4cbaca94eabd66d8ca328dc2",
+  "sha256": "8bb6bf3c433def0540cf0de11304f23820c9afcaf1d90d8cdaa7a2a09a455f2d",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/helpers/fixture.ts

@@ -47,6 +47,8 @@ export async function newFixture(servers: string[] = ['t5_zombie_theater_1']): P
         dbPath,
         cleanup: async () => {
             try {
+                const { closeReports } = await import('../../src/lib/server/reports/index.js');
+                await closeReports();
                 const mod = (await import('../../src/lib/server/db/index.js')) as {
                     closeDb: () => void;
                 };

tests/unit/game-file-uploads.test.ts

@@ -5,7 +5,7 @@ import { join } from 'node:path';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 import {
     getUploadFileProgress, hashUploadedFiles, MAX_UPLOAD_CHUNK_BYTES,
-    receiveUploadChunk, UploadOffsetError, validateUploadPlan, withUploadLock
+    receiveUploadChunk, UploadOffsetError, validateUploadPlan, withUploadLock, withUploadFileLock
 } from '../../src/lib/server/game-files/uploads.js';
 import { manifestChecksum, verifyGameFile } from '../../src/lib/server/game-files/storage.js';
 
@@ -22,6 +22,22 @@ beforeEach(async () => {
 });
 afterEach(async () => { await rm(directory, { recursive: true, force: true }); });
 
+it('allows independent files together, serializes same-file offsets, and makes publication a barrier', async () => {
+    let release!: () => void;
+    const blocked = new Promise<void>(resolve => { release = resolve; });
+    const order: string[] = [];
+    const first = withUploadFileLock(root, 'a', async () => { order.push('a1'); await blocked; });
+    const same = withUploadFileLock(root, 'a', async () => { order.push('a2'); });
+    const other = withUploadFileLock(root, 'b', async () => { order.push('b'); });
+    const publish = withUploadLock(root, async () => { order.push('publish'); });
+    const later = withUploadFileLock(root, 'c', async () => { order.push('c'); });
+    await other;
+    expect(order).toEqual(['a1', 'b']);
+    release();
+    await Promise.all([first, same, publish, later]);
+    expect(order).toEqual(['a1', 'b', 'a2', 'publish', 'c']);
+});
+
 describe('browser folder upload plans', () => {
     it('preserves folder structure and source identity while sorting canonical paths', () => {
         expect(validateUploadPlan([{ path: 'zone/map.ff', size: 10, lastModified: 100 }, { path: 'main/data.iwd', size: 0 }])).toEqual([

tests/unit/host-preparation.test.ts

@@ -89,20 +89,20 @@ describe('automatic host preparation', () => {
         });
         expect(operations(host.id)).toEqual([]);
 
-        const firstPoll = await pollOperations({ request: request('operations') }).json();
+        const firstPoll = await (await pollOperations({ request: request('operations') })).json();
         expect(firstPoll.operations).toHaveLength(1);
         const runtimeOperation = firstPoll.operations[0];
         expect(runtimeOperation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime } });
-        expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([runtimeOperation]);
+        expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]);
         const version = await publish();
-        expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([runtimeOperation]);
+        expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]);
 
         vi.setSystemTime(Date.now() + 1);
         const completedRuntime = await reportOperation({
             request: request(`operations/${runtimeOperation.id}`, { state: 'completed' }), params: { id: runtimeOperation.id }
         });
         expect(completedRuntime.status).toBe(200);
-        expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([]);
+        expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]);
         expect(operations(host.id)).toHaveLength(1);
 
         vi.setSystemTime(Date.now() + 1);
@@ -113,7 +113,7 @@ describe('automatic host preparation', () => {
         const download = operations(host.id)[0];
         expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } });
         expect((await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) })).status).toBe(200);
-        expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([download]);
+        expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([download]);
         expect(operations(host.id)).toHaveLength(2);
 
         vi.setSystemTime(Date.now() + 1);
@@ -129,7 +129,7 @@ describe('automatic host preparation', () => {
         expect(finalHeartbeat.status).toBe(200);
         expect(libraries.requireReadyLibrary(host.id, version.id).id).toBe(version.id);
         expect(preparation.hostPreparationStatus(hosts.getHost(host.id)!).state).toBe('ready');
-        expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([]);
+        expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]);
         expect(operations(host.id)).toHaveLength(2);
     });
 

tests/unit/leaderboard-efficiency-query.test.ts

@@ -123,7 +123,7 @@ it('qualifies before competition ranking, then searches and paginates without ch
     expect(page).toMatchObject({ total: 3, entries: [{ guid: 'Blake', rank: 1 }], summary: full.summary });
     expect(getLeaderboard({ metric: 'averageKills', offset: 3 })).toMatchObject({ total: 3, entries: [], summary: full.summary });
     const { GET } = await import('../../src/routes/api/leaderboard/+server.js');
-    const response = GET({ url: new URL('http://localhost/api/leaderboard?metric=averageKills&game=t5&q=current&limit=1') });
+    const response = await GET({ url: new URL('http://localhost/api/leaderboard?metric=averageKills&game=t5&q=current&limit=1') });
     expect(response.status).toBe(200);
     expect(await response.json()).toMatchObject({ metric: 'averageKills', total: 1,
         entries: [{ guid: 'Casey', rank: 3, averageKills: 10, gamesPlayed: 5 }], summary: full.summary });
@@ -135,7 +135,7 @@ it('accepts each efficiency metric through the API when the qualified group is e
     const { efficiencyMetrics } = await import('../../src/lib/leaderboard-efficiency.js');
     const { GET } = await import('../../src/routes/api/leaderboard/+server.js');
     for (const { key } of efficiencyMetrics) {
-        const response = GET({ url: new URL(`http://localhost/api/leaderboard?metric=${key}`) });
+        const response = await GET({ url: new URL(`http://localhost/api/leaderboard?metric=${key}`) });
         expect(response.status).toBe(200);
         expect(await response.json()).toMatchObject({ metric: key, entries: [], total: 0,
             summary: { players: 1, matches: 4, kills: 80 } });

tests/unit/leaderboard-query.test.ts

@@ -386,15 +386,15 @@ describe('player profiles', () => {
         expect(getPlayerProfile('101', 9999)?.offset).toBe(25);
         for (const offset of [-1, NaN, 0.5]) expect(getPlayerProfile('101', offset)?.offset).toBe(0);
         const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js');
-        expect(load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?offset=25') }).profile).toEqual(second);
+        expect((await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?offset=25') })).profile).toEqual(second);
     });
 
     it('loads canonical profile URLs, encodes GUIDs and returns 404 for missing players', async () => {
         await seedRecords();
         const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js');
         const url = new URL('http://localhost/leaderboard/players/101');
-        expect(load({ params: { guid: '101' }, url }).profile).toMatchObject({ name: 'Ace', stats: { kills: 770, gamesPlayed: 4 } });
-        expect(() => load({ params: { guid: 'missing' }, url })).toThrow(expect.objectContaining({ status: 404 }));
+        expect((await load({ params: { guid: '101' }, url })).profile).toMatchObject({ name: 'Ace', stats: { kills: 770, gamesPlayed: 4 } });
+        await expect(load({ params: { guid: 'missing' }, url })).rejects.toThrow(expect.objectContaining({ status: 404 }));
         const { playerProfileHref } = await import('../../src/lib/leaderboard.js');
         expect(playerProfileHref('001/2?#')).toBe('/leaderboard/players/001%2F2%3F%23');
     });
@@ -417,7 +417,7 @@ describe('leaderboard API', () => {
         const handle = await seedRecords();
         handle.prepare("UPDATE leaderboard_players SET name = 'JoshWright' WHERE guid = '101'").run();
         const { GET } = await import('../../src/routes/api/leaderboard/+server.js');
-        const response = GET({ url: new URL('http://localhost/api/leaderboard?q=JoshWright&metric=kills') });
+        const response = await GET({ url: new URL('http://localhost/api/leaderboard?q=JoshWright&metric=kills') });
         expect(response.status).toBe(200);
         const result = await response.json() as LeaderboardResponse;
         expect(result).toMatchObject({ total: 1, summary: { players: 4, matches: 5, kills: 1140 } });
@@ -429,7 +429,7 @@ describe('leaderboard API', () => {
     it('returns the requested ranking and pagination', async () => {
         await seedRecords();
         const { GET } = await import('../../src/routes/api/leaderboard/+server.js');
-        const response = GET({ url: new URL('http://localhost/api/leaderboard?game=t5&map=t5%3Atheater&metric=kills&limit=1&offset=1') });
+        const response = await GET({ url: new URL('http://localhost/api/leaderboard?game=t5&map=t5%3Atheater&metric=kills&limit=1&offset=1') });
         expect(response.status).toBe(200);
         const result = await response.json() as LeaderboardResponse;
         expect(result).toMatchObject({ metric: 'kills', limit: 1, offset: 1, total: 3 });
@@ -442,7 +442,7 @@ describe('leaderboard API', () => {
         for (const query of ['game=t7', 'game=', 'metric=unknown', 'limit=0', 'limit=101', 'limit=',
             'limit=1.5', 'limit=1e1', 'offset=-1', 'offset=1000001', 'offset=Infinity', 'offset=0.5',
             `q=${'a'.repeat(101)}`, `map=${'a'.repeat(1025)}`]) {
-            const response = GET({ url: new URL(`http://localhost/api/leaderboard?${query}`) });
+            const response = await GET({ url: new URL(`http://localhost/api/leaderboard?${query}`) });
             expect(response.status, query).toBe(400);
             expect(await response.json()).toMatchObject({ ok: false, kind: 'bad-request' });
         }
@@ -455,9 +455,9 @@ describe('leaderboard API', () => {
         handle.prepare('UPDATE leaderboard_matches SET map = ?, map_key = ?, map_name = ? WHERE id = 1')
             .run(map, key, map);
         const { GET } = await import('../../src/routes/api/leaderboard/+server.js');
-        const all = await GET({ url: new URL('http://localhost/api/leaderboard') }).json() as LeaderboardResponse;
+        const all = await (await GET({ url: new URL('http://localhost/api/leaderboard') })).json() as LeaderboardResponse;
         const returnedKey = all.maps.find(option => option.map === map)!.key;
-        const response = GET({ url: new URL(`http://localhost/api/leaderboard?map=${encodeURIComponent(returnedKey)}`) });
+        const response = await GET({ url: new URL(`http://localhost/api/leaderboard?map=${encodeURIComponent(returnedKey)}`) });
         expect(response.status).toBe(200);
         expect(await response.json()).toMatchObject({ total: 2, summary: { matches: 1, kills: 200 } });
     });

tests/unit/metrics.test.ts

@@ -150,16 +150,16 @@ describe('historical metrics', () => {
         vi.spyOn(Date, 'now').mockReturnValue(at + 30_000);
         for (const range of ['1', '7', '30', 'invalid', '365', '-1', '']) {
             const url = new URL(`http://localhost/metrics?days=${range}`);
-            const response = GET({ url });
+            const response = await GET({ url });
             const payload = await response.json();
             expect(response.headers.get('cache-control')).toBe('no-store');
             expect(payload).toEqual(await load({ url } as Parameters<typeof load>[0]));
             expect(payload.days).toBe(['1', '7', '30'].includes(range) ? Number(range) : 7);
         }
-        expect((await GET({ url: new URL('http://localhost/api/metrics') }).json()).days).toBe(7);
+        expect((await (await GET({ url: new URL('http://localhost/api/metrics') })).json()).days).toBe(7);
         for (const search of ['days=1&game=t5&map=Moon&timeZone=Asia%2FKathmandu', 'days=7&map=Moon', 'game=t6&timeZone=Invalid']) {
             const url = new URL(`http://localhost/metrics?${search}`);
-            const payload = await GET({ url }).json();
+            const payload = await (await GET({ url })).json();
             const { days, ...scope } = parseMetricsQuery(url.searchParams);
             expect(payload).toEqual(await load({ url } as Parameters<typeof load>[0]));
             expect(payload).toMatchObject({ days, scope });

tests/unit/native-hosts.test.ts

@@ -15,6 +15,24 @@ async function enrolled() {
 }
 
 describe('native host enrollment and scoped authentication', () => {
+    it('separates observation and mutation delivery and wakes long polls on newly queued work', async () => {
+        const api = await enrolled();
+        const inspect = api.queueHostOperation(api.host.id, 'host.inspect', {});
+        const restart = api.queueHostOperation(api.host.id, 'instance.restart', {});
+        const status = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'status' });
+        const command = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'map_restart' });
+        expect(api.pendingHostOperations(api.host.id, 'read').map(op => op.id).sort()).toEqual([inspect.id, status.id].sort());
+        expect(api.pendingHostOperations(api.host.id, 'write').map(op => op.id).sort()).toEqual([restart.id, command.id].sort());
+        // Let earlier enqueue notifications finish before subscribing.
+        await Promise.resolve();
+        const controller = new AbortController();
+        const waiting = api.waitForHostWork(api.host.id, controller.signal, 20_000);
+        api.queueHostOperation(api.host.id, 'host.inspect', {});
+        await waiting;
+        const aborted = api.waitForHostWork(api.host.id, controller.signal, 20_000);
+        controller.abort();
+        await aborted;
+    });
     it('consumes an enrollment exactly once and stores only credential hashes', async () => {
         const api = await enrolled();
         expect(api.authenticateCredential(api.credentials.credential)?.id).toBe(api.host.id);

tests/unit/player-profile-scope.test.ts

@@ -74,15 +74,15 @@ it('keeps lifetime profiles available when selected history is empty or an effic
     const { db } = await import('../../src/lib/server/db/index.js');
     addMatches(db(), '101', 4);
     const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js');
-    const filtered = load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?game=t6&map=t6%3Atown&offset=25') });
+    const filtered = await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?game=t6&map=t6%3Atown&offset=25') });
     expect(filtered).toMatchObject({ returnTo: '/leaderboard',
         profile: { stats: { gamesPlayed: 4, kills: 40 }, matches: [], matchesTotal: 0, offset: 0 },
         scope: { game: 't6', map: 't6:town', metric: 'highestRound', entry: null } });
-    const unqualified = load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?metric=averageKills') });
+    const unqualified = await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?metric=averageKills') });
     expect(unqualified).toMatchObject({ profile: { matchesTotal: 4, stats: { gamesPlayed: 4 } },
         scope: { game: '', map: '', metric: 'averageKills', entry: null } });
     addMatches(db(), '101', 1);
-    expect(load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?metric=averageKills') }))
+    expect(await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?metric=averageKills') }))
         .toMatchObject({ scope: { entry: { guid: '101', rank: 1, averageKills: 10, gamesPlayed: 5 } } });
 });
 
@@ -93,13 +93,13 @@ it('loads canonical navigation context while keeping source search and paginatio
     const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js');
     const url = new URL('http://localhost/leaderboard/players/101?game=t5&map=t5%3Atheater&metric=kills');
     url.searchParams.set('returnTo', '/leaderboard?game=t5&map=t5%3Atheater&metric=kills&q=101&offset=50&unknown=drop');
-    const loaded = load({ params: { guid: '101' }, url });
+    const loaded = await load({ params: { guid: '101' }, url });
     expect(loaded.returnTo).toBe('/leaderboard?game=t5&map=t5%3Atheater&metric=kills&q=101&offset=50');
     expect(loaded.scope).toMatchObject({ game: 't5', map: 't5:theater', metric: 'kills', entry: { guid: '101', rank: 2 } });
     expect(loaded.profile.offset).toBe(0);
     url.searchParams.set('returnTo', 'https://external.invalid/leaderboard');
-    expect(load({ params: { guid: '101' }, url }).returnTo).toBe('/leaderboard');
-    const defaults = load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101') });
+    expect((await load({ params: { guid: '101' }, url })).returnTo).toBe('/leaderboard');
+    const defaults = await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101') });
     expect(defaults).toMatchObject({ returnTo: '/leaderboard', scope: { game: '', map: '', metric: 'highestRound' } });
 });
 
@@ -108,13 +108,11 @@ it('rejects invalid filter parameters and retains existing offset normalization
     addMatches(db(), '101', 30);
     const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js');
     for (const query of ['game=t7', 'game=', 'metric=', 'metric=unknown', `map=${'x'.repeat(1025)}`]) {
-        expect(() => load({ params: { guid: '101' }, url: new URL(`http://localhost/leaderboard/players/101?${query}`) }))
-            .toThrow(expect.objectContaining({ status: 400 }));
+        await expect(load({ params: { guid: '101' }, url: new URL(`http://localhost/leaderboard/players/101?${query}`) })).rejects.toThrow(expect.objectContaining({ status: 400 }));
     }
     for (const query of ['offset=garbage', 'offset=-1', 'offset=1.5', 'offset=Infinity']) {
-        expect(load({ params: { guid: '101' }, url: new URL(`http://localhost/leaderboard/players/101?${query}`) }).profile.offset).toBe(0);
+        expect((await load({ params: { guid: '101' }, url: new URL(`http://localhost/leaderboard/players/101?${query}`) })).profile.offset).toBe(0);
     }
-    expect(load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?offset=999') }).profile.offset).toBe(25);
-    expect(() => load({ params: { guid: 'missing' }, url: new URL('http://localhost/leaderboard/players/missing?game=t5') }))
-        .toThrow(expect.objectContaining({ status: 404 }));
+    expect((await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?offset=999') })).profile.offset).toBe(25);
+    await expect(load({ params: { guid: 'missing' }, url: new URL('http://localhost/leaderboard/players/missing?game=t5') })).rejects.toThrow(expect.objectContaining({ status: 404 }));
 });

tests/unit/report-workers.test.ts

@@ -0,0 +1,27 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { parseMetricsQuery } from '../../src/lib/metrics.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
+
+it('runs reports off the request thread and invalidates cached history after writes', async () => {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js');
+    const { metricsReport } = await import('../../src/lib/server/reports/index.js');
+    const now = Date.UTC(2026, 9, 4, 12);
+    vi.spyOn(Date, 'now').mockReturnValue(now);
+    const sample = { server: 'one', game: 't5', map: 'Moon', players: 1, capacity: 4 };
+    recordMetrics([sample], now - 30_000);
+    const query = parseMetricsQuery(new URLSearchParams('days=1'));
+    const first = metricsReport(query);
+    const winner = await Promise.race([first.then(() => 'report'), new Promise<string>(resolve => setTimeout(() => resolve('timer'), 0))]);
+    expect(winner).toBe('timer');
+    expect(await first).toEqual(readMetrics(query, now));
+    expect(await metricsReport(query)).toEqual(await first);
+    db().prepare('UPDATE metric_samples SET players=3').run();
+    const changed = await metricsReport(query);
+    expect(changed).toEqual(readMetrics(query, now));
+    expect(changed.summary.playerHours).toBeCloseTo((await first).summary.playerHours * 3);
+});

tests/unit/server-detail-api.test.ts

@@ -18,8 +18,13 @@ describe('server detail data', () => {
             for (let n = 0; n < 501; n++) { deployment.run('other-' + n, '2026-09-18'); member.run('other-' + n, 'other'); }
         })();
         const { GET } = await import('../../src/routes/api/campaigns/+server.js');
-        const all = await (await GET({ url: new URL('http://localhost/api/campaigns') })).json();
+        const all = await (await GET({ url: new URL('http://localhost/api/campaigns?limit=500') })).json();
         expect(all.deployments).toHaveLength(500);
+        const first = await (await GET({ url: new URL('http://localhost/api/campaigns?limit=25') })).json();
+        const second = await (await GET({ url: new URL('http://localhost/api/campaigns?limit=25&offset=25') })).json();
+        expect(first.total).toBe(502);
+        expect(first.deployments).toHaveLength(25);
+        expect(new Set([...first.deployments, ...second.deployments].map(row => row.id)).size).toBe(50);
         const filtered = await (await GET({ url: new URL('http://localhost/api/campaigns?server=target') })).json();
         expect(filtered.deployments.map((row: { id: string }) => row.id)).toEqual(['older-target']);
         expect(filtered.deployments[0].servers).toHaveLength(2);

tests/unit/telemetry.test.ts

@@ -76,6 +76,7 @@ it('timestamps initial telemetry and subsequent status and telemetry events with
     };
     try {
         await reader.read(); // Initial SSE comment.
+        expect(await nextEvent()).toMatchObject({ snapshot: true, serverNow: startedAt, entries: expect.any(Array) });
         expect(await nextEvent()).toMatchObject({ snapshot: true, entries: [], serverNow: startedAt });
         clock.mockReturnValue(startedAt + 30_000);
         emit({ type: 'server-status', entries: [] });

tests/unit/ui-campaigns.test.ts

@@ -0,0 +1,33 @@
+import { afterEach, expect, it, vi } from 'vitest';
+import { fetchCampaigns } from '../../src/lib/ui/campaigns.js';
+
+afterEach(() => vi.unstubAllGlobals());
+
+it('loads history beyond the API page limit without truncating or fetching unwanted older rows', async () => {
+    const requests: URL[] = [];
+    vi.stubGlobal('fetch', vi.fn(async (input: string) => {
+        const url = new URL(input, 'http://localhost');
+        requests.push(url);
+        const start = Number(url.searchParams.get('offset'));
+        const count = Number(url.searchParams.get('limit'));
+        return Response.json({ total: 900, deployments: Array.from({ length: count }, (_, i) => ({ id: start + i })) });
+    }));
+    const result = await fetchCampaigns<{ id: number }>({ server: 'test-server', view: 'history' }, 525);
+    expect(result.total).toBe(900);
+    expect(result.deployments.map(row => row.id)).toEqual(Array.from({ length: 525 }, (_, i) => i));
+    expect(requests.map(url => Object.fromEntries(url.searchParams))).toEqual([
+        { server: 'test-server', view: 'history', offset: '0', limit: '500' },
+        { server: 'test-server', view: 'history', offset: '500', limit: '25' }
+    ]);
+});
+
+it('loads all active campaigns independently of the history limit', async () => {
+    vi.stubGlobal('fetch', vi.fn(async (input: string) => {
+        const url = new URL(input, 'http://localhost');
+        const start = Number(url.searchParams.get('offset'));
+        return Response.json({ total: 550, deployments: Array.from({ length: start ? 50 : 500 }, (_, i) => ({ id: start + i })) });
+    }));
+    const result = await fetchCampaigns<{ id: number }>({ active: 'true' }, Infinity);
+    expect(result.deployments).toHaveLength(550);
+    expect(result.deployments[549].id).toBe(549);
+});

tests/unit/ui-poll.test.ts

@@ -43,6 +43,8 @@ describe('visible page polling', () => {
         stop = pollVisible(task, 1000, false);
         expect(task).not.toHaveBeenCalled();
         await vi.advanceTimersByTimeAsync(2000);
+        expect(task).toHaveBeenCalledTimes(1);
+        await vi.advanceTimersByTimeAsync(1000);
         expect(task).toHaveBeenCalledTimes(2);
     });
 });

tests/unit/ui-server-clock.test.ts

@@ -67,6 +67,34 @@ async function setup() {
 }
 
 describe('Fleet report clock', () => {
+    it('accepts newer navigation snapshots while keeping stream evidence newer than the snapshot', async () => {
+        const { fleetServers, hydrateFleet } = await setup();
+        await vi.advanceTimersByTimeAsync(2000);
+        emit(epoch + 2000, [status(epoch + 2000, { players: 3 })]);
+        const integrations = { rcon: true, komodo: false, rconTimeoutMs: 3000, defaultRconHost: '' };
+        hydrateFleet({ serverNow: epoch + 1000, servers: [server(epoch + 1000), server(epoch, { name: 'two' })], operations: [], integrations });
+        expect(get(fleetServers)).toHaveLength(2);
+        expect(get(fleetServers)[0].last_players).toBe(3);
+        hydrateFleet({ serverNow: epoch, servers: [], operations: [], integrations });
+        expect(get(fleetServers)).toHaveLength(2);
+    });
+    it('merges partial events without dropping other servers or repainting unchanged elapsed ticks', async () => {
+        snapshot.servers = [server(), server(epoch, { name: 'two' })];
+        const { fleetServers, statusByServer } = await setup();
+        emit(epoch, [status(epoch), status(epoch, { server: 'two' })]);
+        await vi.advanceTimersByTimeAsync(1000);
+        source.dispatchEvent(new MessageEvent('server-status', { data: JSON.stringify({
+            delta: true, serverNow: epoch + 1000, entries: [status(epoch + 1000, { players: 2 })]
+        }) }));
+        expect(get(statusByServer).size).toBe(2);
+        expect(get(fleetServers).find(server => server.name === 't5_zombie_theater_1')?.last_players).toBe(2);
+        const changes = vi.fn();
+        const unsubscribe = fleetServers.subscribe(changes);
+        changes.mockClear();
+        await vi.advanceTimersByTimeAsync(5000);
+        expect(changes).not.toHaveBeenCalled();
+        unsubscribe();
+    });
     it('keeps persisted occupancy and match anchors through live reports before the next status poll', async () => {
         const at = (seconds: number) => new Date(epoch + seconds * 1000).toISOString();
         snapshot.servers = [server(epoch, { container_state: 'running', container_checked_at: at(0) })];

vite.reports.config.ts

@@ -0,0 +1,17 @@
+import { defineConfig } from 'vite';
+import { resolve } from 'node:path';
+
+// Build the same report implementations for Node workers. Their database is
+// read-only; migrations and all application writes stay in the controller.
+export default defineConfig({
+    plugins: [{ name: 'report-database', enforce: 'pre', resolveId(source) {
+        if (source.endsWith('/db/index.js')) return resolve('src/lib/server/reports/database.ts');
+    } }],
+    build: {
+        ssr: true, outDir: '.svelte-kit/reports', emptyOutDir: true,
+        rollupOptions: {
+            input: 'src/lib/server/reports/worker.ts', external: ['better-sqlite3'],
+            output: { entryFileNames: 'worker.mjs' }
+        }
+    }
+});