Add filters and context to host operation history

agent/ec_agent.py

@@ -32,9 +32,13 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.7"
+VERSION = "1.4.9"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
+MANIFEST_MAX_ENTRIES = 10000
+MANIFEST_MAX_BYTES = 16 * 1024 * 1024
+MANIFEST_MAX_SECONDS = 15
+MANIFEST_MAX_DEPTH = 64
 LIBRARY_DOWNLOAD_WORKERS = 4
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
 AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
@@ -296,6 +300,151 @@ def managed_read(root, value):
         return None
 
 
+def managed_manifest(root, payload, stopping):
+    """Hash a bounded managed tree without following runtime-controlled symlinks."""
+    deadline = time.monotonic() + MANIFEST_MAX_SECONDS
+
+    def check_budget():
+        if stopping.is_set() or time.monotonic() >= deadline:
+            raise AgentError("Manifest scan exceeded its time limit or was interrupted")
+
+    roots = payload.get("roots")
+    ignored = payload.get("ignoreNames")
+    paths = payload.get("paths", [])
+    if not isinstance(roots, list) or not roots or len(roots) > 64:
+        raise AgentError("Manifest roots must contain between 1 and 64 managed directories")
+    if not isinstance(ignored, list) or len(ignored) > 256 or any(
+        not isinstance(name, str) or not name or name in (".", "..") or any(char in name for char in ("/", "\\", "\x00"))
+        for name in ignored
+    ):
+        raise AgentError("Manifest ignored names must be plain file or directory names")
+    if not isinstance(paths, list) or len(paths) > 10000:
+        raise AgentError("Manifest managed paths must contain at most 10000 files")
+    for value in roots:
+        check_budget()
+        scoped(root, value)
+    roots = sorted(set(roots))
+    roots = [value for index, value in enumerate(roots) if not any(value.startswith(parent + "/") for parent in roots[:index])]
+    ignored = set(ignored)
+    flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
+    entries = 0
+    result_bytes = len(json.dumps({"files": []}).encode())
+    files = []
+
+    def validate_paths():
+        for value in paths:
+            check_budget()
+            try:
+                with managed_open(root, value):
+                    pass
+            except FileNotFoundError:
+                pass
+
+    def identity(info):
+        return info.st_dev, info.st_ino, info.st_mode, info.st_size, info.st_mtime_ns, info.st_ctime_ns
+
+    def open_directory(app, relative):
+        directory = os.dup(app)
+        try:
+            for part in PurePosixPath(relative).parts:
+                nested = os.open(part, flags, dir_fd=directory)
+                os.close(directory)
+                directory = nested
+            return directory
+        except BaseException:
+            os.close(directory)
+            raise
+
+    def scan(directory, relative, depth):
+        nonlocal entries, result_bytes
+        check_budget()
+        if depth > MANIFEST_MAX_DEPTH:
+            raise AgentError("Manifest scan exceeded its directory depth limit")
+        before = os.fstat(directory)
+        with os.scandir(directory) as children:
+            for child in children:
+                check_budget()
+                entries += 1
+                if entries > MANIFEST_MAX_ENTRIES:
+                    raise AgentError("Manifest scan exceeded its entry limit")
+                if child.name in ignored:
+                    continue
+                info = os.stat(child.name, dir_fd=directory, follow_symlinks=False)
+                if stat_module.S_ISLNK(info.st_mode):
+                    continue
+                path = relative + "/" + child.name
+                if stat_module.S_ISDIR(info.st_mode):
+                    nested = os.open(child.name, flags, dir_fd=directory)
+                    try:
+                        if identity(info) != identity(os.fstat(nested)):
+                            raise AgentError("Manifest directory changed during the scan")
+                        scan(nested, path, depth + 1)
+                        if identity(os.fstat(nested)) != identity(os.stat(child.name, dir_fd=directory, follow_symlinks=False)):
+                            raise AgentError("Manifest directory changed during the scan")
+                    finally:
+                        os.close(nested)
+                elif stat_module.S_ISREG(info.st_mode):
+                    descriptor = os.open(child.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory)
+                    try:
+                        if identity(info) != identity(os.fstat(descriptor)):
+                            raise AgentError("Manifest file changed during the scan")
+                        sha = hashlib.sha256()
+                        size = 0
+                        with os.fdopen(descriptor, "rb", closefd=False) as stream:
+                            while True:
+                                check_budget()
+                                block = stream.read(1024 * 1024)
+                                if not block:
+                                    break
+                                sha.update(block)
+                                size += len(block)
+                        if size != info.st_size or identity(info) != identity(os.fstat(descriptor)) or identity(info) != identity(os.stat(child.name, dir_fd=directory, follow_symlinks=False)):
+                            raise AgentError("Manifest file changed during the scan")
+                    finally:
+                        os.close(descriptor)
+                    item = {"path": path, "size": size, "sha256": sha.hexdigest()}
+                    result_bytes += len(json.dumps(item).encode()) + (2 if files else 0)
+                    if result_bytes > MANIFEST_MAX_BYTES:
+                        raise AgentError("Manifest scan exceeded its result size limit")
+                    files.append(item)
+                else:
+                    raise AgentError("Manifest scan encountered a non-regular managed file")
+        if identity(before) != identity(os.fstat(directory)):
+            raise AgentError("Manifest directory changed during the scan")
+
+    try:
+        app = os.open(root, flags)
+        try:
+            app_identity = os.fstat(app)
+            validate_paths()
+            for relative in roots:
+                check_budget()
+                try:
+                    directory = open_directory(app, relative)
+                except FileNotFoundError:
+                    continue  # A destination directory need not exist yet.
+                try:
+                    scan(directory, relative, len(PurePosixPath(relative).parts))
+                    current_directory = open_directory(app, relative)
+                    try:
+                        if identity(os.fstat(directory)) != identity(os.fstat(current_directory)):
+                            raise AgentError("Manifest directory changed during the scan")
+                    finally:
+                        os.close(current_directory)
+                finally:
+                    os.close(directory)
+            validate_paths()
+            current = os.stat(root, follow_symlinks=False)
+            if (current.st_dev, current.st_ino, current.st_mode) != (app_identity.st_dev, app_identity.st_ino, app_identity.st_mode):
+                raise AgentError("Manifest managed root changed during the scan")
+            check_budget()
+        finally:
+            os.close(app)
+    except OSError:
+        raise AgentError("Manifest scan could not read the complete managed tree") from None
+    return {"files": sorted(files, key=lambda item: item["path"])}
+
+
 def managed_write(root, value, content, uid=1000, gid=1000):
     if content is None:
         try:
@@ -367,6 +516,19 @@ class ReadWriteGate:
                 self.condition.notify_all()
 
 
+def read_operation(operation):
+    if not isinstance(operation, dict):
+        return False
+    if operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read"):
+        return True
+    payload = operation.get("payload")
+    return operation.get("kind") == "rcon.execute" and isinstance(payload, dict) and payload.get("command") in ("status", "getstatus", "version")
+
+
+def valid_observation_keep_days(value):
+    return type(value) is int and 0 <= value <= 36500
+
+
 class Journal:
     def __init__(self, path, recover=True):
         self.db = sqlite3.connect(path)
@@ -376,8 +538,14 @@ class Journal:
         self.db.execute("PRAGMA journal_mode=WAL")
         self.db.execute("PRAGMA synchronous=FULL")
         self.db.execute("CREATE TABLE IF NOT EXISTS operations (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)")
-        if "checkpoint" not in {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}:
+        columns = {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}
+        if "checkpoint" not in columns:
             self.db.execute("ALTER TABLE operations ADD COLUMN checkpoint TEXT")
+        if "reported_at" not in columns:
+            self.db.execute("ALTER TABLE operations ADD COLUMN reported_at REAL")
+        if "retention_eligible" not in columns:
+            self.db.execute("ALTER TABLE operations ADD COLUMN retention_eligible INTEGER NOT NULL DEFAULT 0")
+        self.db.execute("CREATE INDEX IF NOT EXISTS operations_observation_retention ON operations(updated,id) WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1")
         self.db.execute("UPDATE operations SET state='unknown', error='Agent restarted during execution; inspect before reconciliation', updated=? WHERE state='running'", (time.time(),))
         self.db.commit()
 
@@ -402,10 +570,37 @@ class Journal:
         return dict(row)
 
     def update(self, op_id, state, result=None, error=None):
-        self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=? WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id))
+        self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=?, reported_at=NULL, retention_eligible=0 WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id))
         self.db.commit()
         return self.get(op_id)
 
+    def confirm_completed(self, row):
+        """Only an acknowledged, unchanged completion can become disposable."""
+        if row["state"] != "completed":
+            return
+        eligible = False
+        try:
+            result = json.loads(row["result"]) if row.get("result") is not None else None
+            eligible = read_operation(json.loads(row["request"])) and not row.get("error") and not (
+                isinstance(result, dict) and (result.get("ok") is False or result.get("healthy") is False or result.get("error") not in (None, ""))
+            )
+        except (ValueError, TypeError, KeyError):
+            pass  # Malformed historical records are retained for inspection.
+        self.db.execute("""UPDATE operations SET reported_at=COALESCE(reported_at,?), retention_eligible=?
+            WHERE id=? AND state='completed' AND updated=? AND result IS ? AND error IS ?""",
+            (time.time(), int(bool(eligible)), row["id"], row["updated"], row.get("result"), row.get("error")))
+        self.db.commit()
+
+    def prune_observations(self, keep_days):
+        if not valid_observation_keep_days(keep_days) or keep_days == 0:
+            return 0
+        cutoff = time.time() - keep_days * 86400
+        removed = self.db.execute("""DELETE FROM operations WHERE id IN (
+            SELECT id FROM operations WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1
+                AND updated<? ORDER BY updated,id LIMIT 500)""", (cutoff,)).rowcount
+        self.db.commit()
+        return removed
+
     def close(self):
         self.db.close()
 
@@ -719,6 +914,7 @@ class Agent:
         self.read_claim_lock = threading.Lock()
         self.read_claims = set()
         self.cancelled_creations = frozenset()
+        self.observation_keep_days = 0  # Older controllers do not authorize cleanup.
         self.library_progress = {}
         self.library_transfers = {}
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
@@ -1094,7 +1290,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1446,6 +1642,8 @@ class Agent:
             return result
         if kind == "logs.read":
             return self.read_log(instance_id, generation, payload)
+        if kind == "files.manifest":
+            return managed_manifest(app, payload, self.stopping)
         path = scoped(app, payload.get("path", ""), allow_root=kind in ("files.list", "files.stat"))
         if kind == "files.read":
             content = managed_read(app, payload["path"])
@@ -1967,7 +2165,7 @@ class Agent:
                     return self.journal.update(operation["id"], "failed", error="Interrupted library download could not be resumed")
             if kind == "files.stage":
                 return self.journal.update(operation["id"], "completed", self.stage(instance_id, generation, payload, operation["id"]))
-            if kind in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "logs.read") or kind == "rcon.execute" and payload.get("command") in ("status", "getstatus", "version"):
+            if self.read_operation(operation):
                 return self.journal.update(operation["id"], "completed", self.dispatch(operation))
             if kind in ("files.activate", "files.rollback") and checkpoint:
                 stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json"
@@ -2021,7 +2219,7 @@ class Agent:
             pass
         return row
 
-    def report(self, row):
+    def report(self, row, journal=None):
         payload = {"state": row["state"]}
         if row.get("result"):
             payload["result"] = json.loads(row["result"])
@@ -2031,13 +2229,19 @@ class Agent:
                 payload["state"] = "running"
         if row.get("error"):
             payload["error"] = row["error"]
-        return self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload)
+        response = self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload)
+        confirmed = response.get("operation") if isinstance(response, dict) else None
+        if row["state"] == "completed" and isinstance(confirmed, dict) and confirmed.get("id") == row["id"] and confirmed.get("state") == "completed":
+            (journal if journal is not None else self.journal).confirm_completed(row)
+        return response
 
     def heartbeat(self):
         response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation})
         # Only the heartbeat thread replaces this immutable snapshot; it never
         # mutates the operation journal or races Docker cleanup against create.
         self.cancelled_creations = frozenset(value for value in response.get("cancelledCreations", []) if isinstance(value, str))
+        keep_days = response.get("observationKeepDays")
+        self.observation_keep_days = keep_days if valid_observation_keep_days(keep_days) else 0
         host = response.get("host", {})
         if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
             self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}
@@ -2054,7 +2258,7 @@ class Agent:
 
     @staticmethod
     def read_operation(operation):
-        return operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "logs.read") or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
+        return read_operation(operation)
 
     def instance_lock(self, operation):
         key = operation.get("instanceId")
@@ -2068,12 +2272,19 @@ class Agent:
         # Each lane owns its SQLite connection and never writes executing_id,
         # checkpoints, or current_operation used by the mutation state machine.
         journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
+        next_prune = time.monotonic()
         try:
             while not self.stopping.is_set():
                 try:
                     if self.update_pending():
                         self.stopping.wait(3)
                         continue
+                    if lane == "read" and time.monotonic() >= next_prune:
+                        next_prune = time.monotonic() + 60
+                        with self.activity(read_only=True) as acquired:
+                            if acquired and not self.update_pending():
+                                if journal.prune_observations(self.observation_keep_days) == 500:
+                                    next_prune = time.monotonic() + 15
                     response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20")
                     handled = False
                     for operation in response.get("operations", []):
@@ -2091,7 +2302,7 @@ class Agent:
                                     continue
                                 row, _ = journal.accept(operation)
                                 if row["state"] in ("accepted", "unknown", "running"):
-                                    self.report(journal.update(operation["id"], "running"))
+                                    self.report(journal.update(operation["id"], "running"), journal=journal)
                                     with self.read_gate.read(), self.instance_lock(operation):
                                         try:
                                             result = self.dispatch(operation)
@@ -2100,7 +2311,7 @@ class Agent:
                                             row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
                                 # Receipt delivery can take seconds; it must not
                                 # hold the instance lock needed by health checks.
-                                self.report(row)
+                                self.report(row, journal=journal)
                                 handled = True
                         finally:
                             with self.read_claim_lock:

agent/tests/test_agent.py

@@ -57,6 +57,318 @@ class Response(io.BytesIO):
         self.headers = headers or {}
 
 
+class JournalRetentionTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.path = Path(self.temp.name) / "operations.sqlite3"
+        self.journal = Journal(self.path)
+        self.now = 2000000
+        self.sequence = 0
+
+    def tearDown(self):
+        self.journal.close()
+        self.temp.cleanup()
+
+    def record(self, kind="files.stat", payload=None, result=None, state="completed", error=None, at=None, confirmed=True):
+        self.sequence += 1
+        operation = {"id": f"operation-{self.sequence}", "kind": kind, "payload": payload or {}}
+        self.journal.accept(operation)
+        with patch.object(module.time, "time", return_value=self.now - 172800 if at is None else at):
+            row = self.journal.update(operation["id"], state, result, error)
+        if confirmed:
+            self.journal.confirm_completed(row)
+        return row
+
+    def prune(self, days=1):
+        with patch.object(module.time, "time", return_value=self.now):
+            return self.journal.prune_observations(days)
+
+    def test_cutoff_requires_an_older_confirmed_success(self):
+        old = self.record(at=self.now - 86401)
+        boundary = self.record(at=self.now - 86400)
+        recent = self.record(at=self.now - 10)
+        unconfirmed = self.record(confirmed=False)
+        self.assertEqual(self.prune(), 1)
+        with self.assertRaisesRegex(AgentError, "Unknown operation"):
+            self.journal.get(old["id"])
+        for row in (boundary, recent, unconfirmed):
+            self.assertEqual(self.journal.get(row["id"])["state"], "completed")
+
+    def test_zero_and_invalid_retention_preserve_every_record(self):
+        row = self.record()
+        for days in (0, -1, 36501, True, False, None, "1", 1.0, 1.5):
+            with self.subTest(days=days):
+                self.assertEqual(self.prune(days), 0)
+                self.assertEqual(self.journal.get(row["id"])["state"], "completed")
+
+    def test_all_read_kinds_are_eligible_but_mutations_keep_their_replay_journal(self):
+        for kind in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read"):
+            self.record(kind, result={"healthy": True})
+        for command in ("status", "getstatus", "version"):
+            self.record("rcon.execute", {"command": command}, {"ok": True})
+        mutations = [self.record(kind) for kind in ("instance.restart", "instance.stop", "instance.create", "files.stage", "files.activate", "credentials.rotate")]
+        mutations.append(self.record("rcon.execute", {"command": "map_restart"}, {"ok": True}))
+        self.assertEqual(self.prune(), 10)
+        for row in mutations:
+            saved, accepted = self.journal.accept(json.loads(row["request"]))
+            self.assertFalse(accepted)
+            self.assertEqual(saved["id"], row["id"])
+
+    def test_errors_failures_and_unfinished_operations_are_retained(self):
+        retained = [self.record(result=result) for result in ({"ok": False}, {"healthy": False}, {"error": "partial result"}, {"error": False}, {"error": []})]
+        retained.append(self.record(error="read did not finish"))
+        retained.extend(self.record(state=state) for state in ("accepted", "running", "unknown", "failed"))
+        self.assertEqual(self.prune(), 0)
+        for row in retained:
+            self.assertEqual(self.journal.get(row["id"])["state"], row["state"])
+
+    def test_old_receipt_cannot_confirm_a_changed_or_reexecuted_result(self):
+        old = self.record(confirmed=False)
+        with patch.object(module.time, "time", return_value=self.now - 90000):
+            current = self.journal.update(old["id"], "completed", {"new": "result"})
+        self.journal.confirm_completed(old)
+        self.assertIsNone(self.journal.get(old["id"])["reported_at"])
+        self.assertEqual(self.prune(), 0)
+        self.journal.confirm_completed(current)
+        self.assertEqual(self.prune(), 1)
+
+    def test_pruning_uses_bounded_indexed_batches(self):
+        template = self.record()
+        self.journal.db.executemany("""INSERT INTO operations(id,fingerprint,request,state,result,updated,reported_at,retention_eligible)
+            VALUES(?,?,?,'completed',NULL,?,?,1)""", [
+                (f"batch-{index}", template["fingerprint"], template["request"], self.now - 172800, self.now - 172800)
+                for index in range(504)
+            ])
+        self.journal.db.commit()
+        self.assertEqual(self.prune(), 500)
+        self.assertEqual(self.journal.db.execute("SELECT COUNT(*) FROM operations").fetchone()[0], 5)
+        self.assertEqual(self.prune(), 5)
+        plan = self.journal.db.execute("""EXPLAIN QUERY PLAN SELECT id FROM operations
+            WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1 AND updated<? ORDER BY updated,id LIMIT 500""", (self.now,)).fetchall()
+        self.assertIn("operations_observation_retention", " ".join(str(tuple(row)) for row in plan))
+
+    def test_migration_keeps_legacy_completions_until_their_receipt_is_reacknowledged(self):
+        self.journal.close()
+        self.path.unlink()
+        legacy = module.sqlite3.connect(self.path)
+        legacy.execute("CREATE TABLE operations(id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)")
+        request = {"id": "legacy-read", "kind": "files.list", "instanceId": None, "generation": None, "payload": {}}
+        canonical = json.dumps(request, sort_keys=True, separators=(",", ":"))
+        legacy.execute("INSERT INTO operations VALUES(?,?,?,'completed',?,NULL,?)", (request["id"], module.digest(canonical.encode()), canonical, '{"entries":[]}', self.now - 172800))
+        legacy.commit()
+        legacy.close()
+        self.journal = Journal(self.path)
+        row, accepted = self.journal.accept(request)
+        self.assertFalse(accepted)
+        self.assertIsNone(row["reported_at"])
+        self.assertEqual(self.prune(), 0)
+        self.journal.confirm_completed(row)
+        self.assertEqual(self.prune(), 1)
+
+
+class ManifestTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp.name) / "app"
+        self.root.mkdir()
+        self.stopping = threading.Event()
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def write(self, path, content=b"content"):
+        target = self.root / path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(content)
+        return target
+
+    def manifest(self, roots=None, ignore_names=None, paths=None):
+        return module.managed_manifest(self.root, {
+            "roots": ["scripts", "configs", "plugins"] if roots is None else roots,
+            "ignoreNames": [] if ignore_names is None else ignore_names,
+            "paths": [] if paths is None else paths,
+        }, self.stopping)
+
+    def test_hashes_sorted_unique_files_across_overlapping_roots_and_skips_ignored_names(self):
+        self.write("scripts/nested/z.gsc", b"script")
+        self.write("configs/server.cfg", b"config")
+        self.write("scripts/cache/ignored.gsc")
+        self.write("scripts/nested/ignore.gsc")
+        expected = [{"path": path, "size": len(content), "sha256": module.digest(content)} for path, content in (
+            ("configs/server.cfg", b"config"), ("scripts/nested/z.gsc", b"script"),
+        )]
+        self.assertEqual(self.manifest(["scripts/nested", "configs", "scripts", "scripts", "plugins"], ["cache", "ignore.gsc"]), {"files": expected})
+
+    def test_nested_symlinks_are_skipped_but_requested_symlink_roots_fail(self):
+        self.write("scripts/good.gsc")
+        outside = Path(self.temp.name) / "outside"
+        outside.mkdir()
+        (outside / "secret").write_bytes(b"secret")
+        (self.root / "scripts/file-link").symlink_to(outside / "secret")
+        (self.root / "scripts/dir-link").symlink_to(outside, target_is_directory=True)
+        (self.root / "scripts/dangling").symlink_to(outside / "missing")
+        self.assertEqual([item["path"] for item in self.manifest()["files"]], ["scripts/good.gsc"])
+        for roots in (["scripts/dir-link"], ["scripts", "scripts/dir-link"]):
+            with self.subTest(roots=roots), self.assertRaisesRegex(AgentError, "symlink|Symlink"):
+                self.manifest(roots)
+
+    def test_validates_roots_and_ignored_names(self):
+        for roots in ([], "scripts", [""], ["."], [".."], ["../scripts"], ["/scripts"], ["scripts//nested"], ["scripts/../configs"], [None], ["scripts"] * 65):
+            with self.subTest(roots=roots), self.assertRaises(AgentError):
+                self.manifest(roots)
+        for ignored in ("cache", [None], [""], ["."], [".."], ["scripts/cache"], ["a\\b"], ["a\x00b"], ["cache"] * 257):
+            with self.subTest(ignored=ignored), self.assertRaises(AgentError):
+                self.manifest(ignore_names=ignored)
+        for paths in ("scripts/a", [None], ["../outside"], ["."], ["scripts/a"] * 10001):
+            with self.subTest(paths=paths), self.assertRaises(AgentError):
+                self.manifest(paths=paths)
+
+    def test_explicit_managed_paths_cannot_be_symlinks_or_directories(self):
+        self.write("scripts/good.gsc")
+        (self.root / "scripts/link.gsc").symlink_to(self.root / "scripts/good.gsc")
+        (self.root / "scripts/dangling.gsc").symlink_to(self.root / "missing")
+        for path in ("scripts", "scripts/link.gsc", "scripts/dangling.gsc"):
+            with self.subTest(path=path), self.assertRaises(AgentError):
+                self.manifest(paths=[path])
+        self.assertEqual([item["path"] for item in self.manifest(paths=["scripts/good.gsc", "scripts/missing.gsc", "missing/future.gsc"])["files"]], ["scripts/good.gsc"])
+
+    def test_explicit_managed_paths_are_validated_again_after_the_scan(self):
+        self.write("scripts/good.gsc")
+        original = module.managed_open
+        opened = 0
+
+        @module.contextlib.contextmanager
+        def swap(root, value):
+            nonlocal opened
+            opened += 1
+            if opened == 2:
+                (self.root / "scripts/good.gsc").unlink()
+                (self.root / "scripts/good.gsc").symlink_to(self.root / "missing")
+            with original(root, value) as stream:
+                yield stream
+
+        with patch.object(module, "managed_open", side_effect=swap), self.assertRaisesRegex(AgentError, "Symlinks"):
+            self.manifest(paths=["scripts/good.gsc"])
+
+    def test_missing_destinations_are_empty_but_unavailable_app_and_regular_roots_fail(self):
+        self.assertEqual(self.manifest(), {"files": []})
+        self.write("scripts", b"not a directory")
+        with self.assertRaisesRegex(AgentError, "complete managed tree"):
+            self.manifest()
+        (self.root / "scripts").unlink()
+        self.root.rmdir()
+        with self.assertRaisesRegex(AgentError, "complete managed tree"):
+            self.manifest()
+        replacement = Path(self.temp.name) / "replacement"
+        replacement.mkdir()
+        self.root.symlink_to(replacement, target_is_directory=True)
+        with self.assertRaisesRegex(AgentError, "symlink"):
+            self.manifest()
+
+    def test_errors_and_special_files_never_return_a_partial_manifest(self):
+        self.write("scripts/good.gsc")
+        self.write("scripts/unreadable.gsc")
+        original = module.os.stat
+
+        def denied(path, *args, **kwargs):
+            if path == "unreadable.gsc":
+                raise PermissionError("unreadable")
+            return original(path, *args, **kwargs)
+
+        with patch.object(module.os, "stat", side_effect=denied), self.assertRaisesRegex(AgentError, "complete managed tree"):
+            self.manifest()
+        module.os.mkfifo(self.root / "scripts/fifo")
+        with self.assertRaisesRegex(AgentError, "non-regular"):
+            self.manifest()
+
+    def test_scan_limits_fail_instead_of_truncating_results(self):
+        self.write("scripts/a.gsc")
+        self.write("scripts/b.gsc")
+        for setting, limit, message in (
+            ("MANIFEST_MAX_ENTRIES", 1, "entry limit"),
+            ("MANIFEST_MAX_BYTES", 1, "result size limit"),
+            ("MANIFEST_MAX_SECONDS", 0, "time limit"),
+            ("MANIFEST_MAX_DEPTH", 0, "depth limit"),
+        ):
+            with self.subTest(setting=setting), patch.object(module, setting, limit), self.assertRaisesRegex(AgentError, message):
+                self.manifest()
+        self.stopping.set()
+        with self.assertRaisesRegex(AgentError, "interrupted"):
+            self.manifest()
+
+    def test_files_are_streamed_and_not_limited_by_the_content_read_api(self):
+        content = b"x" * (2 * 1024 * 1024 + 13)
+        self.write("plugins/large.dll", content)
+        original = module.os.fdopen
+        reads = []
+
+        @module.contextlib.contextmanager
+        def tracked(*args, **kwargs):
+            with original(*args, **kwargs) as stream:
+                proxy = MagicMock(wraps=stream)
+                def read(size):
+                    reads.append(size)
+                    return stream.read(size)
+                proxy.read.side_effect = read
+                yield proxy
+
+        with patch.object(module, "MAX_FILE", 1), patch.object(module.os, "fdopen", side_effect=tracked):
+            result = self.manifest()
+        self.assertEqual(result, {"files": [{"path": "plugins/large.dll", "size": len(content), "sha256": module.digest(content)}]})
+        self.assertGreater(len(reads), 2)
+        self.assertEqual(set(reads), {1024 * 1024})
+
+    def test_file_changes_while_hashing_invalidate_the_scan(self):
+        target = self.write("scripts/changing.gsc", b"before")
+        original = module.hashlib.sha256
+
+        def changing_hash():
+            digest = original()
+            proxy = MagicMock(wraps=digest)
+            def update(block):
+                digest.update(block)
+                prior = target.stat()
+                target.write_bytes(b"after!")
+                module.os.utime(target, ns=(prior.st_atime_ns, prior.st_mtime_ns + 1))
+            proxy.update.side_effect = update
+            return proxy
+
+        with patch.object(module.hashlib, "sha256", side_effect=changing_hash), self.assertRaisesRegex(AgentError, "file changed"):
+            self.manifest()
+
+    def test_symlink_swap_after_validation_cannot_hash_outside_the_app(self):
+        self.write("scripts/good.gsc")
+        outside = Path(self.temp.name) / "outside"
+        outside.mkdir()
+        (outside / "secret").write_bytes(b"secret")
+        original = module.scoped
+
+        def swap(root, relative, **kwargs):
+            result = original(root, relative, **kwargs)
+            (self.root / "scripts").rename(self.root / "saved")
+            (self.root / "scripts").symlink_to(outside, target_is_directory=True)
+            return result
+
+        with patch.object(module, "scoped", side_effect=swap), self.assertRaisesRegex(AgentError, "complete managed tree"):
+            self.manifest(["scripts"])
+
+    def test_disappearing_app_after_hashing_is_not_successful_evidence(self):
+        self.write("scripts/good.gsc")
+        original = module.hashlib.sha256
+
+        def moving_hash():
+            digest = original()
+            proxy = MagicMock(wraps=digest)
+            def update(block):
+                digest.update(block)
+                self.root.rename(self.root.with_name("moved-app"))
+            proxy.update.side_effect = update
+            return proxy
+
+        with patch.object(module.hashlib, "sha256", side_effect=moving_hash), self.assertRaisesRegex(AgentError, "complete managed tree"):
+            self.manifest()
+
+
 class AgentTests(unittest.TestCase):
     def setUp(self):
         self.temp = tempfile.TemporaryDirectory()
@@ -227,6 +539,20 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(second["result"], first["result"])
         self.assertEqual(sum(call[:2] == ("container", "restart") for call in self.docker.calls), 1)
 
+    def test_manifest_is_advertised_and_reconciles_as_a_read_operation(self):
+        self.create()
+        operation = self.operation("files.manifest", {"roots": ["configs", "scripts"], "ignoreNames": [], "paths": ["configs/server.cfg"]})
+        self.assertIn("files.manifest", self.agent.inventory()["capabilities"])
+        self.assertTrue(self.agent.read_operation(operation))
+        self.assertFalse(self.agent.health_operation(operation))
+        self.agent.journal.accept(operation)
+        self.agent.journal.update(operation["id"], "unknown")
+        calls = list(self.docker.calls)
+        result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed")
+        self.assertEqual(json.loads(result["result"]), {"files": [{"path": "configs/server.cfg", "size": 8, "sha256": module.digest(b"original")}]})
+        self.assertEqual(self.docker.calls, calls)
+
     def test_reusing_id_with_different_payload_is_rejected(self):
         self.agent.execute(self.operation("host.inspect"))
         with self.assertRaisesRegex(AgentError, "different content"):
@@ -1201,6 +1527,85 @@ class AgentTests(unittest.TestCase):
         self.agent.heartbeat()
         self.assertEqual(self.agent.config["portRange"], {"min": 30000, "max": 30100})
 
+    def test_heartbeat_requires_valid_retention_and_never_changes_the_journal(self):
+        self.assertEqual(self.agent.observation_keep_days, 0)
+        with patch.object(self.agent.journal, "prune_observations", side_effect=AssertionError("heartbeat must not use the journal")), patch.object(self.agent.journal, "confirm_completed", side_effect=AssertionError("heartbeat must not use the journal")):
+            for value, expected in ((1, 1), (36500, 36500), (0, 0), (True, 0), (False, 0), (1.0, 0), ("1", 0), (-1, 0), (36501, 0), (None, 0)):
+                with self.subTest(value=value):
+                    self.agent.observation_keep_days = 3
+                    self.agent.request = lambda *_, value=value: {"observationKeepDays": value}
+                    self.agent.heartbeat()
+                    self.assertEqual(self.agent.observation_keep_days, expected)
+            self.agent.observation_keep_days = 3
+            self.agent.request = lambda *_: {}
+            self.agent.heartbeat()
+            self.assertEqual(self.agent.observation_keep_days, 0)
+
+    def test_lost_completed_receipt_is_retained_and_replay_reports_the_saved_result(self):
+        operation = self.operation("files.stat", {"path": "configs/server.cfg"})
+        self.agent.journal.accept(operation)
+        with patch.object(module.time, "time", return_value=module.time.time() - 172800):
+            row = self.agent.journal.update(operation["id"], "completed", {"size": 8})
+        self.agent.request = MagicMock(side_effect=AgentError("receipt lost"))
+        with self.assertRaisesRegex(AgentError, "receipt lost"):
+            self.agent.report(row)
+        self.assertIsNone(self.agent.journal.get(operation["id"])["reported_at"])
+        self.assertEqual(self.agent.journal.prune_observations(1), 0)
+        with patch.object(self.agent, "dispatch", side_effect=AssertionError("completed read must not be executed again")):
+            replay = self.agent.execute(operation)
+        self.assertEqual(replay["result"], row["result"])
+        for response in ({}, None, {"operation": {"id": "different", "state": "completed"}}, {"operation": {"id": operation["id"], "state": "running"}}):
+            self.agent.request = lambda *_, response=response: response
+            self.agent.report(replay)
+            self.assertIsNone(self.agent.journal.get(operation["id"])["reported_at"])
+        self.agent.request = lambda *_: {"operation": {"id": operation["id"], "state": "completed"}}
+        self.agent.report(replay)
+        self.assertIsNotNone(self.agent.journal.get(operation["id"])["reported_at"])
+        self.assertEqual(self.agent.journal.prune_observations(1), 1)
+
+    def test_threaded_read_lane_confirms_and_prunes_through_its_own_connection(self):
+        self.create()
+        old_operation = self.operation("files.list", op_id="old-observation")
+        self.agent.journal.accept(old_operation)
+        with patch.object(module.time, "time", return_value=module.time.time() - 172800):
+            old = self.agent.journal.update(old_operation["id"], "completed", {"entries": []})
+        self.agent.journal.confirm_completed(old)
+        self.agent.observation_keep_days = 1
+        operation = self.operation("files.stat", {"path": "configs/server.cfg"}, "thread-observation")
+        polls = 0
+        failures = []
+
+        def request(method, path, payload=None):
+            nonlocal polls
+            if method == "POST":
+                return {"operation": {"id": operation["id"], "state": payload["state"]}}
+            polls += 1
+            if polls == 1:
+                return {"operations": [operation]}
+            self.agent.stopping.set()
+            return {"operations": []}
+
+        def run():
+            try:
+                self.agent.read_loop()
+            except BaseException as error:
+                failures.append(error)
+
+        self.agent.request = request
+        with patch.object(self.agent.journal, "confirm_completed", side_effect=AssertionError("main journal used by reader")), patch.object(self.agent.journal, "prune_observations", side_effect=AssertionError("main journal used by reader")):
+            thread = threading.Thread(target=run)
+            thread.start()
+            thread.join(timeout=5)
+            self.agent.stopping.set()
+            thread.join(timeout=5)
+        self.assertFalse(thread.is_alive())
+        self.assertEqual(failures, [])
+        self.assertEqual(polls, 2)
+        self.assertIsNotNone(self.agent.journal.get(operation["id"])["reported_at"])
+        with self.assertRaisesRegex(AgentError, "Unknown operation"):
+            self.agent.journal.get(old_operation["id"])
+        self.assertEqual(self.agent.journal.get("create")["state"], "completed")
+
     def test_download_content_permissions_allow_container_read_access_with_private_umask(self):
         content = b"complete-game-file"
         manifest = self.manifest(content)

docs/native-hosts.md

@@ -60,10 +60,20 @@ or enrolling another host needs no manual propagation. Setup waits for an active
 healthy host with sufficient storage. Failed operations retry automatically with
 backoff from one minute to fifteen minutes; their errors remain visible in the
 host's setup panel and **Hosts → Operation history**. This shared history lists
-operations across hosts, newest first, with a host column and 25 rows per page.
+operations across hosts, newest first, with 25 rows per page and host, target,
+and source details. It defaults to **Actions & errors**; **Routine checks** and
+**All operations** include successful monitoring reads. Older records may lack
+source details.
 Disconnected hosts resume when they
 reconnect. Draining and disabled hosts do not receive new automatic setup work.
 
+**Settings → Routine operation history retention (days)** controls cleanup of
+successful monitoring and file-read records. The default is one day; zero keeps
+them indefinitely. Actions, failures, negative checks, and unresolved operations
+are retained. Agent 1.4.9 also cleans up successful routine journal records after
+the controller confirms their final receipt. Older unconfirmed journal entries
+remain until a replay supplies that confirmation. Cleanup runs in bounded batches.
+
 An automatic download that has not been delivered to the agent can be replaced
 when a newer upload becomes available. A delivered operation finishes safely
 before the host prepares the latest files. Failed or incomplete uploads never
@@ -287,8 +297,11 @@ explicit installer `--reenroll` when local credentials already exist.
 API outline:
 
 - `GET /api/hosts/:id`: host details and deletion blockers.
-- `GET /api/hosts/operations?page=1`: operation history across hosts, newest first,
+- `GET /api/hosts/operations?page=1&filter=important`: operation history across
-  with 25 summaries per page and total/page counts. Payloads and results are omitted.
+  hosts, newest first, with 25 summaries per page and filtered total/page counts.
+  Filters are `important` (default), `routine`, and `all`. Summaries include the
+  server, safe target path or status command, and initiating source when recorded.
+  Payloads and results are omitted.
 - `DELETE /api/hosts/:id`: administrator requests cleanup of an unused host;
   returns `202` while remote removal is pending, `200` for an unenrolled host,
   or `409` when the agent needs updating or unresolved ownership remains.
@@ -551,8 +564,14 @@ of which task delayed a previous deployment.
 Campaign, beta and maintenance observers run independently of provisioning.
 New requests and completed agent commands wake provisioning; an arrival during a
 pass schedules another pass. Routine file drift scans run every five minutes;
-manual checks bypass that interval. Overlapping identical native observations
+manual checks bypass that interval. Agent 1.4.9 supports one hash manifest per
-share one operation until it completes, without caching results for later checks.
+drift scan, avoiding separate list/stat/read operations for unchanged files.
+Older agents use the existing file checks. Overlapping identical native
+observations share one operation until it completes. Routine status monitoring
+and maintenance additionally share successful health results for up to 30 seconds
+from the request start, preserving the original observation time. Changed server
+identity or lifecycle intent invalidates reuse. Manual checks and disruption
+safety checks continue to request fresh evidence.
 Agent 1.4.5 reserves two observation workers for instance health and read-only RCON,
 and one for files, logs, and host inventory. Background reads arrive in batches of
 up to four. File receipt uploads release the instance lock before contacting the

src/hooks.server.ts

@@ -5,6 +5,7 @@ import { interruptDeployments } from '$lib/server/provisioning/history.js';
 import { authenticationHandle } from '$lib/server/auth/guard.js';
 import { emit } from '$lib/server/pipeline/bus.js';
 import { startPerformanceMonitor } from '$lib/server/performance.js';
+import { withOperationSource } from '$lib/server/hosts/operation-context.js';
 
 let booted = false;
 function boot(): void {
@@ -22,7 +23,8 @@ function boot(): void {
 export const handle: Handle = async (input) => {
     boot();
     const started = performance.now();
-    const response = await authenticationHandle(input);
+    const response = await withOperationSource(`${input.event.request.method} ${input.event.route.id ?? 'Web request'}`,
+        () => authenticationHandle(input));
     const elapsed = performance.now() - started;
     response.headers.append('Server-Timing', `app;dur=${elapsed.toFixed(1)}`);
     if (process.env.EC_PERF_LOG === 'true' && elapsed > 250 && !input.event.route.id?.startsWith('/api/agent/')) {

src/lib/fleet-settings.ts

@@ -1,5 +1,6 @@
 export interface FleetSettings {
     deployKeepDays: number;
+    observationKeepDays: number;
     restartConcurrency: number;
     createConcurrency: number;
     destroyConcurrency: number;
@@ -7,6 +8,7 @@ export interface FleetSettings {
 
 export const defaultFleetSettings: FleetSettings = {
     deployKeepDays: 7,
+    observationKeepDays: 1,
     restartConcurrency: 1,
     createConcurrency: 1,
     destroyConcurrency: 1
@@ -14,6 +16,7 @@ export const defaultFleetSettings: FleetSettings = {
 
 export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
     deployKeepDays: 'Campaign history retention',
+    observationKeepDays: 'Routine operation history retention',
     restartConcurrency: 'Concurrent restarts',
     createConcurrency: 'Concurrent server startups',
     destroyConcurrency: 'Concurrent server destructions'
@@ -27,11 +30,14 @@ export function validateFleetSettings(input: unknown): FleetSettings {
     }
     for (const key of Object.keys(defaultFleetSettings) as (keyof FleetSettings)[]) {
         const value = values[key];
-        if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) {
+        const minimum = key === 'observationKeepDays' ? 0 : 1;
-            throw new Error(`${fleetSettingLabels[key]} must be a positive whole number.`);
+        if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) {
+            throw new Error(`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`);
+        }
+        if ((key === 'deployKeepDays' || key === 'observationKeepDays') && value > 36500) {
+            throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`);
         }
-        if (key === 'deployKeepDays' && value > 36500) throw new Error('Campaign history retention must be at most 36500 days.');
     }
-    return { deployKeepDays: values.deployKeepDays as number, restartConcurrency: values.restartConcurrency as number,
+    return { deployKeepDays: values.deployKeepDays as number, observationKeepDays: values.observationKeepDays as number, restartConcurrency: values.restartConcurrency as number,
         createConcurrency: values.createConcurrency as number, destroyConcurrency: values.destroyConcurrency as number };
 }

src/lib/hosts.ts

@@ -102,9 +102,17 @@ export interface HostOperation {
 }
 export interface HostOperationSummary extends Omit<HostOperation, 'payload' | 'result'> {
     hostName: string | null;
+    serverName: string | null;
+    targetPath: string | null;
+    source: string | null;
+    routine: boolean;
+}
+export type HostOperationFilter = 'important' | 'routine' | 'all';
+export function isHostOperationFilter(value: unknown): value is HostOperationFilter {
+    return typeof value === 'string' && ['important', 'routine', 'all'].includes(value);
 }
 export interface HostOperationHistoryPage {
-    operations: HostOperationSummary[]; page: number; pageSize: 25; total: number; totalPages: number;
+    operations: HostOperationSummary[]; page: number; pageSize: 25; total: number; totalPages: number; filter: HostOperationFilter;
 }
 
 export interface HostPreparationItem {

src/lib/server/backends/files.ts

@@ -1,7 +1,8 @@
 import * as fs from 'node:fs/promises';
-import { relative, resolve, sep } from 'node:path';
+import { join, posix, relative, resolve, sep } from 'node:path';
 import { config } from '../config.js';
 import { nativeInstance, nativeRequest } from './native.js';
+import { getHost } from '../hosts/index.js';
 
 function remote(path: string) {
     for (const root of [config.serversRoot, config.t6ServersRoot]) {
@@ -16,6 +17,35 @@ function remote(path: string) {
 }
 function missing(path: string): never { throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' }); }
 
+export interface FileFingerprint { size: number; sha256: string }
+
+/** One scoped agent scan; null preserves compatibility with older agents and local files. */
+export async function nativeFileManifest(root: string, directories: string[], ignoreNames: string[], managedPaths: string[]): Promise<Map<string, FileFingerprint> | null> {
+    const target = remote(root);
+    if (!target || target.path !== '.' || !getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest')) return null;
+    const roots = [...new Set(directories.map(directory => relative(root, directory).split(sep).join('/')))];
+    // Preserve legacy configurations that scan the entire instance root.
+    if (roots.some(directory => !directory)) return null;
+    const paths = [...new Set(managedPaths.map(path => relative(root, path).split(sep).join('/')))];
+    const result = await nativeRequest<{ files: Array<FileFingerprint & { path: string }> }>(target.instance, 'files.manifest', {
+        roots, paths, ignoreNames: ignoreNames.filter(name => !paths.some(path => path.split('/').includes(name)))
+    });
+    if (!result || !Array.isArray(result.files) || result.files.length > 10_000) throw new Error('Agent returned an invalid file manifest.');
+    const manifest = new Map<string, FileFingerprint>();
+    for (const entry of result.files) {
+        if (!entry || typeof entry.path !== 'string' || !entry.path || entry.path.includes('\\') || entry.path.includes('\0')
+            || posix.normalize(entry.path) !== entry.path || entry.path.split('/').some(part => part === '..' || part === '.')
+            || !roots.some(directory => entry.path.startsWith(`${directory}/`))
+            || !Number.isSafeInteger(entry.size) || entry.size < 0 || !/^[a-f0-9]{64}$/.test(entry.sha256)) {
+            throw new Error('Agent returned an invalid file manifest entry.');
+        }
+        const path = join(root, entry.path);
+        if (manifest.has(path)) throw new Error('Agent returned duplicate file manifest entries.');
+        manifest.set(path, { size: entry.size, sha256: entry.sha256 });
+    }
+    return manifest;
+}
+
 export async function readFile(path: string): Promise<Buffer> {
     const target = remote(path);
     if (!target) return fs.readFile(path);

src/lib/server/backends/native.ts

@@ -31,7 +31,7 @@ export function unplacedNativeCreation(server: string): string | null {
 }
 
 const observations = new Map<string, Promise<unknown>>();
-const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'logs.read']);
+const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'files.manifest', 'logs.read']);
 
 /** Share overlapping observations only, never completed results or mutations. */
 export async function nativeRequest<T = Record<string, unknown>>(instance: NativeInstance, kind: string, payload: Record<string, unknown> = {}, key?: string): Promise<T> {

src/lib/server/db/fleet-settings.ts

@@ -8,8 +8,9 @@ export function initializeFleetSettings(handle: Database.Database): void {
     const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
     if (saved) {
         const settings = JSON.parse(saved.value);
-        if (!Object.hasOwn(settings, 'destroyConcurrency')) {
+        if (!Object.hasOwn(settings, 'destroyConcurrency') || !Object.hasOwn(settings, 'observationKeepDays')) {
-            const upgraded = validateFleetSettings({ ...settings, destroyConcurrency: defaultFleetSettings.destroyConcurrency });
+            const upgraded = validateFleetSettings({ destroyConcurrency: defaultFleetSettings.destroyConcurrency,
+                observationKeepDays: defaultFleetSettings.observationKeepDays, ...settings });
             handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
         }
         return;

src/lib/server/fleet/drift.ts

@@ -1,4 +1,4 @@
-import { readdir, readFile, stat } from '../backends/files.js';
+import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js';
 import { randomUUID } from 'node:crypto';
 import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
 import { join, posix, relative, sep } from 'node:path';
@@ -11,7 +11,7 @@ import { getBaselineForScope, getVersion } from './scripts.js';
 import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
 import { selectedZombieMap } from './map-aliases.js';
 import { getServer, listServers } from './servers.js';
-import { promotedArtifacts } from './promoted-artifacts.js';
+import { promotedArtifactMetadata } from './promoted-artifacts.js';
 import { queueFileChange } from './staged-files.js';
 import { trackActivity } from '../pipeline/activity.js';
 
@@ -88,6 +88,12 @@ async function readExistingFile(path: string): Promise<Buffer | null> {
     });
 }
 
+async function fingerprint(path: string, manifest: Map<string, FileFingerprint> | null): Promise<FileFingerprint | null> {
+    if (manifest) return manifest.get(path) ?? null;
+    const content = await readExistingFile(path);
+    return content === null ? null : { size: content.byteLength, sha256: sha256Buffer(content) };
+}
+
 async function walkFiles(root: string): Promise<string[]> {
     const out: string[] = [];
     let directories = [root];
@@ -128,11 +134,15 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
     const srv = getServer(server);
     if (!srv || srv.destroyed_at || hasActiveBeta(server)) return [];
     // A missing mount is unavailable evidence, not an empty managed tree.
-    await requireServerDirectory(server);
+    const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) => join(destOfKind(server, kind), script.target_rel_path));
-    const drift = await scanManagedForServer(server);
+    if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id) expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server)));
+    if (srv.created_by && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE));
+    const manifest = await nativeFileManifest(serverDir(server), DRIFT_DESTS.map(kind => destOfKind(server, kind)), [...DRIFT_IGNORE_FILENAMES], expectedPaths);
+    if (!manifest) await requireServerDirectory(server);
+    const drift = await scanManagedForServer(server, manifest);
     // Previous inventory is an audit of installed bytes, not permission to
     // retain artifacts whose scope was moved or removed from the library.
-    const managed = new Set(promotedArtifacts(srv.game, srv.map)
+    const managed = new Set(promotedArtifactMetadata(srv.game, srv.map)
         .map(({ kind, script }) => `${kind}:${script.target_rel_path}`));
     const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined;
     if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`);
@@ -145,7 +155,10 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
     // Scan for untracked files only after resolving managed targets.
     for (const kind of DRIFT_DESTS) {
         const dest = destOfKind(server, kind);
-        const onDisk = await walkFiles(dest);
+        const onDisk = manifest
+            ? [...manifest.keys()].filter(path => path.startsWith(`${dest}${sep}`)).map(path => toPosix(relative(dest, path)))
+                .filter(path => !path.split('/').some(name => DRIFT_IGNORE_FILENAMES.has(name))).sort()
+            : await walkFiles(dest);
         if (onDisk.length === 0) continue;
 
         const untracked = await scanBatch(onDisk, async (relPath): Promise<DriftEntry | null> => {
@@ -155,11 +168,11 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
                 && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) return null;
             const abs = join(dest, relPath);
             try {
-                const buf = await readFile(abs);
+                const info = await fingerprint(abs, manifest);
+                if (!info) return null;
                 return {
                     server, relPath,
-                    size: buf.byteLength,
+                    ...info,
-                    sha256: sha256Buffer(buf),
                     kind,
                     driftKind: 'untracked'
                 };
@@ -172,14 +185,14 @@ async function scanServerFiles(server: string): Promise<DriftEntry[]> {
         drift.push(...untracked.filter((entry): entry is DriftEntry => entry !== null));
     }
 
-    await requireServerDirectory(server);
+    if (!manifest) await requireServerDirectory(server);
     return hasActiveBeta(server) ? [] : drift;
 }
 
 // Every promoted artifact whose scope matches this server's game/map should
 // have its target file present on disk. Anything absent is 'missing' drift.
 // Configs are handled as a composed layer (baseline + overlays → one cfg).
-async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
+async function scanManagedForServer(server: string, manifest: Map<string, FileFingerprint> | null): Promise<DriftEntry[]> {
     const srv = getServer(server);
     if (!srv) return [];
     const out: DriftEntry[] = [];
@@ -190,8 +203,8 @@ async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
         ? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null;
     if (mapConfig) {
         const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE);
-        const content = await readExistingFile(abs);
+        const content = await fingerprint(abs, manifest);
-        if (!content?.equals(mapConfig)) {
+        if (content?.sha256 !== sha256Buffer(mapConfig)) {
             out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing',
                 size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) });
         }
@@ -199,10 +212,10 @@ async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
 
     // Inventory records the last managed write, not the current disk bytes.
     // Check both so edits and outdated versions are detected on tracked files.
-    const managed = await scanBatch(promotedArtifacts(srv.game, srv.map), async ({ kind, script, version }): Promise<DriftEntry | null> => {
+    const managed = await scanBatch(promotedArtifactMetadata(srv.game, srv.map), async ({ kind, script, version }): Promise<DriftEntry | null> => {
-        const content = await readExistingFile(join(destOfKind(server, kind), script.target_rel_path));
+        const content = await fingerprint(join(destOfKind(server, kind), script.target_rel_path), manifest);
         const inventory = getServerScript(server, script.target_rel_path);
-        if (content?.equals(version.content) && inventory?.sha256 === version.sha256
+        if (content?.sha256 === version.sha256 && inventory?.sha256 === version.sha256
             && inventory.script_version_id === version.id) return null;
         return {
             server, relPath: script.target_rel_path, kind,
@@ -226,8 +239,8 @@ async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
                 const targetRel = configFileOf(server);
                 const abs = join(dest, targetRel);
                 const inv = getServerScript(server, targetRel);
-                const content = await readExistingFile(abs);
+                const content = await fingerprint(abs, manifest);
-                const onDiskMatches = content?.equals(composed.content) && inv?.sha256 === composed.sha256;
+                const onDiskMatches = content?.sha256 === composed.sha256 && inv?.sha256 === composed.sha256;
                 if (!onDiskMatches) {
                     out.push({
                         server,
@@ -366,7 +379,7 @@ export async function prepareDriftCampaign(server: string): Promise<PreparedDrif
             beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
             beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } };
     });
-    await requireServerDirectory(server);
+    if (entries.length) await requireServerDirectory(server);
     if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
     return { plan: buildPlan(servers, files.map(({ file }) => file)), files };
 }

src/lib/server/fleet/health-observations.ts

@@ -0,0 +1,108 @@
+import { config } from '../config.js';
+import { db } from '../db/index.js';
+import { getHost } from '../hosts/index.js';
+import { nativeInstance } from '../backends/native.js';
+import { komodoInspectHealthy } from '../integrations/komodo.js';
+import { rconStatusResult } from '../integrations/rcon.js';
+import { getServer } from './servers.js';
+import { latestGameSubmission } from './lifecycle-evidence.js';
+
+const MAX_AGE_MS = 30_000;
+const MAX_SERVERS = 1024;
+type Probe = 'inspect' | 'rcon';
+interface Observation<T> { result: T; observedAt: string }
+interface Identity { key: string; port: number | null; checkedAt: string | null }
+interface Entry<T> {
+    identity: Identity;
+    startedAt: number;
+    pending?: Promise<Observation<T>>;
+    observation?: Observation<T>;
+}
+
+export class MonitoringChangedError extends Error {}
+
+function identity(server: string, probe: Probe): Identity {
+    const row = getServer(server);
+    if (!row || row.destroyed_at) throw new MonitoringChangedError('Server is no longer registered; monitoring evidence is unavailable.');
+    const native = row.native_instance_id ? nativeInstance(server) : null;
+    if (row.native_instance_id && (!native || getHost(native.host_id)?.connectionState !== 'online')) {
+        throw new Error('Host agent is offline; monitoring evidence is unavailable.');
+    }
+    // Stop requests also invalidate observations, even though they do not
+    // establish a new game-telemetry epoch in latestGameSubmission().
+    const lifecycle = db().prepare(`SELECT id, action, submitted_at FROM server_deployments
+        WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) ?? null;
+    return {
+        key: JSON.stringify([row.created_at, row.desired_state, latestGameSubmission(server), lifecycle,
+            native ? [native.host_id, native.id, native.generation] : null,
+            native ? null : [row.komodo_type, row.komodo_name, row.komodo_server, config.komodo, config.httpTimeoutMs],
+            probe === 'rcon' ? [row.rcon_host, config.rcon] : null]),
+        port: probe === 'rcon' ? native?.port ?? row.game_port : null,
+        checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at
+    };
+}
+
+function sameIdentity(left: Identity, right: Identity): boolean {
+    return left.key === right.key && left.port === right.port;
+}
+function newerEvidence(current: Identity, startedAt: number): boolean {
+    return Date.parse(current.checkedAt ?? '') > startedAt;
+}
+function reusable(startedAt: number): boolean {
+    const age = Date.now() - startedAt;
+    return age >= 0 && age < MAX_AGE_MS;
+}
+
+async function observe<T extends { ok: boolean }>(server: string, fresh: boolean, probe: Probe,
+    entries: Map<string, Entry<T>>, request: (server: string) => Promise<T>): Promise<Observation<T>> {
+    let current: Identity;
+    try { current = identity(server, probe); }
+    catch (error) { entries.delete(server); throw error; }
+    const saved = entries.get(server);
+    const savedAt = saved?.observation ? Date.parse(saved.observation.observedAt) : saved?.startedAt ?? 0;
+    if (!fresh && saved && sameIdentity(saved.identity, current) && !newerEvidence(current, savedAt)) {
+        if (saved.pending) return saved.pending;
+        if (saved.observation && reusable(saved.startedAt)) return saved.observation;
+    }
+
+    // Replace the previous slot before dispatch, including for forced probes.
+    // A forced failure must not leave an older successful result reusable.
+    const entry: Entry<T> = { identity: current, startedAt: Date.now() };
+    entries.delete(server);
+    entries.set(server, entry);
+    while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!);
+    const pending = Promise.resolve().then(() => request(server)).then(result => {
+        const latest = identity(server, probe);
+        // Legacy RCON may discover and save its first port during this request.
+        // Accept that enrichment only when all other identity fields agree.
+        const endpoint = 'endpoint' in result ? result.endpoint : null;
+        const learnedPort = probe === 'rcon' && !current.port && result.ok
+            && endpoint !== null && typeof endpoint === 'object' && 'port' in endpoint && endpoint.port === latest.port;
+        if (current.key !== latest.key || (current.port !== latest.port && !learnedPort)
+            || newerEvidence(latest, entry.startedAt)) {
+            throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.');
+        }
+        const observation = { result, observedAt: new Date().toISOString() };
+        if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) {
+            entry.identity = latest;
+            entry.observation = observation;
+        }
+        return observation;
+    }).finally(() => {
+        entry.pending = undefined;
+        if (entries.get(server) === entry && !entry.observation) entries.delete(server);
+    });
+    entry.pending = pending;
+    return pending;
+}
+
+const inspections = new Map<string, Entry<Awaited<ReturnType<typeof komodoInspectHealthy>>>>();
+const rcon = new Map<string, Entry<Awaited<ReturnType<typeof rconStatusResult>>>>();
+
+/** Routine monitoring only. Manual diagnostics and disruption gates use fresh probes directly. */
+export function inspectForMonitoring(server: string, fresh = false) {
+    return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy);
+}
+export function rconForMonitoring(server: string, fresh = false) {
+    return observe(server, fresh, 'rcon', rcon, rconStatusResult);
+}

src/lib/server/hosts/index.ts

@@ -4,9 +4,12 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt
 import { isIP } from 'node:net';
 import { db } from '../db/index.js';
 import { ensureHostSchema } from './schema.js';
+import { successfulObservation, routineObservationSql, importantOperationSql, operationTargetSql, safeOperationTarget } from './observations.js';
+import { currentOperationSource } from './operation-context.js';
+import { fleetSettings } from '../settings.js';
 import { recordInstanceResources, validateInstanceResources } from './calibration.js';
 import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
-import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationHistoryPage, HostOperationState } from '../../hosts.js';
+import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationFilter, HostOperationHistoryPage, HostOperationState } from '../../hosts.js';
 
 export const AGENT_PROTOCOL_VERSION = 1;
 export const HEARTBEAT_MAX_AGE_MS = 90_000;
@@ -131,7 +134,8 @@ export function deleteHost(id: string): { deleted: boolean; operationId?: string
         const operationId = randomUUID();
         const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
         const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` };
-        handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp);
+        handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,source) VALUES(?,?,'host.decommission',?,'pending',?,?,?)")
+            .run(operationId, id, JSON.stringify(payload), timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200));
         handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp);
         handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
             ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
@@ -357,19 +361,44 @@ export function listHostOperations(hostId: string, limit = 100, summary = false)
         .all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow);
 }
 /** Browse retained history without loading command payloads or credential-bearing results. */
-export function listHostOperationHistory(page = 1): HostOperationHistoryPage {
+export function listHostOperationHistory(page = 1, filter: HostOperationFilter = 'important'): HostOperationHistoryPage {
     const handle = database();
     const pageSize = 25;
+    const where = filter === 'all' ? '' : `WHERE ${filter === 'routine' ? 'o.routine=1' : importantOperationSql}`;
     return handle.transaction((): HostOperationHistoryPage => {
-        const { total } = handle.prepare('SELECT COUNT(*) AS total FROM ec_host_operations').get() as { total: number };
+        const { total } = handle.prepare(`SELECT COUNT(*) AS total FROM ec_host_operations o ${where}`).get() as { total: number };
         const totalPages = Math.max(1, Math.ceil(total / pageSize));
         const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1);
         const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind,
-            o.instance_id AS instanceId,o.generation,o.state,o.error,o.created_at AS createdAt,o.updated_at AS updatedAt
+            o.instance_id AS instanceId,o.generation,o.state,
+            CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0
+                THEN COALESCE(NULLIF(o.error,''),'Observation reported a problem') ELSE o.error END AS error,
+            o.created_at AS createdAt,o.updated_at AS updatedAt,o.routine,o.server_name AS serverName,o.target_path AS targetPath,o.source
             FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL
+            ${where}
             ORDER BY o.created_at DESC,o.rowid DESC LIMIT ? OFFSET ?`)
             .all(pageSize, (currentPage - 1) * pageSize) as HostOperationHistoryPage['operations'];
-        return { operations, page: currentPage, pageSize, total, totalPages };
+        return { operations: operations.map(operation => ({ ...operation, routine: !!operation.routine,
+            targetPath: safeOperationTarget(operation.kind, operation.targetPath) })), filter,
+            page: currentPage, pageSize, total, totalPages };
+    })();
+}
+
+/** Bound each cleanup pass so retained history cannot stall monitoring or receipts. */
+export function pruneHostObservations(): number {
+    const days = fleetSettings().observationKeepDays;
+    if (!Number.isSafeInteger(days) || days <= 0 || days > 36500) return 0;
+    const handle = database();
+    const cutoff = new Date(Date.now() - days * 86_400_000).toISOString();
+    return handle.transaction(() => {
+        const rows = handle.prepare(`SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed'
+            AND updated_at<? ORDER BY updated_at LIMIT 500`).all(cutoff) as { id: string }[];
+        if (!rows.length) return 0;
+        const ids = rows.map(row => row.id);
+        const placeholders = ids.map(() => '?').join(',');
+        handle.prepare(`DELETE FROM ec_host_operation_events WHERE operation_id IN (${placeholders})`).run(...ids);
+        handle.prepare(`DELETE FROM ec_host_operation_payloads WHERE operation_id IN (${placeholders})`).run(...ids);
+        return handle.prepare(`DELETE FROM ec_host_operations WHERE id IN (${placeholders})`).run(...ids).changes;
     })();
 }
 /** Heartbeats can tell an executing create to stop at its next safe command boundary. */
@@ -408,8 +437,15 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
         if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
         if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
         const id = options.id ?? randomUUID(); const timestamp = now();
-        database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, routingPayload ?? payload, timestamp, timestamp);
+        const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?')
+            .get(options.instanceId) as { server: string } | undefined)?.server ?? null : null;
+        database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name)
+            VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?)`)
+            .run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null,
+                routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName);
         if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload);
+        database().prepare(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
+            target_path=${operationTargetSql} WHERE o.id=?`).run(id);
         database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
         return getHostOperationStatus(id)!;
     })();
@@ -435,7 +471,7 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
         WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
     // Accepted/running/unknown commands keep the same immutable ID. The agent
     // consults its durable journal before acknowledging or executing a replay.
-    const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
+    const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','files.manifest','logs.read')
         OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
     const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
     const healthLane = host.inventory.capabilities?.includes('operations.health-lane');
@@ -479,7 +515,10 @@ export function reportHostOperationStatus(hostId: string, id: string, input: { s
         const error = input.error === undefined || input.error === null ? null : String(input.error).slice(0, 4000);
         const result = input.result === undefined ? null : JSON.stringify(input.result);
         if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
-        database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id);
+        const observation = state === 'completed' && !error && successfulObservation(operation.kind,
+            operation.kind === 'rcon.execute' ? JSON.parse(getHostOperationPayload(hostId, id).toString('utf8')) : {}, input.result);
+        database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=?,observation_success=? WHERE id=?')
+            .run(state, result, error, now(), Number(observation), id);
         if (operation.kind === 'host.decommission') {
             if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id);
             else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id);

src/lib/server/hosts/observations.ts

@@ -0,0 +1,38 @@
+export const observationKinds = ['host.inspect', 'instance.inspect', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read'];
+const commands = ['status', 'getstatus', 'version'];
+
+export function safeOperationTarget(kind: string, value: string | null): string | null {
+    if (!value || value.length > 1024) return null;
+    const paths = kind === 'files.manifest' ? value.split(', ') : [value];
+    return paths.every(path => path === '.' || !/[\\\x00-\x1f\x7f]/.test(path)
+        && path.split('/').every(part => part !== '' && part !== '.' && part !== '..')) ? value : null;
+}
+
+export function successfulObservation(kind: string, payload: Record<string, unknown>, result: unknown): boolean {
+    const routine = observationKinds.includes(kind) || kind === 'rcon.execute' && commands.includes(String(payload.command));
+    const value = result && typeof result === 'object' ? result as Record<string, unknown> : {};
+    return routine && value.ok !== false && value.healthy !== false && (value.error == null || value.error === '');
+}
+
+/** Uses the operation alias o; serialized payloads may live outside the routing row. */
+export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')}) THEN 1
+    WHEN o.kind='rcon.execute' THEN json_extract(COALESCE(
+        (SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version')
+    ELSE 0 END)`;
+
+export const successfulObservationSql = `(o.state='completed' AND COALESCE(o.error,'')='' AND ${routineObservationSql}
+    AND CASE WHEN o.result IS NULL THEN 1 WHEN json_valid(o.result)
+        THEN json_extract(o.result,'$.ok') IS NOT 0 AND json_extract(o.result,'$.healthy') IS NOT 0
+            AND COALESCE(json_extract(o.result,'$.error'),'')='' ELSE 0 END)`;
+
+export const importantOperationSql = `(o.routine=0 OR o.state IN ('failed','unknown')
+    OR (o.state='completed' AND o.observation_success=0) OR COALESCE(o.error,'')!='')`;
+
+const payloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
+/** Deliberately exclude arbitrary RCON commands and all file contents/credentials. */
+export const operationTargetSql = `CASE
+    WHEN o.kind IN ('files.read','files.stat','files.list','logs.read') THEN
+        CASE WHEN json_type(${payloadSql},'$.path')='text' THEN substr(json_extract(${payloadSql},'$.path'),1,1024) END
+    WHEN o.kind='files.manifest' THEN substr((SELECT group_concat(value,', ') FROM json_each(${payloadSql},'$.roots') WHERE type='text'),1,1024)
+    WHEN o.kind='rcon.execute' AND ${routineObservationSql} THEN json_extract(${payloadSql},'$.command')
+    ELSE NULL END`;

src/lib/server/hosts/operation-context.ts

@@ -0,0 +1,12 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+const operationSource = new AsyncLocalStorage<string>();
+
+/** Carry the initiating request or background task through asynchronous work. */
+export function withOperationSource<T>(source: string, work: () => T): T {
+    return operationSource.run(source, work);
+}
+
+export function currentOperationSource(): string | null {
+    return operationSource.getStore() ?? null;
+}

src/lib/server/hosts/schema.ts

@@ -1,4 +1,5 @@
 import type Database from 'better-sqlite3';
+import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
 
 const initialized = new WeakSet<Database.Database>();
 export function ensureHostSchema(handle: Database.Database): void {
@@ -28,6 +29,8 @@ export function ensureHostSchema(handle: Database.Database): void {
             payload TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'pending'
                 CHECK (state IN ('pending','accepted','running','completed','failed','unknown')),
             result TEXT, error TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, last_delivered_at TEXT,
+            observation_success INTEGER NOT NULL DEFAULT 0,
+            routine INTEGER NOT NULL DEFAULT 0, server_name TEXT, target_path TEXT, source TEXT,
             UNIQUE(host_id, idempotency_key)
         );
         CREATE INDEX IF NOT EXISTS ec_host_operations_queue ON ec_host_operations(host_id, state, created_at);
@@ -45,6 +48,7 @@ export function ensureHostSchema(handle: Database.Database): void {
             id INTEGER PRIMARY KEY AUTOINCREMENT, operation_id TEXT NOT NULL REFERENCES ec_host_operations(id),
             state TEXT NOT NULL, created_at TEXT NOT NULL, error TEXT
         );
+        CREATE INDEX IF NOT EXISTS ec_host_operation_events_operation ON ec_host_operation_events(operation_id);
         CREATE TABLE IF NOT EXISTS ec_host_ports (
             host_id TEXT NOT NULL REFERENCES ec_hosts(id), instance_id TEXT NOT NULL,
             bind_address TEXT NOT NULL, protocol TEXT NOT NULL CHECK (protocol IN ('udp','tcp')),
@@ -82,6 +86,29 @@ export function ensureHostSchema(handle: Database.Database): void {
     `);
     const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
     if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
+    if (!columns.some(column => column.name === 'observation_success')) {
+        handle.transaction(() => {
+            handle.exec('ALTER TABLE ec_host_operations ADD COLUMN observation_success INTEGER NOT NULL DEFAULT 0');
+            handle.exec(`UPDATE ec_host_operations AS o SET observation_success=1 WHERE ${successfulObservationSql}`);
+        })();
+    }
+    handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_observation_retention ON ec_host_operations(updated_at)
+        WHERE observation_success=1 AND state='completed'`);
+    if (!columns.some(column => column.name === 'routine')) {
+        handle.transaction(() => {
+            handle.exec(`ALTER TABLE ec_host_operations ADD COLUMN routine INTEGER NOT NULL DEFAULT 0;
+                ALTER TABLE ec_host_operations ADD COLUMN server_name TEXT;
+                ALTER TABLE ec_host_operations ADD COLUMN target_path TEXT;
+                ALTER TABLE ec_host_operations ADD COLUMN source TEXT;`);
+            const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as { name: string }[];
+            const nativeInstances = ['id', 'server'].every(name => nativeColumns.some(column => column.name === name));
+            handle.exec(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
+                target_path=${operationTargetSql}, server_name=${nativeInstances ? '(SELECT n.server FROM native_instances n WHERE n.id=o.instance_id)' : 'NULL'}`);
+        })();
+    }
+    handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_routine_history ON ec_host_operations(routine,created_at);
+        CREATE INDEX IF NOT EXISTS ec_host_operations_important_history ON ec_host_operations(created_at)
+            WHERE ${importantOperationSql.replaceAll('o.', '')}`);
     const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
     if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
     const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[];

src/lib/server/maintenance/recovery.ts

@@ -4,13 +4,13 @@ import { komodoConfigured } from '../config.js';
 import { nativeInstance } from '../backends/native.js';
 import { getServer, listServers } from '../fleet/servers.js';
 import { hasActiveProvisioning } from '../fleet/provisioning.js';
-import { komodoInspectHealthy } from '../integrations/komodo.js';
+import { inspectForMonitoring, rconForMonitoring, MonitoringChangedError } from '../fleet/health-observations.js';
-import { rconStatusResult } from '../integrations/rcon.js';
 import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js';
 import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
 import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
 import { deploymentStage, finishDeployment } from '../provisioning/history.js';
 import { recordProgress } from '../provisioning/progress.js';
+import { withOperationSource } from '../hosts/operation-context.js';
 import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js';
 
 import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
@@ -223,17 +223,27 @@ async function reconcileActions(): Promise<void> {
 }
 
 interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; players?: number; source?: 'rcon' | 'telemetry' }
-async function observe(server: string, operationId?: string): Promise<Observation | null> {
+async function observe(server: string, operationId?: string, fresh = false): Promise<Observation | null> {
     const epoch = latestGameSubmission(server);
     const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server))
         || protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server);
-    const health = await komodoInspectHealthy(server);
+    const inspected = await inspectForMonitoring(server, fresh || !!operationId).catch(error => {
+        if (error instanceof MonitoringChangedError) return null;
+        throw error;
+    });
+    if (!inspected) return null;
+    const health = inspected.result;
     if (suppressed()) return null;
     const state = health.state?.toLowerCase() ?? null;
     if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
     const telemetry = telemetryEvidence(server);
     if (telemetry) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
-    const rcon = await rconStatusResult(server);
+    const probed = await rconForMonitoring(server, fresh || !!operationId).catch(error => {
+        if (error instanceof MonitoringChangedError) return null;
+        throw error;
+    });
+    if (!probed) return null;
+    const rcon = probed.result;
     if (suppressed()) return null;
     const latest = telemetryEvidence(server);
     if (rcon.ok || latest) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state,
@@ -245,7 +255,7 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
     const row = getServer(server);
     if (!row || row.desired_state === 'stopped') return;
     if (scanSuppressed(server) || protectedWork(server)) return;
-    const observation = await observe(server);
+    const observation = await observe(server, undefined, forceScan);
     // Probes can overlap an API request; recheck ownership before queuing anything.
     if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
     if (!scanSuppressed(server, 'health') && observation.healthy) {
@@ -300,7 +310,8 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
 let inFlight: Promise<void> | null = null;
 export function watchdogTick(forceScan = false): Promise<void> {
     if (inFlight) return inFlight;
-    inFlight = run(forceScan).finally(() => { inFlight = null; });
+    inFlight = withOperationSource(forceScan ? 'Manual maintenance check' : 'Maintenance watchdog', () => run(forceScan))
+        .finally(() => { inFlight = null; });
     return inFlight;
 }
 async function run(forceScan: boolean): Promise<void> {

src/lib/server/pipeline/activity.ts

@@ -1,4 +1,5 @@
 import type { WorkerActivity } from '../../provisioning-detail.js';
+import { currentOperationSource, withOperationSource } from '../hosts/operation-context.js';
 
 const active = new Map<string, WorkerActivity>();
 let sequence = 0;
@@ -11,6 +12,6 @@ export function workerActivity(): WorkerActivity[] {
 export async function trackActivity<T>(key: string, label: string, work: () => Promise<T>): Promise<T> {
     if (active.has(key)) key = `${key}:${++sequence}`;
     active.set(key, { key, label, startedAt: new Date().toISOString() });
-    try { return await work(); }
+    try { return await withOperationSource(currentOperationSource() ?? label, work); }
     finally { active.delete(key); }
 }

src/lib/server/pipeline/sweep.ts

@@ -2,6 +2,7 @@ import { fleetSettings } from '../settings.js';
 import { db } from '../db/index.js';
 import { emit } from './bus.js';
 import { requestRestart } from '../provisioning/index.js';
+import { pruneHostObservations } from '../hosts/index.js';
 
 interface PendingRow {
     deployment_id: string; server: string; restart_state: string;
@@ -18,6 +19,7 @@ export function sweep(): Promise<SweepResult> {
     return sweepInFlight;
 }
 async function runSweep(): Promise<SweepResult> {
+    pruneHostObservations();
     const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
         WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
     const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };

src/lib/server/pipeline/worker.ts

@@ -4,10 +4,9 @@ import { sweep } from './sweep.js';
 import { betaTick } from './beta-runner.js';
 import { emit } from './bus.js';
 import type { ServerStatusEntry } from './bus.js';
-import { rconStatusResult } from '../integrations/rcon.js';
 import type { RconFailure } from '../integrations/rcon.js';
 import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
-import { komodoInspectHealthy } from '../integrations/komodo.js';
+import { inspectForMonitoring, rconForMonitoring, MonitoringChangedError } from '../fleet/health-observations.js';
 import { restartSettling } from '../provisioning/index.js';
 import { serverObservation, observationReader } from '../fleet/status.js';
 import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
@@ -18,6 +17,7 @@ import { selectedZombieMap } from '../../map-aliases.js';
 import { runMaintenance } from '../maintenance/index.js';
 import { cleanupControllerLibraries } from '../game-files/index.js';
 import { trackActivity } from './activity.js';
+import { withOperationSource } from '../hosts/operation-context.js';
 
 const TICK_MS = 15_000;
 // Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
@@ -105,7 +105,7 @@ function queueStatus(server: string): Promise<number | null> {
     statusQueue.push(async () => {
         let capacity: number | null = null;
         try {
-            capacity = await refreshServerStatus(server);
+            capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server));
             // Publish this server's latest evidence without waiting for the fleet.
             const current = getServer(server);
             if (current) publishStatus(statusEntry(current));
@@ -143,15 +143,23 @@ async function refreshServerStatus(server: string): Promise<number | null> {
     const flush = () => { setServerFields(srv.name, updates); updates = {}; };
     if (!isProvisioning(srv.name)) {
         const epoch = latestGameSubmission(srv.name);
-        const unchanged = () => epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
+        const unchanged = () => {
-            && getServer(srv.name)?.created_at === srv.created_at;
+            const current = getServer(srv.name);
-        const container = restartSettling(srv.name)
+            return epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
-            ? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
+                && current?.created_at === srv.created_at && current.desired_state === srv.desired_state
-            : await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
+                && current.native_instance_id === srv.native_instance_id && current.instance_generation === srv.instance_generation;
+        };
+        const inspectionStartedAt = new Date().toISOString();
+        const inspected = restartSettling(srv.name)
+            ? { result: { ok: false as const, state: 'restarting', reason: 'Restart requested' }, observedAt: inspectionStartedAt }
+            : await inspectForMonitoring(srv.name).catch(err => err instanceof MonitoringChangedError ? null
+                : { result: { ok: false as const, state: 'unknown', reason: (err as Error).message }, observedAt: inspectionStartedAt });
+        if (!inspected) return null;
+        const container = inspected.result;
         // A lifecycle request may be submitted while an inspection is in flight.
-        if (unchanged()) {
+        if (unchanged() && (getServer(srv.name)?.container_checked_at ?? '') <= inspected.observedAt) {
             updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown';
-            updates.container_checked_at = new Date().toISOString();
+            updates.container_checked_at = inspected.observedAt;
             // An unavailable inspection is not evidence that the game stopped.
             // Probe it just as a manual RCON check would; explicit lifecycle or
             // unhealthy states still take precedence over game replies.
@@ -161,10 +169,13 @@ async function refreshServerStatus(server: string): Promise<number | null> {
                 updates.last_reply_snippet = null;
             } else if (unavailable || !telemetryEvidence(srv.name)) {
                 flush();
-                const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
+                const probeStartedAt = new Date().toISOString();
-                    reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
+                const probe = await rconForMonitoring(srv.name).catch(err => err instanceof MonitoringChangedError ? null
-                if (unchanged()) {
+                    : { result: { ok: false as const, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
-                    const now = new Date().toISOString();
+                if (!probe) return null;
+                const result = probe.result;
+                if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) {
+                    const now = probe.observedAt;
                     updates.last_checked_at = now;
                     updates.last_reply_snippet = result.rawReply ?? null;
                     if (result.ok) {

src/routes/api/agent/heartbeat/+server.ts

@@ -3,6 +3,7 @@ import { heartbeat, pendingCreationCancellations } from '$lib/server/hosts/index
 import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js';
 import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js';
 import { reconcileHostLibraryCleanup, cleanupControllerLibraries } from '$lib/server/game-files/index.js';
+import { fleetSettings } from '$lib/server/settings.js';
 export async function POST({ request }: { request: Request }) {
     const authenticated = authenticateAgent(request);
     try {
@@ -16,6 +17,7 @@ export async function POST({ request }: { request: Request }) {
         reconcileHostPreparation(host.id);
         reconcileHostLibraryCleanup(host.id);
         await cleanupControllerLibraries();
-        return json({ host, cancelledCreations: pendingCreationCancellations(host.id), heartbeatSeconds: 15, serverTime: new Date().toISOString() });
+        return json({ host, cancelledCreations: pendingCreationCancellations(host.id), heartbeatSeconds: 15,
+            observationKeepDays: fleetSettings().observationKeepDays, serverTime: new Date().toISOString() });
     } catch (cause) { return hostApiError(cause); }
 }

src/routes/api/hosts/operations/+server.ts

@@ -1,10 +1,14 @@
 import { error, json } from '@sveltejs/kit';
 import { listHostOperationHistory } from '$lib/server/hosts/index.js';
 import { requireManagementOrigin } from '$lib/server/hosts/http.js';
+import { isHostOperationFilter } from '$lib/hosts.js';
 
 export function GET({ request }: { request: Request }) {
     requireManagementOrigin(request);
-    const page = Number(new URL(request.url).searchParams.get('page') ?? '1');
+    const params = new URL(request.url).searchParams;
+    const page = Number(params.get('page') ?? '1');
+    const filter = params.get('filter') ?? 'important';
     if (!Number.isSafeInteger(page) || page < 1) error(400, 'Invalid operation history page.');
-    return json(listHostOperationHistory(page), { headers: { 'Cache-Control': 'no-store' } });
+    if (!isHostOperationFilter(filter)) error(400, 'Invalid operation history filter.');
+    return json(listHostOperationHistory(page, filter), { headers: { 'Cache-Control': 'no-store' } });
 }

src/routes/hosts/operations/+page.server.ts

@@ -1,12 +1,19 @@
 import { redirect } from '@sveltejs/kit';
 import { listHostOperationHistory } from '$lib/server/hosts/index.js';
+import { isHostOperationFilter } from '$lib/hosts.js';
 import type { PageServerLoad } from './$types';
 
 export const load: PageServerLoad = ({ url }) => {
     const requestedPage = url.searchParams.get('page');
-    const history = listHostOperationHistory(Number(requestedPage ?? 1));
+    const requestedFilter = url.searchParams.get('filter');
-    if (requestedPage !== null && requestedPage !== String(history.page)) {
+    const filter = isHostOperationFilter(requestedFilter) ? requestedFilter : 'important';
-        redirect(307, history.page === 1 ? '/hosts/operations' : `/hosts/operations?page=${history.page}`);
+    const history = listHostOperationHistory(Number(requestedPage ?? 1), filter);
+    if ((requestedPage !== null && requestedPage !== String(history.page))
+        || (requestedFilter !== null && requestedFilter !== filter)) {
+        const params = new URLSearchParams();
+        if (requestedFilter !== null) params.set('filter', filter);
+        if (history.page !== 1) params.set('page', String(history.page));
+        redirect(307, `/hosts/operations${params.size ? `?${params}` : ''}`);
     }
     return { history };
 };

src/routes/hosts/operations/+page.svelte

@@ -2,7 +2,7 @@
     import { onMount, untrack } from 'svelte';
     import { beforeNavigate } from '$app/navigation';
     import { navigating } from '$app/stores';
-    import type { HostOperationHistoryPage } from '$lib/hosts.js';
+    import type { HostOperationFilter, HostOperationHistoryPage } from '$lib/hosts.js';
     import type { PageData } from './$types';
     import { pollVisible } from '$ui/poll.js';
     import { timeFormat } from '$ui/stores/time-zone.js';
@@ -19,7 +19,17 @@
     const changingPage = $derived($navigating?.to?.url.pathname === '/hosts/operations');
     const firstOperation = $derived((history.page - 1) * history.pageSize + 1);
     const numbers = new Intl.NumberFormat('en-US');
-    function pageHref(page: number) { return page === 1 ? '/hosts/operations' : `/hosts/operations?page=${page}`; }
+    const filters: Array<{ id: HostOperationFilter; label: string; description: string; empty: string }> = [
+        { id: 'important', label: 'Actions & errors', description: 'Actions, failures, and checks that need attention.', empty: 'No actions or checks needing attention recorded.' },
+        { id: 'routine', label: 'Routine checks', description: 'Monitoring and file reads across all states.', empty: 'No routine checks recorded.' },
+        { id: 'all', label: 'All operations', description: 'Every recorded remote operation.', empty: 'No remote operations yet.' }
+    ];
+    const selectedFilter = $derived(filters.find(filter => filter.id === history.filter)!);
+    function pageHref(page: number, filter: HostOperationFilter = history.filter) {
+        const params = new URLSearchParams({ filter });
+        if (page !== 1) params.set('page', String(page));
+        return `/hosts/operations?${params}`;
+    }
     function cancel() { sequence++; request?.abort(); request = null; refreshing = false; }
 
     async function refresh(signal?: AbortSignal) {
@@ -30,7 +40,8 @@
         request = controller;
         refreshing = true;
         try {
-            const response = await fetch(`/api/hosts/operations?page=${history.page}`, {
+            const params = new URLSearchParams({ page: String(history.page), filter: history.filter });
+            const response = await fetch(`/api/hosts/operations?${params}`, {
                 signal: AbortSignal.any([controller.signal, AbortSignal.timeout(15000), ...(signal ? [signal] : [])])
             });
             const body = await response.json();
@@ -55,18 +66,32 @@
     <SectionHeader title="Operation history" eyebrow="Hosts" subtitle="Remote operations across all hosts, newest first.">
         {#snippet actions()}<Button disabled={refreshing || changingPage} onclick={() => void refresh()}>{refreshing ? 'Refreshing…' : 'Refresh'}</Button>{/snippet}
     </SectionHeader>
+    <div class="history-filters">
+        <nav class="filter-links" aria-label="Operation history filters">
+            {#each filters as filter}
+                <a href={pageHref(1, filter.id)} class:active={history.filter === filter.id}
+                    aria-current={history.filter === filter.id ? 'page' : undefined}>{filter.label}</a>
+            {/each}
+        </nav>
+        <p>{selectedFilter.description}</p>
+    </div>
     {#if refreshError}<p class="ui-error" role="alert">{refreshError}</p>{/if}
     <section class="panel" aria-label="Host operation history">
-        {#if !history.operations.length}<p class="empty" role="status">No remote operations yet.</p>
+        {#if !history.operations.length}<p class="empty" role="status">{selectedFilter.empty}</p>
         {:else}
             <!-- svelte-ignore a11y_no_noninteractive_tabindex (Scrollable table for keyboard users.) -->
             <div class="table-wrap" role="region" aria-label="Operations" tabindex="0">
                 <table>
-                    <thead><tr><th scope="col">Operation</th><th scope="col">Host</th><th scope="col">State</th><th scope="col">Updated</th><th scope="col">Result</th></tr></thead>
+                    <thead><tr><th scope="col">Operation</th><th scope="col">Host</th><th scope="col">Target</th><th scope="col">Source</th><th scope="col">State</th><th scope="col">Updated</th><th scope="col">Result</th></tr></thead>
                     <tbody>{#each history.operations as operation (operation.id)}
                         <tr>
-                            <td>{operation.kind}<small>{operation.id}</small></td>
+                            <td class="operation">{operation.kind}<small>{operation.id}</small></td>
                             <td>{#if operation.hostName !== null}<a href={`/hosts/${operation.hostId}`}>{operation.hostName}</a>{:else}Deleted host<small>{operation.hostId}</small>{/if}</td>
+                            <td class="target">
+                                <span>{operation.serverName ?? operation.instanceId ?? 'Host-level'}</span>
+                                {#if operation.targetPath}<small class="target-path">{operation.targetPath}</small>{/if}
+                            </td>
+                            <td class="source">{operation.source ?? 'Not recorded'}</td>
                             <td class="state" class:failed={operation.state === 'failed'}>{operation.state}</td>
                             <td class="updated">{$timeFormat.dateTime(operation.updatedAt)}</td>
                             <td class="result">{operation.error ?? '—'}</td>
@@ -88,14 +113,25 @@
 
 <style>
     .page { display: grid; gap: 24px; min-width: 0; }
+    .history-filters { min-width: 0; }
+    .history-filters p { margin: 10px 0 0; color: var(--text-secondary); font-size: var(--text-sm); }
+    .filter-links { display: flex; gap: 4px; width: fit-content; max-width: 100%; padding: 4px; overflow-x: auto; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); }
+    .filter-links a { padding: 8px 16px; border-radius: var(--radius-sm); color: var(--text-secondary); font-size: var(--text-sm); text-decoration: none; white-space: nowrap; }
+    .filter-links a:hover { color: var(--text-primary); }
+    .filter-links a.active { color: var(--text-primary); background: var(--surface-raised); }
+    .filter-links a:focus-visible { outline: 2px solid var(--accent-primary); outline-offset: -2px; }
     .panel { min-width: 0; border: 1px solid var(--border-strong); border-radius: var(--radius-md); background: var(--surface); overflow: hidden; }
     .table-wrap { overflow: auto; }
     .table-wrap:focus-visible { outline: 2px solid var(--accent-primary); outline-offset: -2px; }
-    table { width: 100%; min-width: 760px; border-collapse: collapse; text-align: left; font-size: var(--text-sm); }
+    table { width: 100%; min-width: 1050px; border-collapse: collapse; text-align: left; font-size: var(--text-sm); }
     td, th { padding: 16px; border-bottom: 1px solid var(--border-subtle); vertical-align: top; }
     th { color: var(--text-secondary); font-size: var(--text-xs); font-weight: var(--weight-semibold); }
     td { line-height: 1.6; overflow-wrap: anywhere; }
     small { display: block; color: var(--text-secondary); font-size: var(--text-xs); margin-top: 4px; }
+    .operation { min-width: 180px; max-width: 240px; }
+    .target { min-width: 180px; max-width: 320px; }
+    .target-path { font-family: var(--font-mono); }
+    .source { min-width: 120px; max-width: 220px; }
     .state { text-transform: capitalize; }
     .failed { color: var(--state-danger); }
     .updated { white-space: nowrap; }
@@ -103,5 +139,9 @@
     .pagination { padding: 16px; display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 16px; color: var(--text-secondary); font-size: var(--text-xs); }
     .page-controls { display: flex; align-items: center; gap: 12px; }
     .empty { padding: 24px; margin: 0; color: var(--text-secondary); font-size: var(--text-sm); }
-    @media (max-width: 600px) { .page-controls { width: 100%; justify-content: space-between; } }
+    @media (max-width: 600px) {
+        td, th { padding: 12px; }
+        .page-controls { width: 100%; justify-content: space-between; }
+        .filter-links a { padding: 8px 12px; }
+    }
 </style>

src/routes/settings/+page.svelte

@@ -54,6 +54,9 @@
                     <label for="retention">Campaign history retention (days)</label>
                     <input id="retention" type="number" min="1" max="36500" step="1" required bind:value={settings.deployKeepDays} aria-describedby="retention-help" />
                     <p id="retention-help">Default: 7 days. Removes eligible old campaign records and file diffs on the next cleanup check. Pending, failed, held, and health-recovery-linked campaigns are kept. Provisioning history is retained.</p>
+                    <label for="observation-retention">Routine operation history retention (days)</label>
+                    <input id="observation-retention" type="number" min="0" max="36500" step="1" required bind:value={settings.observationKeepDays} aria-describedby="observation-retention-help" />
+                    <p id="observation-retention-help">Default: 1 day. Removes successful monitoring and file-read operation records on the next cleanup check. Failures and actions are retained. Set to 0 to keep these records indefinitely.</p>
                     <label for="restarts">Concurrent restarts</label>
                     <input id="restarts" type="number" min="1" step="1" required bind:value={settings.restartConcurrency} aria-describedby="restarts-help" />
                     <p id="restarts-help">Default: 1 across the fleet. Campaigns, betas, maintenance, and lifecycle actions share recovery slots until container and game health are verified.</p>

static/agent/ec-agent.py

@@ -32,9 +32,13 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.7"
+VERSION = "1.4.9"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
+MANIFEST_MAX_ENTRIES = 10000
+MANIFEST_MAX_BYTES = 16 * 1024 * 1024
+MANIFEST_MAX_SECONDS = 15
+MANIFEST_MAX_DEPTH = 64
 LIBRARY_DOWNLOAD_WORKERS = 4
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
 AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
@@ -296,6 +300,151 @@ def managed_read(root, value):
         return None
 
 
+def managed_manifest(root, payload, stopping):
+    """Hash a bounded managed tree without following runtime-controlled symlinks."""
+    deadline = time.monotonic() + MANIFEST_MAX_SECONDS
+
+    def check_budget():
+        if stopping.is_set() or time.monotonic() >= deadline:
+            raise AgentError("Manifest scan exceeded its time limit or was interrupted")
+
+    roots = payload.get("roots")
+    ignored = payload.get("ignoreNames")
+    paths = payload.get("paths", [])
+    if not isinstance(roots, list) or not roots or len(roots) > 64:
+        raise AgentError("Manifest roots must contain between 1 and 64 managed directories")
+    if not isinstance(ignored, list) or len(ignored) > 256 or any(
+        not isinstance(name, str) or not name or name in (".", "..") or any(char in name for char in ("/", "\\", "\x00"))
+        for name in ignored
+    ):
+        raise AgentError("Manifest ignored names must be plain file or directory names")
+    if not isinstance(paths, list) or len(paths) > 10000:
+        raise AgentError("Manifest managed paths must contain at most 10000 files")
+    for value in roots:
+        check_budget()
+        scoped(root, value)
+    roots = sorted(set(roots))
+    roots = [value for index, value in enumerate(roots) if not any(value.startswith(parent + "/") for parent in roots[:index])]
+    ignored = set(ignored)
+    flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
+    entries = 0
+    result_bytes = len(json.dumps({"files": []}).encode())
+    files = []
+
+    def validate_paths():
+        for value in paths:
+            check_budget()
+            try:
+                with managed_open(root, value):
+                    pass
+            except FileNotFoundError:
+                pass
+
+    def identity(info):
+        return info.st_dev, info.st_ino, info.st_mode, info.st_size, info.st_mtime_ns, info.st_ctime_ns
+
+    def open_directory(app, relative):
+        directory = os.dup(app)
+        try:
+            for part in PurePosixPath(relative).parts:
+                nested = os.open(part, flags, dir_fd=directory)
+                os.close(directory)
+                directory = nested
+            return directory
+        except BaseException:
+            os.close(directory)
+            raise
+
+    def scan(directory, relative, depth):
+        nonlocal entries, result_bytes
+        check_budget()
+        if depth > MANIFEST_MAX_DEPTH:
+            raise AgentError("Manifest scan exceeded its directory depth limit")
+        before = os.fstat(directory)
+        with os.scandir(directory) as children:
+            for child in children:
+                check_budget()
+                entries += 1
+                if entries > MANIFEST_MAX_ENTRIES:
+                    raise AgentError("Manifest scan exceeded its entry limit")
+                if child.name in ignored:
+                    continue
+                info = os.stat(child.name, dir_fd=directory, follow_symlinks=False)
+                if stat_module.S_ISLNK(info.st_mode):
+                    continue
+                path = relative + "/" + child.name
+                if stat_module.S_ISDIR(info.st_mode):
+                    nested = os.open(child.name, flags, dir_fd=directory)
+                    try:
+                        if identity(info) != identity(os.fstat(nested)):
+                            raise AgentError("Manifest directory changed during the scan")
+                        scan(nested, path, depth + 1)
+                        if identity(os.fstat(nested)) != identity(os.stat(child.name, dir_fd=directory, follow_symlinks=False)):
+                            raise AgentError("Manifest directory changed during the scan")
+                    finally:
+                        os.close(nested)
+                elif stat_module.S_ISREG(info.st_mode):
+                    descriptor = os.open(child.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory)
+                    try:
+                        if identity(info) != identity(os.fstat(descriptor)):
+                            raise AgentError("Manifest file changed during the scan")
+                        sha = hashlib.sha256()
+                        size = 0
+                        with os.fdopen(descriptor, "rb", closefd=False) as stream:
+                            while True:
+                                check_budget()
+                                block = stream.read(1024 * 1024)
+                                if not block:
+                                    break
+                                sha.update(block)
+                                size += len(block)
+                        if size != info.st_size or identity(info) != identity(os.fstat(descriptor)) or identity(info) != identity(os.stat(child.name, dir_fd=directory, follow_symlinks=False)):
+                            raise AgentError("Manifest file changed during the scan")
+                    finally:
+                        os.close(descriptor)
+                    item = {"path": path, "size": size, "sha256": sha.hexdigest()}
+                    result_bytes += len(json.dumps(item).encode()) + (2 if files else 0)
+                    if result_bytes > MANIFEST_MAX_BYTES:
+                        raise AgentError("Manifest scan exceeded its result size limit")
+                    files.append(item)
+                else:
+                    raise AgentError("Manifest scan encountered a non-regular managed file")
+        if identity(before) != identity(os.fstat(directory)):
+            raise AgentError("Manifest directory changed during the scan")
+
+    try:
+        app = os.open(root, flags)
+        try:
+            app_identity = os.fstat(app)
+            validate_paths()
+            for relative in roots:
+                check_budget()
+                try:
+                    directory = open_directory(app, relative)
+                except FileNotFoundError:
+                    continue  # A destination directory need not exist yet.
+                try:
+                    scan(directory, relative, len(PurePosixPath(relative).parts))
+                    current_directory = open_directory(app, relative)
+                    try:
+                        if identity(os.fstat(directory)) != identity(os.fstat(current_directory)):
+                            raise AgentError("Manifest directory changed during the scan")
+                    finally:
+                        os.close(current_directory)
+                finally:
+                    os.close(directory)
+            validate_paths()
+            current = os.stat(root, follow_symlinks=False)
+            if (current.st_dev, current.st_ino, current.st_mode) != (app_identity.st_dev, app_identity.st_ino, app_identity.st_mode):
+                raise AgentError("Manifest managed root changed during the scan")
+            check_budget()
+        finally:
+            os.close(app)
+    except OSError:
+        raise AgentError("Manifest scan could not read the complete managed tree") from None
+    return {"files": sorted(files, key=lambda item: item["path"])}
+
+
 def managed_write(root, value, content, uid=1000, gid=1000):
     if content is None:
         try:
@@ -367,6 +516,19 @@ class ReadWriteGate:
                 self.condition.notify_all()
 
 
+def read_operation(operation):
+    if not isinstance(operation, dict):
+        return False
+    if operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read"):
+        return True
+    payload = operation.get("payload")
+    return operation.get("kind") == "rcon.execute" and isinstance(payload, dict) and payload.get("command") in ("status", "getstatus", "version")
+
+
+def valid_observation_keep_days(value):
+    return type(value) is int and 0 <= value <= 36500
+
+
 class Journal:
     def __init__(self, path, recover=True):
         self.db = sqlite3.connect(path)
@@ -376,8 +538,14 @@ class Journal:
         self.db.execute("PRAGMA journal_mode=WAL")
         self.db.execute("PRAGMA synchronous=FULL")
         self.db.execute("CREATE TABLE IF NOT EXISTS operations (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)")
-        if "checkpoint" not in {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}:
+        columns = {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}
+        if "checkpoint" not in columns:
             self.db.execute("ALTER TABLE operations ADD COLUMN checkpoint TEXT")
+        if "reported_at" not in columns:
+            self.db.execute("ALTER TABLE operations ADD COLUMN reported_at REAL")
+        if "retention_eligible" not in columns:
+            self.db.execute("ALTER TABLE operations ADD COLUMN retention_eligible INTEGER NOT NULL DEFAULT 0")
+        self.db.execute("CREATE INDEX IF NOT EXISTS operations_observation_retention ON operations(updated,id) WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1")
         self.db.execute("UPDATE operations SET state='unknown', error='Agent restarted during execution; inspect before reconciliation', updated=? WHERE state='running'", (time.time(),))
         self.db.commit()
 
@@ -402,10 +570,37 @@ class Journal:
         return dict(row)
 
     def update(self, op_id, state, result=None, error=None):
-        self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=? WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id))
+        self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=?, reported_at=NULL, retention_eligible=0 WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id))
         self.db.commit()
         return self.get(op_id)
 
+    def confirm_completed(self, row):
+        """Only an acknowledged, unchanged completion can become disposable."""
+        if row["state"] != "completed":
+            return
+        eligible = False
+        try:
+            result = json.loads(row["result"]) if row.get("result") is not None else None
+            eligible = read_operation(json.loads(row["request"])) and not row.get("error") and not (
+                isinstance(result, dict) and (result.get("ok") is False or result.get("healthy") is False or result.get("error") not in (None, ""))
+            )
+        except (ValueError, TypeError, KeyError):
+            pass  # Malformed historical records are retained for inspection.
+        self.db.execute("""UPDATE operations SET reported_at=COALESCE(reported_at,?), retention_eligible=?
+            WHERE id=? AND state='completed' AND updated=? AND result IS ? AND error IS ?""",
+            (time.time(), int(bool(eligible)), row["id"], row["updated"], row.get("result"), row.get("error")))
+        self.db.commit()
+
+    def prune_observations(self, keep_days):
+        if not valid_observation_keep_days(keep_days) or keep_days == 0:
+            return 0
+        cutoff = time.time() - keep_days * 86400
+        removed = self.db.execute("""DELETE FROM operations WHERE id IN (
+            SELECT id FROM operations WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1
+                AND updated<? ORDER BY updated,id LIMIT 500)""", (cutoff,)).rowcount
+        self.db.commit()
+        return removed
+
     def close(self):
         self.db.close()
 
@@ -719,6 +914,7 @@ class Agent:
         self.read_claim_lock = threading.Lock()
         self.read_claims = set()
         self.cancelled_creations = frozenset()
+        self.observation_keep_days = 0  # Older controllers do not authorize cleanup.
         self.library_progress = {}
         self.library_transfers = {}
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
@@ -1094,7 +1290,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1446,6 +1642,8 @@ class Agent:
             return result
         if kind == "logs.read":
             return self.read_log(instance_id, generation, payload)
+        if kind == "files.manifest":
+            return managed_manifest(app, payload, self.stopping)
         path = scoped(app, payload.get("path", ""), allow_root=kind in ("files.list", "files.stat"))
         if kind == "files.read":
             content = managed_read(app, payload["path"])
@@ -1967,7 +2165,7 @@ class Agent:
                     return self.journal.update(operation["id"], "failed", error="Interrupted library download could not be resumed")
             if kind == "files.stage":
                 return self.journal.update(operation["id"], "completed", self.stage(instance_id, generation, payload, operation["id"]))
-            if kind in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "logs.read") or kind == "rcon.execute" and payload.get("command") in ("status", "getstatus", "version"):
+            if self.read_operation(operation):
                 return self.journal.update(operation["id"], "completed", self.dispatch(operation))
             if kind in ("files.activate", "files.rollback") and checkpoint:
                 stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json"
@@ -2021,7 +2219,7 @@ class Agent:
             pass
         return row
 
-    def report(self, row):
+    def report(self, row, journal=None):
         payload = {"state": row["state"]}
         if row.get("result"):
             payload["result"] = json.loads(row["result"])
@@ -2031,13 +2229,19 @@ class Agent:
                 payload["state"] = "running"
         if row.get("error"):
             payload["error"] = row["error"]
-        return self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload)
+        response = self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload)
+        confirmed = response.get("operation") if isinstance(response, dict) else None
+        if row["state"] == "completed" and isinstance(confirmed, dict) and confirmed.get("id") == row["id"] and confirmed.get("state") == "completed":
+            (journal if journal is not None else self.journal).confirm_completed(row)
+        return response
 
     def heartbeat(self):
         response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation})
         # Only the heartbeat thread replaces this immutable snapshot; it never
         # mutates the operation journal or races Docker cleanup against create.
         self.cancelled_creations = frozenset(value for value in response.get("cancelledCreations", []) if isinstance(value, str))
+        keep_days = response.get("observationKeepDays")
+        self.observation_keep_days = keep_days if valid_observation_keep_days(keep_days) else 0
         host = response.get("host", {})
         if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
             self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}
@@ -2054,7 +2258,7 @@ class Agent:
 
     @staticmethod
     def read_operation(operation):
-        return operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "logs.read") or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
+        return read_operation(operation)
 
     def instance_lock(self, operation):
         key = operation.get("instanceId")
@@ -2068,12 +2272,19 @@ class Agent:
         # Each lane owns its SQLite connection and never writes executing_id,
         # checkpoints, or current_operation used by the mutation state machine.
         journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
+        next_prune = time.monotonic()
         try:
             while not self.stopping.is_set():
                 try:
                     if self.update_pending():
                         self.stopping.wait(3)
                         continue
+                    if lane == "read" and time.monotonic() >= next_prune:
+                        next_prune = time.monotonic() + 60
+                        with self.activity(read_only=True) as acquired:
+                            if acquired and not self.update_pending():
+                                if journal.prune_observations(self.observation_keep_days) == 500:
+                                    next_prune = time.monotonic() + 15
                     response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20")
                     handled = False
                     for operation in response.get("operations", []):
@@ -2091,7 +2302,7 @@ class Agent:
                                     continue
                                 row, _ = journal.accept(operation)
                                 if row["state"] in ("accepted", "unknown", "running"):
-                                    self.report(journal.update(operation["id"], "running"))
+                                    self.report(journal.update(operation["id"], "running"), journal=journal)
                                     with self.read_gate.read(), self.instance_lock(operation):
                                         try:
                                             result = self.dispatch(operation)
@@ -2100,7 +2311,7 @@ class Agent:
                                             row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
                                 # Receipt delivery can take seconds; it must not
                                 # hold the instance lock needed by health checks.
-                                self.report(row)
+                                self.report(row, journal=journal)
                                 handled = True
                         finally:
                             with self.read_claim_lock:

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.4.7",
+  "version": "1.4.9",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "e9e1b3869d66eca351dadb045d99451b97c91d15c5b7199a3de3241b550de998",
+  "sha256": "c0a02509a8307b2abb75e5450bf3f39596252f26e041ed938d662bc7127bec62",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/fleet-settings.test.ts

@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 
 let fixture: Fixture;
-const defaults = { deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1, destroyConcurrency: 1 };
+const defaults = { deployKeepDays: 7, observationKeepDays: 1, restartConcurrency: 1, createConcurrency: 1, destroyConcurrency: 1 };
 beforeEach(async () => {
     vi.resetModules();
     fixture = await newFixture([]);
@@ -14,7 +14,7 @@ describe('fleet settings API and persistence', () => {
     it('returns defaults and saves all settings across a module reload and database reopen', async () => {
         const { GET, PUT } = await import('../../src/routes/api/settings/+server.js');
         expect(await GET().json()).toEqual(defaults);
-        const saved = { deployKeepDays: 30, restartConcurrency: 3, createConcurrency: 2, destroyConcurrency: 4 };
+        const saved = { deployKeepDays: 30, observationKeepDays: 5, restartConcurrency: 3, createConcurrency: 2, destroyConcurrency: 4 };
         const response = await PUT({ request: new Request('http://localhost/api/settings', { method: 'PUT', body: JSON.stringify(saved) }) });
         expect(response.status).toBe(200);
         expect(await response.json()).toEqual(saved);
@@ -37,7 +37,7 @@ describe('fleet settings API and persistence', () => {
         vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '4');
         vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
         const { fleetSettings } = await import('../../src/lib/server/settings.js');
-        expect(fleetSettings()).toEqual({ deployKeepDays: 14, restartConcurrency: 4, createConcurrency: 2, destroyConcurrency: 1 });
+        expect(fleetSettings()).toEqual({ deployKeepDays: 14, observationKeepDays: 1, restartConcurrency: 4, createConcurrency: 2, destroyConcurrency: 1 });
         closeDb();
         vi.stubEnv('DEPLOY_KEEP_DAYS', '7');
         expect(fleetSettings().deployKeepDays).toBe(14);
@@ -53,18 +53,47 @@ describe('fleet settings API and persistence', () => {
         vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '9');
         vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '8');
         const { GET } = await import('../../src/routes/api/settings/+server.js');
-        const upgraded = { ...previous, destroyConcurrency: 1 };
+        const upgraded = { ...previous, destroyConcurrency: 1, observationKeepDays: 1 };
         expect(await GET().json()).toEqual(upgraded);
         expect(JSON.parse((db().prepare("SELECT value FROM settings WHERE key = 'fleet_settings'").get() as { value: string }).value)).toEqual(upgraded);
         closeDb();
         expect(await GET().json()).toEqual(upgraded);
     });
 
+    it('adds routine observation retention to modern saved settings without changing any existing values', async () => {
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const previous = { deployKeepDays: 45, restartConcurrency: 3, createConcurrency: 2, destroyConcurrency: 4 };
+        db().prepare("UPDATE settings SET value = ? WHERE key = 'fleet_settings'").run(JSON.stringify(previous));
+        db().prepare("INSERT INTO settings(key,value) VALUES('unrelated','keep')").run();
+        closeDb();
+        vi.stubEnv('DEPLOY_KEEP_DAYS', 'bad');
+        const { GET } = await import('../../src/routes/api/settings/+server.js');
+        const upgraded = { ...previous, observationKeepDays: 1 };
+        expect(await GET().json()).toEqual(upgraded);
+        expect(JSON.parse((db().prepare("SELECT value FROM settings WHERE key = 'fleet_settings'").get() as { value: string }).value)).toEqual(upgraded);
+        expect(db().prepare("SELECT value FROM settings WHERE key='unrelated'").get()).toEqual({ value: 'keep' });
+        closeDb();
+        expect(await GET().json()).toEqual(upgraded);
+    });
+
+    it.each([0, 36500])('persists routine observation retention of %s days across database reopen', async observationKeepDays => {
+        const { GET, PUT } = await import('../../src/routes/api/settings/+server.js');
+        const saved = { ...defaults, observationKeepDays };
+        const response = await PUT({ request: new Request('http://localhost/api/settings', { method: 'PUT', body: JSON.stringify(saved) }) });
+        expect(response.status).toBe(200);
+        expect(await response.json()).toEqual(saved);
+        const { closeDb } = await import('../../src/lib/server/db/index.js');
+        closeDb();
+        expect(await GET().json()).toEqual(saved);
+    });
+
     it.each([
         'null', '[]', '{}', '{',
         ...['deployKeepDays', 'restartConcurrency', 'createConcurrency', 'destroyConcurrency'].flatMap(key =>
             [0, -1, 1.5, '2', true, null, 9007199254740992].map(value => JSON.stringify({ ...defaults, [key]: value }))),
+        ...[-1, 1.5, '2', true, null, 36501, 9007199254740992].map(value => JSON.stringify({ ...defaults, observationKeepDays: value })),
         JSON.stringify({ deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1 }),
+        JSON.stringify({ deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1, destroyConcurrency: 1 }),
         JSON.stringify({ ...defaults, deployKeepDays: 36501 }),
         JSON.stringify({ ...defaults, unknown: 1 })
     ])('rejects invalid input without partially saving: %s', async body => {

tests/unit/health-observations.test.ts

@@ -0,0 +1,259 @@
+import { randomUUID } from 'node:crypto';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn() }));
+vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn() }));
+
+const server = 'managed';
+const now = Date.parse('2026-10-05T16:00:00Z');
+const healthy = { ok: true as const, state: 'running' };
+const playing = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
+    status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } };
+let fixture: Fixture;
+
+beforeEach(async () => {
+    vi.resetModules(); vi.clearAllMocks();
+    fixture = await newFixture([]);
+    vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now);
+});
+afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.restoreAllMocks(); });
+
+async function setup() {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const database = db();
+    database.prepare('INSERT INTO servers(name,komodo_name,created_at,game_port) VALUES(?,?,?,28960)')
+        .run(server, server, new Date(now).toISOString());
+    const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+    const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
+    const inspect = vi.mocked(komodoInspectHealthy).mockResolvedValue(healthy);
+    const rcon = vi.mocked(rconStatusResult).mockResolvedValue(playing);
+    return { database, inspect, rcon, ...await import('../../src/lib/server/fleet/health-observations.js') };
+}
+
+async function native(api: Awaited<ReturnType<typeof setup>>) {
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    const host = hosts.createHost({ name: 'Monitoring host' });
+    hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 });
+    const instanceId = randomUUID();
+    api.database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
+        VALUES(?,?,?,'library','image',28960,?)`).run(instanceId, server, host.id, new Date(now).toISOString());
+    return { host, instanceId, hosts };
+}
+
+function deferred<T>() {
+    let resolve!: (value: T) => void;
+    const promise = new Promise<T>(done => { resolve = done; });
+    return { promise, resolve };
+}
+
+describe('shared monitoring observations', () => {
+    it.each(['inspect', 'rcon'] as const)('reuses %s results until 30 seconds after request start without refreshing their timestamp', async kind => {
+        const api = await setup();
+        const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring;
+        const request = kind === 'inspect' ? api.inspect : api.rcon;
+        const first = await call(server);
+        expect(first.observedAt).toBe(new Date(now).toISOString());
+        vi.setSystemTime(now + 29_999);
+        expect(await call(server)).toEqual(first);
+        expect(request).toHaveBeenCalledTimes(1);
+        vi.setSystemTime(now + 30_000);
+        expect((await call(server)).observedAt).toBe(new Date(now + 30_000).toISOString());
+        expect(request).toHaveBeenCalledTimes(2);
+    });
+
+    it('shares overlapping monitoring requests, but keeps inspection and RCON separate', async () => {
+        const api = await setup();
+        const pending = deferred<typeof healthy>();
+        api.inspect.mockReturnValueOnce(pending.promise);
+        const observations = [api.inspectForMonitoring(server), api.inspectForMonitoring(server)];
+        await api.rconForMonitoring(server);
+        expect(api.inspect).toHaveBeenCalledTimes(1);
+        expect(api.rcon).toHaveBeenCalledTimes(1);
+        vi.setSystemTime(now + 5000);
+        pending.resolve(healthy);
+        expect(await Promise.all(observations)).toEqual(Array(2).fill({ result: healthy, observedAt: new Date(now + 5000).toISOString() }));
+    });
+
+    it('timestamps a slow result on completion without making it reusable after its request expires', async () => {
+        const api = await setup();
+        const pending = deferred<typeof healthy>();
+        api.inspect.mockReturnValueOnce(pending.promise);
+        const observation = api.inspectForMonitoring(server);
+        await Promise.resolve();
+        vi.setSystemTime(now + 31_000);
+        pending.resolve(healthy);
+        expect(await observation).toEqual({ result: healthy, observedAt: new Date(now + 31_000).toISOString() });
+        await api.inspectForMonitoring(server);
+        expect(api.inspect).toHaveBeenCalledTimes(2);
+    });
+
+    it('reuses the saved completion timestamp while expiring from the earlier request start', async () => {
+        const api = await setup();
+        const pending = deferred<typeof healthy>();
+        api.inspect.mockReturnValueOnce(pending.promise);
+        const observation = api.inspectForMonitoring(server);
+        await Promise.resolve();
+        vi.setSystemTime(now + 20_000);
+        pending.resolve(healthy);
+        const completed = await observation;
+        api.database.prepare('UPDATE servers SET container_checked_at=? WHERE name=?').run(completed.observedAt, server);
+        vi.setSystemTime(now + 25_000);
+        expect(await api.inspectForMonitoring(server)).toEqual(completed);
+        expect(api.inspect).toHaveBeenCalledTimes(1);
+        vi.setSystemTime(now + 30_000);
+        await api.inspectForMonitoring(server);
+        expect(api.inspect).toHaveBeenCalledTimes(2);
+    });
+
+    it('does not reuse observations from a future clock', async () => {
+        const api = await setup();
+        await api.inspectForMonitoring(server);
+        vi.setSystemTime(now - 1);
+        await api.inspectForMonitoring(server);
+        expect(api.inspect).toHaveBeenCalledTimes(2);
+    });
+
+    it('does not cache failed results or rejected requests', async () => {
+        const api = await setup();
+        api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' });
+        expect((await api.inspectForMonitoring(server)).result.ok).toBe(false);
+        await api.inspectForMonitoring(server);
+        expect(api.inspect).toHaveBeenCalledTimes(2);
+        api.rcon.mockRejectedValueOnce(new Error('Unavailable'));
+        await expect(api.rconForMonitoring(server)).rejects.toThrow('Unavailable');
+        await api.rconForMonitoring(server);
+        expect(api.rcon).toHaveBeenCalledTimes(2);
+    });
+
+    it('forced checks bypass completed results and evict old success when the forced result fails', async () => {
+        const api = await setup();
+        await api.inspectForMonitoring(server);
+        api.inspect.mockResolvedValueOnce({ ok: false, state: 'stopped', reason: 'Stopped' });
+        expect((await api.inspectForMonitoring(server, true)).result.ok).toBe(false);
+        await api.inspectForMonitoring(server);
+        expect(api.inspect).toHaveBeenCalledTimes(3);
+    });
+
+    it('forced checks bypass wrapper overlap and an older reply cannot replace the newer cache entry', async () => {
+        const api = await setup();
+        const old = deferred<typeof healthy>();
+        api.inspect.mockReturnValueOnce(old.promise);
+        const first = api.inspectForMonitoring(server);
+        await Promise.resolve();
+        vi.setSystemTime(now + 1000);
+        const latest = await api.inspectForMonitoring(server, true);
+        old.resolve(healthy); await first;
+        expect(await api.inspectForMonitoring(server)).toEqual(latest);
+        expect(api.inspect).toHaveBeenCalledTimes(2);
+    });
+
+    it.each(['registration', 'lifecycle', 'desired state', 'stop'] as const)('invalidates cached observations after a %s change', async change => {
+        const api = await setup();
+        await api.inspectForMonitoring(server);
+        if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server);
+        if (change === 'lifecycle') api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString());
+        if (change === 'desired state') api.database.prepare("UPDATE servers SET desired_state='stopped' WHERE name=?").run(server);
+        if (change === 'stop') api.database.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
+            VALUES('stop',?,'stop','awaiting-health','Stopping',?,?,'ui')`).run(server, new Date(now).toISOString(), new Date(now + 1).toISOString());
+        await api.inspectForMonitoring(server);
+        expect(api.inspect).toHaveBeenCalledTimes(2);
+    });
+
+    it('invalidates integration changes and accepts legacy port discovery by the successful RCON probe', async () => {
+        const api = await setup();
+        await api.inspectForMonitoring(server);
+        api.database.prepare("UPDATE servers SET komodo_name='replacement' WHERE name=?").run(server);
+        await api.inspectForMonitoring(server);
+        expect(api.inspect).toHaveBeenCalledTimes(2);
+        api.database.prepare('UPDATE servers SET game_port=NULL WHERE name=?').run(server);
+        api.rcon.mockImplementationOnce(async () => {
+            api.database.prepare('UPDATE servers SET game_port=28960 WHERE name=?').run(server);
+            return playing;
+        });
+        expect((await api.rconForMonitoring(server)).result).toEqual(playing);
+        await api.rconForMonitoring(server);
+        expect(api.rcon).toHaveBeenCalledTimes(1);
+        api.database.prepare('UPDATE servers SET game_port=28961 WHERE name=?').run(server);
+        await api.rconForMonitoring(server);
+        expect(api.rcon).toHaveBeenCalledTimes(2);
+    });
+
+    it.each(['inspect', 'rcon'] as const)('does not reuse %s after a newer manual observation', async kind => {
+        const api = await setup();
+        const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring;
+        const request = kind === 'inspect' ? api.inspect : api.rcon;
+        const field = kind === 'inspect' ? 'container_checked_at' : 'last_checked_at';
+        await call(server);
+        vi.setSystemTime(now + 1000);
+        api.database.prepare(`UPDATE servers SET ${field}=? WHERE name=?`).run(new Date(now + 1000).toISOString(), server);
+        await call(server);
+        expect(request).toHaveBeenCalledTimes(2);
+    });
+
+    it('rejects a previous generation completing after replacement without discarding the new generation cache', async () => {
+        const api = await setup();
+        const { instanceId } = await native(api);
+        const pending = deferred<typeof healthy>();
+        api.inspect.mockReturnValueOnce(pending.promise);
+        const old = api.inspectForMonitoring(server);
+        const rejected = expect(old).rejects.toThrow('Server changed');
+        await Promise.resolve();
+        api.database.prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId);
+        const current = await api.inspectForMonitoring(server);
+        pending.resolve(healthy); await rejected;
+        expect(await api.inspectForMonitoring(server)).toEqual(current);
+        expect(api.inspect).toHaveBeenCalledTimes(2);
+    });
+
+    it('rejects a result when a lifecycle request arrives while its probe is in flight', async () => {
+        const api = await setup();
+        const pending = deferred<typeof playing>();
+        api.rcon.mockReturnValueOnce(pending.promise);
+        const old = api.rconForMonitoring(server);
+        const rejected = expect(old).rejects.toThrow('Server changed');
+        await Promise.resolve();
+        api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString());
+        pending.resolve(playing); await rejected;
+        await api.rconForMonitoring(server);
+        expect(api.rcon).toHaveBeenCalledTimes(2);
+    });
+
+    it('discards a slow result when a newer manual observation was saved during the request', async () => {
+        const api = await setup();
+        const pending = deferred<typeof playing>();
+        api.rcon.mockReturnValueOnce(pending.promise);
+        const old = api.rconForMonitoring(server);
+        const rejected = expect(old).rejects.toBeInstanceOf(api.MonitoringChangedError);
+        await Promise.resolve();
+        vi.setSystemTime(now + 1000);
+        api.database.prepare('UPDATE servers SET last_checked_at=? WHERE name=?').run(new Date().toISOString(), server);
+        pending.resolve(playing); await rejected;
+        await api.rconForMonitoring(server);
+        expect(api.rcon).toHaveBeenCalledTimes(2);
+    });
+
+    it('discards cached native success while its host is offline and probes again after reconnecting', async () => {
+        const api = await setup();
+        const { host, hosts } = await native(api);
+        await api.inspectForMonitoring(server);
+        api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id);
+        await expect(api.inspectForMonitoring(server)).rejects.toThrow('Host agent is offline');
+        expect(api.inspect).toHaveBeenCalledTimes(1);
+        hosts.heartbeat(host.id, { protocolVersion: 1 });
+        await api.inspectForMonitoring(server);
+        expect(api.inspect).toHaveBeenCalledTimes(2);
+    });
+
+    it('rejects an otherwise successful result if the native host went offline during the probe', async () => {
+        const api = await setup();
+        const { host } = await native(api);
+        const pending = deferred<typeof healthy>();
+        api.inspect.mockReturnValueOnce(pending.promise);
+        const observation = api.inspectForMonitoring(server);
+        const rejected = expect(observation).rejects.toThrow('Host agent is offline');
+        await Promise.resolve();
+        api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id);
+        pending.resolve(healthy); await rejected;
+    });
+});

tests/unit/host-observation-retention.test.ts

@@ -0,0 +1,156 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+const now = Date.parse('2026-10-05T16:00:00Z');
+const day = 86_400_000;
+const at = (age: number) => new Date(now - age).toISOString();
+const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
+    storageAvailable: 100 * 1024 ** 3, runtimeImages: ['plutainer:test'], libraries: [{ id: 'library', state: 'ready' }] };
+let fixture: Fixture;
+
+beforeEach(async () => {
+    vi.resetModules(); fixture = await newFixture([]);
+    vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now);
+});
+afterEach(async () => { vi.useRealTimers(); vi.restoreAllMocks(); await fixture.cleanup(); });
+
+async function setup() {
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+    const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+    const host = hosts.createHost({ name: 'Retention host' });
+    const credentials = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
+    const handle = db();
+    function add(kind: string, options: { state?: string; payload?: Record<string, unknown>; result?: unknown;
+        error?: string; age?: number; serialized?: boolean; routing?: Record<string, unknown> } = {}) {
+        const payload = options.payload ?? {};
+        const operation = options.serialized
+            ? hosts.queueHostOperationSerialized(host.id, kind, Buffer.from(JSON.stringify(payload)), {}, options.routing ?? {})
+            : hosts.queueHostOperation(host.id, kind, payload);
+        const state = options.state ?? 'completed';
+        if (state !== 'pending') hosts.reportHostOperationStatus(host.id, operation.id, {
+            state, result: options.result === undefined ? { ok: true } : options.result, error: options.error
+        });
+        handle.prepare('UPDATE ec_host_operations SET created_at=?,updated_at=? WHERE id=?')
+            .run(at(10 * day), at(options.age ?? 2 * day), operation.id);
+        return operation.id;
+    }
+    return { ...hosts, handle, db, closeDb, host, credentials, fleetSettings, saveFleetSettings, add };
+}
+
+describe('routine host observation retention', () => {
+    it('removes aged successful reads, manifests, and read-only RCON while retaining failures, actions, and recent evidence', async () => {
+        const api = await setup();
+        const removed = ['host.inspect', 'instance.inspect', 'files.read', 'files.stat', 'files.list', 'files.manifest', 'logs.read']
+            .map(kind => api.add(kind));
+        removed.push(...['status', 'getstatus', 'version'].map(command => api.add('rcon.execute', { payload: { command } })));
+        const retained = [
+            ...['pending', 'accepted', 'running', 'failed', 'unknown'].map(state => api.add('files.read', { state })),
+            ...['instance.restart', 'instance.stop', 'instance.remove', 'files.stage', 'files.activate', 'files.rollback', 'runtime.pull', 'unknown.read']
+                .map(kind => api.add(kind)),
+            api.add('rcon.execute', { payload: { command: 'map_restart' } }),
+            api.add('rcon.execute', { payload: { command: 'status; map_restart' } }),
+            api.add('rcon.execute', { payload: { command: 'status' }, result: { ok: false, reason: 'timeout' } }),
+            api.add('instance.inspect', { result: { healthy: false, state: 'stopped' } }),
+            api.add('files.read', { error: 'Read failed' }),
+            api.add('files.read', { result: { error: 'Read failed' } }),
+            api.add('files.stat', { age: day }),
+            api.add('files.read', { age: day - 1 })
+        ];
+        expect(api.pruneHostObservations()).toBe(removed.length);
+        const ids = (api.handle.prepare('SELECT id FROM ec_host_operations').all() as { id: string }[]).map(row => row.id);
+        expect(ids.sort()).toEqual(retained.sort());
+        expect(api.pruneHostObservations()).toBe(0);
+    });
+
+    it('classifies full serialized RCON payloads and deletes their events and immutable bytes together', async () => {
+        const api = await setup();
+        const read = api.add('rcon.execute', { serialized: true, payload: { command: 'status' }, routing: { command: 'map_restart' } });
+        const action = api.add('rcon.execute', { serialized: true, payload: { command: 'map_restart' }, routing: { command: 'status' } });
+        expect(api.handle.prepare('SELECT COUNT(*) AS count FROM ec_host_operation_payloads').get()).toEqual({ count: 2 });
+        expect(api.pruneHostObservations()).toBe(1);
+        expect(api.getHostOperationStatus(read)).toBeNull();
+        expect(api.getHostOperation(action)?.payload).toEqual({ command: 'map_restart' });
+        expect(api.handle.prepare('SELECT DISTINCT operation_id FROM ec_host_operation_events').all()).toEqual([{ operation_id: action }]);
+        expect(api.handle.prepare('SELECT operation_id FROM ec_host_operation_payloads').all()).toEqual([{ operation_id: action }]);
+        expect(api.handle.pragma('foreign_key_check')).toEqual([]);
+    });
+
+    it('honors disabled retention and changes to the saved period using the last receipt time', async () => {
+        const api = await setup();
+        const old = api.add('files.read', { age: 4 * day });
+        const recentReceipt = api.add('files.read', { age: 2 * day });
+        api.saveFleetSettings({ ...api.fleetSettings(), observationKeepDays: 0 });
+        expect(api.pruneHostObservations()).toBe(0);
+        api.saveFleetSettings({ ...api.fleetSettings(), observationKeepDays: 3 });
+        expect(api.pruneHostObservations()).toBe(1);
+        expect(api.getHostOperationStatus(old)).toBeNull();
+        expect(api.getHostOperationStatus(recentReceipt)).not.toBeNull();
+        api.saveFleetSettings({ ...api.fleetSettings(), observationKeepDays: 1 });
+        expect(api.pruneHostObservations()).toBe(1);
+    });
+
+    it('bounds each cleanup batch and seeks through the retention index despite extensive retained action history', async () => {
+        const api = await setup();
+        api.handle.transaction(() => {
+            const insert = api.handle.prepare(`INSERT INTO ec_host_operations
+                (id,host_id,kind,payload,state,created_at,updated_at,observation_success)
+                VALUES(?,?,?,'{}','completed',?,?,?)`);
+            for (let index = 0; index < 2000; index++) insert.run(`action-${index}`, api.host.id, 'instance.restart', at(10 * day), at(10 * day), 0);
+            for (let index = 0; index < 503; index++) insert.run(`read-${index}`, api.host.id, 'files.read', at(2 * day), at(2 * day), 1);
+        })();
+        const prepare = vi.spyOn(api.handle, 'prepare');
+        expect(api.pruneHostObservations()).toBe(500);
+        const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('SELECT id FROM ec_host_operations WHERE observation_success'))!;
+        prepare.mockRestore();
+        const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(at(day)) as { detail: string }[];
+        expect(plan.some(row => row.detail.includes('USING INDEX ec_host_operations_observation_retention'))).toBe(true);
+        expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false);
+        expect(api.pruneHostObservations()).toBe(3);
+        expect(api.pruneHostObservations()).toBe(0);
+        expect(api.handle.prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 2000 });
+    });
+
+    it('backfills legacy observation classification once while preserving actions and negative or invalid results', async () => {
+        const api = await setup();
+        const read = api.add('files.manifest');
+        const serializedRead = api.add('rcon.execute', { serialized: true, payload: { command: 'status' } });
+        const retained = [api.add('instance.restart'), api.add('rcon.execute', { payload: { command: 'map_restart' } }),
+            api.add('instance.inspect', { result: { healthy: false } }), api.add('files.read', { result: { ok: false } }),
+            api.add('files.read', { error: 'Unavailable' }), api.add('files.read', { result: { error: 'Unavailable' } }),
+            api.add('files.stat', { state: 'unknown' })];
+        const malformed = api.add('files.read');
+        api.handle.prepare('UPDATE ec_host_operations SET result=? WHERE id=?').run('{not-json', malformed);
+        retained.push(malformed);
+        api.handle.exec('DROP INDEX ec_host_operations_observation_retention');
+        api.handle.exec('DROP INDEX ec_host_operations_important_history');
+        api.handle.exec('ALTER TABLE ec_host_operations DROP COLUMN observation_success');
+        api.closeDb();
+
+        expect(api.pruneHostObservations()).toBe(2);
+        expect(api.getHostOperationStatus(read)).toBeNull();
+        expect(api.getHostOperationStatus(serializedRead)).toBeNull();
+        const handle = api.db();
+        expect((handle.prepare('SELECT id FROM ec_host_operations').all() as { id: string }[]).map(row => row.id).sort()).toEqual(retained.sort());
+        // Subsequent boots honor the recorded classification instead of repeatedly
+        // deriving it from historical rows that were deliberately preserved.
+        handle.prepare("UPDATE ec_host_operations SET result='{}' WHERE id=?").run(malformed);
+        api.closeDb();
+        expect(api.pruneHostObservations()).toBe(0);
+        expect(api.getHostOperationStatus(malformed)).not.toBeNull();
+    });
+
+    it('returns the current shared retention setting to an agent on every heartbeat', async () => {
+        const api = await setup();
+        const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js');
+        for (const observationKeepDays of [7, 0]) {
+            api.saveFleetSettings({ ...api.fleetSettings(), observationKeepDays });
+            const response = await POST({ request: new Request('https://ec.test/api/agent/heartbeat', {
+                method: 'POST', headers: { authorization: `Bearer ${api.credentials.credential}`, 'Content-Type': 'application/json' },
+                body: JSON.stringify({ inventory })
+            }) });
+            expect(response.status).toBe(200);
+            expect(await response.json()).toMatchObject({ observationKeepDays, heartbeatSeconds: 15 });
+        }
+    });
+});

tests/unit/host-operation-history.test.ts

@@ -30,27 +30,43 @@ async function seedHistory(count = 53) {
     return { ...hosts, handle, ids, first, second, deleted };
 }
 
+async function operationHistory() {
+    const api = await seedHistory(0);
+    function add(kind: string, options: { payload?: Record<string, unknown>; state?: string; result?: unknown; error?: string } = {}) {
+        const operation = api.queueHostOperation(api.first.id, kind, options.payload ?? {});
+        if (options.state !== 'pending') api.reportHostOperationStatus(api.first.id, operation.id, {
+            state: options.state ?? 'completed', result: options.result, error: options.error
+        });
+        return operation.id;
+    }
+    return { ...api, add };
+}
+
 describe('host operation history', () => {
     it('pages all hosts in stable newest-first order, retaining deleted-host records', async () => {
         const api = await seedHistory();
-        const pages = [1, 2, 3].map(page => api.listHostOperationHistory(page));
+        const pages = [1, 2, 3].map(page => api.listHostOperationHistory(page, 'all'));
         expect(pages.map(page => page.operations.length)).toEqual([25, 25, 3]);
-        pages.forEach((page, index) => expect(page).toMatchObject({ page: index + 1, pageSize: 25, total: 53, totalPages: 3 }));
+        pages.forEach((page, index) => expect(page).toMatchObject({ page: index + 1, pageSize: 25, total: 53, totalPages: 3, filter: 'all' }));
         const operations = pages.flatMap(page => page.operations);
         expect(operations.map(operation => operation.id)).toEqual([...api.ids].reverse());
         expect(operations.find(operation => operation.hostId === api.first.id)?.hostName).toBe('Alpha host');
         expect(operations.find(operation => operation.hostId === api.second.id)?.hostName).toBe('Beta host');
         expect(operations.filter(operation => operation.hostId === api.deleted.id).length).toBeGreaterThan(0);
         expect(operations.filter(operation => operation.hostId === api.deleted.id).every(operation => operation.hostName === null)).toBe(true);
-        expect(api.listHostOperationHistory(999)).toEqual(pages[2]);
+        expect(api.listHostOperationHistory(999, 'all')).toEqual(pages[2]);
     });
 
     it('bounds reads in SQLite using the history index and omits private payloads and results', async () => {
         const api = await seedHistory(1003);
         const prepare = vi.spyOn(api.handle, 'prepare');
-        const history = api.listHostOperationHistory(2);
+        const history = api.listHostOperationHistory(2, 'all');
+        api.listHostOperationHistory(1, 'important');
+        api.listHostOperationHistory(1, 'routine');
+        const statements = prepare.mock.calls.map(([sql]) => sql);
         const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o LEFT JOIN'))!;
         prepare.mockRestore();
+        expect(statements.every(sql => !/\b(?:payload|result|json_extract|ec_host_operation_payloads)\b/.test(sql))).toBe(true);
         const projection = query.split('FROM ec_host_operations')[0];
         expect(projection).not.toContain('payload');
         expect(projection).not.toContain('result');
@@ -65,28 +81,167 @@ describe('host operation history', () => {
         expect(JSON.stringify(history)).not.toContain('secret');
     });
 
+    it('seeks through the selected history index without sorting or reading private payloads', async () => {
+        const api = await seedHistory(1003);
+        for (const filter of ['important', 'routine'] as const) {
+            const prepare = vi.spyOn(api.handle, 'prepare');
+            api.listHostOperationHistory(2, filter);
+            const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o LEFT JOIN'))!;
+            prepare.mockRestore();
+            const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(25, 25) as { detail: string }[];
+            expect(plan.some(row => row.detail.includes(`USING INDEX ec_host_operations_${filter}_history`))).toBe(true);
+            expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false);
+            expect(query).not.toMatch(/\b(?:payload|result|json_extract)\b/);
+        }
+    });
+
     it('returns an empty first page and normalizes invalid internal page numbers', async () => {
         const api = await seedHistory(0);
-        const empty = { operations: [], page: 1, pageSize: 25, total: 0, totalPages: 1 };
+        const empty = { operations: [], page: 1, pageSize: 25, total: 0, totalPages: 1, filter: 'important' };
         expect(api.listHostOperationHistory()).toEqual(empty);
         for (const page of [0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, 999]) {
             expect(api.listHostOperationHistory(page)).toEqual(empty);
         }
     });
 
+    it('defaults to actions and errors, with independent counts and pagination for every filter', async () => {
+        const api = await operationHistory();
+        const actions = Array.from({ length: 26 }, () => api.add('instance.restart'));
+        actions.push(api.add('rcon.execute', { payload: { command: 'map_restart' } }));
+        const successful = Array.from({ length: 26 }, () => api.add('files.manifest', { result: { files: [] } }));
+        const pending = ['pending', 'accepted', 'running'].map(state => api.add('files.read', { state }));
+        const failures = ['failed', 'unknown'].map(state => api.add('files.read', { state, error: 'Observation unavailable' }));
+        const negative = [
+            api.add('rcon.execute', { payload: { command: 'status' }, result: { ok: false, reason: 'timeout' } }),
+            api.add('instance.inspect', { result: { healthy: false, state: 'stopped' } }),
+            api.add('files.read', { result: { error: 'Read was incomplete' } }),
+            api.add('files.stat', { error: 'Could not inspect the file' })
+        ];
+        const ids = [...actions, ...successful, ...pending, ...failures, ...negative];
+        for (const [filter, matching] of [
+            ['important', [...actions, ...failures, ...negative]],
+            ['routine', [...successful, ...pending, ...failures, ...negative]],
+            ['all', ids]
+        ] as const) {
+            const pages = Array.from({ length: Math.ceil(matching.length / 25) }, (_, index) => api.listHostOperationHistory(index + 1, filter));
+            expect(pages[0]).toMatchObject({ filter, total: matching.length, totalPages: pages.length, pageSize: 25 });
+            expect(pages.flatMap(page => page.operations.map(operation => operation.id))).toEqual([...matching].reverse());
+            expect(api.listHostOperationHistory(999, filter)).toEqual(pages.at(-1));
+        }
+        expect(api.listHostOperationHistory()).toEqual(api.listHostOperationHistory(1, 'important'));
+        const routine = api.listHostOperationHistory(1, 'routine').operations;
+        expect(routine.every(operation => operation.routine)).toBe(true);
+        expect(api.listHostOperationHistory(2, 'important').operations.every(operation => !operation.routine)).toBe(true);
+    });
+
+    it('keeps manual RCON commands visible and only exposes safe read command names', async () => {
+        const api = await operationHistory();
+        const reads = ['status', 'getstatus', 'version'].map(command => api.add('rcon.execute', { payload: { command } }));
+        const manual = api.add('rcon.execute', { payload: { command: 'say credential-secret' } });
+        const combined = api.add('rcon.execute', { payload: { command: 'status; map_restart' } });
+        const important = api.listHostOperationHistory();
+        expect(important.operations.map(operation => operation.id)).toEqual([combined, manual]);
+        expect(important.operations.every(operation => !operation.routine && operation.targetPath === null)).toBe(true);
+        const routine = api.listHostOperationHistory(1, 'routine');
+        expect(routine.operations.map(operation => operation.id)).toEqual([...reads].reverse());
+        expect(routine.operations.map(operation => operation.targetPath)).toEqual(['version', 'getstatus', 'status']);
+        expect(JSON.stringify(api.listHostOperationHistory(1, 'all'))).not.toContain('credential-secret');
+    });
+
+    it('persists server, target, and source summaries without depending on current instance records', async () => {
+        const api = await operationHistory();
+        const { withOperationSource } = await import('../../src/lib/server/hosts/operation-context.js');
+        api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
+            VALUES('history-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)`)
+            .run(api.first.id, new Date().toISOString());
+        const read = withOperationSource('Routine status', () => api.queueHostOperation(api.first.id, 'files.read', {
+            path: 'configs/server.cfg', credential: 'payload-secret'
+        }, { instanceId: 'history-instance', generation: 1 }));
+        const manifest = withOperationSource('Watchdog drift scan', () => api.queueHostOperation(api.first.id, 'files.manifest', {
+            roots: ['scripts', 'configs', 'plugins'], ignoreNames: [], paths: ['configs/server.cfg']
+        }, { instanceId: 'history-instance', generation: 1 }));
+        api.reportHostOperationStatus(api.first.id, read.id, { state: 'completed', result: { contentBase64: 'result-secret' } });
+        api.handle.prepare('DELETE FROM native_instances WHERE id=?').run('history-instance');
+        const history = api.listHostOperationHistory(1, 'all');
+        expect(history.operations.find(operation => operation.id === read.id)).toMatchObject({
+            serverName: 't5_zombie_theater_1', targetPath: 'configs/server.cfg', source: 'Routine status', routine: true
+        });
+        expect(history.operations.find(operation => operation.id === manifest.id)).toMatchObject({
+            serverName: 't5_zombie_theater_1', targetPath: 'scripts, configs, plugins', source: 'Watchdog drift scan', routine: true
+        });
+        expect(JSON.stringify(history)).not.toContain('secret');
+    });
+
+    it('refuses unsafe target summaries and never derives labels from arbitrary payload fields', async () => {
+        const api = await operationHistory();
+        const unsafe = ['/etc/private-secret', '../private-secret', 'configs/../../private-secret', 'configs/line\nprivate-secret', 'configs\\private-secret'];
+        const ids = unsafe.map(path => api.add('files.read', { payload: { path, command: 'command-secret', name: 'name-secret' } }));
+        ids.push(api.add('files.manifest', { payload: { roots: ['scripts', '../private-secret'] } }));
+        ids.push(api.add('instance.restart', { payload: { path: 'configs/private-secret', command: 'command-secret' } }));
+        const history = api.listHostOperationHistory(1, 'all');
+        expect(history.operations).toHaveLength(ids.length);
+        expect(history.operations.every(operation => operation.targetPath === null && operation.serverName === null)).toBe(true);
+        expect(JSON.stringify(history)).not.toContain('secret');
+    });
+
+    it('backfills safe legacy summaries from immutable payloads while leaving source unknown', async () => {
+        const api = await operationHistory();
+        const { closeDb } = await import('../../src/lib/server/db/index.js');
+        api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
+            VALUES('legacy-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)`)
+            .run(api.first.id, new Date().toISOString());
+        const read = api.queueHostOperationSerialized(api.first.id, 'files.read', Buffer.from(JSON.stringify({
+            path: 'configs/server.cfg', credential: 'payload-secret'
+        })), { instanceId: 'legacy-instance', generation: 1 }, { path: 'configs/routing-secret' });
+        api.reportHostOperationStatus(api.first.id, read.id, { state: 'completed', result: { credential: 'result-secret' } });
+        const command = api.queueHostOperationSerialized(api.first.id, 'rcon.execute', Buffer.from('{"command":"status"}'), {}, { command: 'command-secret' });
+        const manual = api.add('rcon.execute', { payload: { command: 'say command-secret' } });
+        const unsafe = api.add('files.read', { payload: { path: '../private-secret' } });
+        api.handle.exec(`DROP INDEX ec_host_operations_routine_history;
+            DROP INDEX ec_host_operations_important_history;
+            ALTER TABLE ec_host_operations DROP COLUMN routine;
+            ALTER TABLE ec_host_operations DROP COLUMN server_name;
+            ALTER TABLE ec_host_operations DROP COLUMN target_path;
+            ALTER TABLE ec_host_operations DROP COLUMN source;`);
+        closeDb();
+        const history = api.listHostOperationHistory(1, 'all');
+        expect(history.operations).toHaveLength(4);
+        expect(history.operations.find(operation => operation.id === read.id)).toMatchObject({
+            routine: true, serverName: 't5_zombie_theater_1', targetPath: 'configs/server.cfg', source: null
+        });
+        expect(history.operations.find(operation => operation.id === command.id)).toMatchObject({ routine: true, targetPath: 'status', source: null });
+        expect(history.operations.find(operation => operation.id === manual)).toMatchObject({ routine: false, targetPath: null, source: null });
+        expect(history.operations.find(operation => operation.id === unsafe)).toMatchObject({ routine: true, targetPath: null, source: null });
+        expect(JSON.stringify(history)).not.toContain('secret');
+        closeDb();
+        expect(api.listHostOperationHistory(1, 'all')).toEqual(history);
+    });
+
     it('serves fixed-size API pages without caching or exposing private operation data', async () => {
         const api = await seedHistory();
         const { GET } = await import('../../src/routes/api/hosts/operations/+server.js');
-        const first = GET({ request: new Request('https://ec.test/api/hosts/operations') });
+        const first = GET({ request: new Request('https://ec.test/api/hosts/operations?filter=all') });
         expect(first.headers.get('cache-control')).toBe('no-store');
-        expect(await first.json()).toEqual(api.listHostOperationHistory(1));
+        expect(await first.json()).toEqual(api.listHostOperationHistory(1, 'all'));
-        const second = GET({ request: new Request('https://ec.test/api/hosts/operations?page=2&pageSize=500') });
+        const second = GET({ request: new Request('https://ec.test/api/hosts/operations?page=2&pageSize=500&filter=all') });
         const body = await second.json();
-        expect(body).toEqual(api.listHostOperationHistory(2));
+        expect(body).toEqual(api.listHostOperationHistory(2, 'all'));
         expect(body.operations).toHaveLength(25);
         expect(JSON.stringify(body)).not.toContain('secret');
-        const last = GET({ request: new Request('https://ec.test/api/hosts/operations?page=999') });
+        const last = GET({ request: new Request('https://ec.test/api/hosts/operations?page=999&filter=all') });
-        expect(await last.json()).toEqual(api.listHostOperationHistory(3));
+        expect(await last.json()).toEqual(api.listHostOperationHistory(3, 'all'));
+    });
+
+    it('serves the selected filter and defaults the API to actions and errors', async () => {
+        const api = await operationHistory();
+        const action = api.add('instance.restart');
+        const read = api.add('instance.inspect', { result: { healthy: true } });
+        const { GET } = await import('../../src/routes/api/hosts/operations/+server.js');
+        const request = (query = '') => GET({ request: new Request(`https://ec.test/api/hosts/operations${query}`) }).json();
+        expect(await request()).toEqual(api.listHostOperationHistory());
+        expect((await request()).operations.map((operation: { id: string }) => operation.id)).toEqual([action]);
+        expect((await request('?filter=routine')).operations.map((operation: { id: string }) => operation.id)).toEqual([read]);
+        expect(await request('?filter=all')).toEqual(api.listHostOperationHistory(1, 'all'));
     });
 
     it('rejects invalid API pages and keeps host management access restrictions', async () => {
@@ -95,6 +250,9 @@ describe('host operation history', () => {
         for (const page of ['', '0', '-1', '1.5', 'nope', 'Infinity', '9007199254740992']) {
             expect(() => GET({ request: new Request(`${url}?page=${page}`) })).toThrow(expect.objectContaining({ status: 400 }));
         }
+        for (const filter of ['', 'unknown', 'ALL', 'important%27%20OR%201=1']) {
+            expect(() => GET({ request: new Request(`${url}?filter=${filter}`) })).toThrow(expect.objectContaining({ status: 400 }));
+        }
         const blockedHeaders: HeadersInit[] = [
             { origin: 'https://other.test' },
             { 'sec-fetch-site': 'cross-site' },

tests/unit/native-drift-manifest.test.ts

@@ -0,0 +1,183 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+const server = 't5_zombie_theater_1';
+const scriptRoot = 'runtime/plutonium/storage/t5/raw/scripts/sp';
+const pluginRoot = 'runtime/plutonium/plugins';
+const hash = (content: Buffer) => createHash('sha256').update(content).digest('hex');
+let fixture: Fixture;
+
+beforeEach(async () => {
+    vi.resetModules();
+    fixture = await newFixture([]);
+});
+afterEach(async () => { await fixture.cleanup(); vi.restoreAllMocks(); });
+
+async function setup(capable = true) {
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    const host = hosts.createHost({ name: 'Drift manifest host' });
+    hosts.enrollHost(hosts.createEnrollment(host.id).token, {
+        protocolVersion: 1, capabilities: capable ? ['files.manifest'] : []
+    });
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const database = db();
+    database.prepare(`INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,created_by)
+        VALUES(?,'','t5','zombie_theater','zombie','theater','ui')`).run(server);
+    database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
+        VALUES(?,?,?,'library','image',28960,?)`).run(randomUUID(), server, host.id, new Date().toISOString());
+
+    const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
+    const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
+    const contents = new Map<string, Buffer>();
+    function promoted(kind: 'script' | 'plugin' | 'baseline', name: string, content: Buffer, targetRelPath?: string) {
+        const artifact = createScript({ name, kind, targetRelPath, scopeType: 'game', scopeValue: 't5', content });
+        database.prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(artifact.version.id, artifact.script.id);
+        return artifact;
+    }
+    const script = promoted('script', 'Managed script', Buffer.from('main() { wait 1; }'), 'managed.gsc');
+    const plugin = promoted('plugin', 'Managed plugin', Buffer.from('promoted plugin'), 'managed.dll');
+    const baseline = promoted('baseline', 'Baseline', Buffer.from('set sv_voice "1"\n'));
+    for (const [root, artifact] of [[scriptRoot, script], [pluginRoot, plugin]] as const) {
+        const { version } = artifact;
+        contents.set(`${root}/${artifact.script.target_rel_path}`, version.content);
+        recordScript(server, artifact.script.target_rel_path, version.id, version.sha256, version.size, 'test');
+    }
+    const { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } = await import('../../src/lib/server/fleet/server-config.js');
+    const config = getPromotedServerConfig('t5', 'zombie_theater', 'theater')!;
+    contents.set('configs/dedicated_sp.cfg', config);
+    recordScript(server, 'dedicated_sp.cfg', null, hash(config), config.length, 'test');
+    contents.set(`configs/${MAP_CONFIG_FILE}`, getServerMapConfig('t5', 'zombie_theater', 'theater')!);
+    // The runtime owns this generated file; neither scanner should classify it as drift.
+    contents.set('configs/dedicated.cfg', Buffer.from('runtime generated'));
+
+    const backend = await import('../../src/lib/server/backends/native.js');
+    const request = vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind, payload = {}) => {
+        if (kind === 'files.manifest') {
+            const roots = payload.roots as string[];
+            const ignored = payload.ignoreNames as string[];
+            return { files: [...contents].filter(([path]) => roots.some(root => path.startsWith(`${root}/`))
+                && !path.split('/').some(part => ignored.includes(part)))
+                .map(([path, content]) => ({ path, size: content.length, sha256: hash(content) })) } as never;
+        }
+        if (capable) throw new Error(`Manifest scan requested an extra remote operation: ${kind}`);
+        const path = String(payload.path);
+        const content = contents.get(path);
+        if (kind === 'files.read') return { contentBase64: content?.toString('base64') ?? null } as never;
+        if (kind === 'files.stat') {
+            const isDirectory = path === '.' || [...contents.keys()].some(file => file.startsWith(`${path}/`));
+            return { exists: content !== undefined || isDirectory, size: content?.length ?? 0,
+                mtime: new Date().toISOString(), isDirectory, isFile: content !== undefined } as never;
+        }
+        if (kind === 'files.list') {
+            const entries = new Map<string, { name: string; isDirectory: boolean; isFile: boolean }>();
+            for (const file of contents.keys()) {
+                if (!file.startsWith(`${path}/`)) continue;
+                const [name, ...nested] = file.slice(path.length + 1).split('/');
+                entries.set(name, { name, isDirectory: nested.length > 0, isFile: nested.length === 0 });
+            }
+            return { entries: [...entries.values()] } as never;
+        }
+        throw new Error(`Unexpected remote operation: ${kind}`);
+    });
+    const drift = await import('../../src/lib/server/fleet/drift.js');
+    return { ...drift, database, request, contents, script, plugin, baseline, promoted, mapConfigFile: MAP_CONFIG_FILE };
+}
+
+function expectOneManifest(request: Awaited<ReturnType<typeof setup>>['request']) {
+    expect(request).toHaveBeenCalledTimes(1);
+    expect(request.mock.calls[0][1]).toBe('files.manifest');
+    expect(request.mock.calls[0][2]).toEqual({
+        roots: [scriptRoot, 'configs', pluginRoot], ignoreNames: ['dedicated.cfg'],
+        paths: [`${scriptRoot}/managed.gsc`, `${pluginRoot}/managed.dll`, 'configs/dedicated_sp.cfg', 'configs/electric_cherry_map.cfg']
+    });
+}
+
+describe('native drift manifests', () => {
+    it('checks managed files, composed config, generated config, and untracked trees in one request', async () => {
+        const api = await setup();
+        expect(await api.scanServerDrift(server)).toEqual([]);
+        expectOneManifest(api.request);
+
+        api.request.mockClear();
+        expect(await api.prepareDriftCampaign(server)).toMatchObject({ plan: { files: [] }, files: [] });
+        expectOneManifest(api.request);
+    });
+
+    it('detects changed, missing, and nested untracked files from the same manifest', async () => {
+        const api = await setup();
+        api.contents.set(`${scriptRoot}/managed.gsc`, Buffer.alloc(0));
+        api.contents.delete(`${pluginRoot}/managed.dll`);
+        api.contents.set('configs/dedicated_sp.cfg', Buffer.from('changed config'));
+        const stray = Buffer.from('untracked bytes');
+        api.contents.set(`${scriptRoot}/custom/stray.gsc`, stray);
+
+        const changes = await api.scanServerDrift(server);
+        expect(changes).toHaveLength(4);
+        expect(changes).toEqual(expect.arrayContaining([
+            expect.objectContaining({ kind: 'script', relPath: 'managed.gsc', driftKind: 'replace', sha256: api.script.version.sha256 }),
+            expect.objectContaining({ kind: 'plugin', relPath: 'managed.dll', driftKind: 'missing', sha256: api.plugin.version.sha256 }),
+            expect.objectContaining({ kind: 'config', relPath: 'dedicated_sp.cfg', driftKind: 'replace' }),
+            expect.objectContaining({ kind: 'script', relPath: 'custom/stray.gsc', driftKind: 'untracked', size: stray.length, sha256: hash(stray) })
+        ]));
+        expectOneManifest(api.request);
+    });
+
+    it('still repairs missing inventory and generated map config', async () => {
+        const api = await setup();
+        api.database.prepare('DELETE FROM server_scripts WHERE server=? AND rel_path=?').run(server, 'managed.gsc');
+        api.contents.delete(`configs/${api.mapConfigFile}`);
+        expect(await api.scanServerDrift(server)).toEqual(expect.arrayContaining([
+            expect.objectContaining({ relPath: 'managed.gsc', driftKind: 'replace' }),
+            expect.objectContaining({ relPath: api.mapConfigFile, driftKind: 'missing' })
+        ]));
+        expectOneManifest(api.request);
+    });
+
+    it('compares explicitly managed ignored filenames without treating runtime-owned files as untracked', async () => {
+        const api = await setup();
+        api.promoted('plugin', 'Managed ignored name', Buffer.from('expected'), 'dedicated.cfg');
+        api.contents.set(`${pluginRoot}/dedicated.cfg`, Buffer.from('changed'));
+        expect(await api.scanServerDrift(server)).toEqual([
+            expect.objectContaining({ kind: 'plugin', relPath: 'dedicated.cfg', driftKind: 'replace' })
+        ]);
+        expect(api.request).toHaveBeenCalledTimes(1);
+        expect(api.request.mock.calls[0][2]).toMatchObject({ ignoreNames: [], paths: expect.arrayContaining([`${pluginRoot}/dedicated.cfg`]) });
+    });
+
+    it.each(['running', 'ready-promote'])('skips remote reads while a %s beta owns the server', async state => {
+        const api = await setup();
+        const beta = api.database.prepare("INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual',?)")
+            .run(api.script.script.id, api.script.version.id, state).lastInsertRowid;
+        api.database.prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')").run(beta, server);
+        expect(await api.scanServerDrift(server)).toEqual([]);
+        expect(api.request).not.toHaveBeenCalled();
+    });
+
+    it('preserves unavailable evidence as a failed scan without falling back to per-file reads', async () => {
+        const api = await setup();
+        api.request.mockRejectedValueOnce(new Error('Agent manifest unavailable'));
+        await expect(api.scanServerDrift(server)).rejects.toThrow('Agent manifest unavailable');
+        expectOneManifest(api.request);
+    });
+
+    it('rejects a malformed manifest instead of treating it as an empty tree', async () => {
+        const api = await setup();
+        api.request.mockResolvedValueOnce({} as never);
+        await expect(api.scanServerDrift(server)).rejects.toThrow();
+        expectOneManifest(api.request);
+    });
+
+    it('keeps legacy agents on the existing per-file scan and detects their drift', async () => {
+        const api = await setup(false);
+        api.contents.set(`${scriptRoot}/managed.gsc`, Buffer.from('old script'));
+        api.contents.set('configs/stray.cfg', Buffer.from('untracked config'));
+        expect(await api.scanServerDrift(server)).toEqual(expect.arrayContaining([
+            expect.objectContaining({ relPath: 'managed.gsc', driftKind: 'replace' }),
+            expect.objectContaining({ relPath: 'stray.cfg', driftKind: 'untracked' })
+        ]));
+        const operations = api.request.mock.calls.map(([, kind]) => kind);
+        expect(operations).not.toContain('files.manifest');
+        expect(operations).toEqual(expect.arrayContaining(['files.stat', 'files.list', 'files.read']));
+    });
+});

tests/unit/native-hosts.test.ts

@@ -119,7 +119,7 @@ describe('native host enrollment and scoped authentication', () => {
 
     it('separates observation and mutation delivery and wakes long polls on newly queued work', async () => {
         const api = await enrolled();
-        const inspect = api.queueHostOperation(api.host.id, 'host.inspect', {});
+        const inspect = api.queueHostOperation(api.host.id, 'files.manifest', { roots: ['configs'] });
         const restart = api.queueHostOperation(api.host.id, 'instance.restart', {});
         const status = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'status' });
         const command = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'map_restart' });

tests/unit/native-observations.test.ts

@@ -19,7 +19,7 @@ async function setup() {
 }
 
 describe('overlapping native observations', () => {
-    it.each(['files.read', 'files.list', 'files.stat', 'logs.read', 'instance.inspect'])('shares %s only until completion', async kind => {
+    it.each(['files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read', 'instance.inspect'])('shares %s only until completion', async kind => {
         const api = await setup();
         const requests = Array.from({ length: 4 }, () => api.nativeRequest(api.instance, kind, { path: 'same' }));
         const operations = api.listHostOperations(api.host.id);

tests/unit/operation-context.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, it } from 'vitest';
+import { currentOperationSource, withOperationSource } from '../../src/lib/server/hosts/operation-context.js';
+import { trackActivity, workerActivity } from '../../src/lib/server/pipeline/activity.js';
+
+describe('host operation source context', () => {
+    it('isolates overlapping asynchronous sources and restores the caller', async () => {
+        let release!: () => void;
+        const barrier = new Promise<void>(resolve => { release = resolve; });
+        const first = withOperationSource('GET /servers/[name]', async () => {
+            await barrier;
+            return currentOperationSource();
+        });
+        const second = withOperationSource('Status monitor', async () => {
+            release();
+            await Promise.resolve();
+            return currentOperationSource();
+        });
+        expect(currentOperationSource()).toBeNull();
+        expect(await Promise.all([first, second])).toEqual(['GET /servers/[name]', 'Status monitor']);
+        expect(currentOperationSource()).toBeNull();
+    });
+
+    it('lets explicit background work override incidental request context without leaking it', async () => {
+        await withOperationSource('POST /api/servers/[name]/refresh', async () => {
+            await withOperationSource('Status monitor', async () => {
+                await Promise.resolve();
+                expect(currentOperationSource()).toBe('Status monitor');
+            });
+            expect(currentOperationSource()).toBe('POST /api/servers/[name]/refresh');
+            expect(() => withOperationSource('Failing task', () => { throw new Error('failed'); })).toThrow('failed');
+            expect(currentOperationSource()).toBe('POST /api/servers/[name]/refresh');
+        });
+        expect(currentOperationSource()).toBeNull();
+    });
+
+    it('gives background activity a source while preserving its initiating source through nested activity', async () => {
+        await trackActivity('context-test', 'Reconciling campaigns', async () => {
+            await Promise.resolve();
+            expect(currentOperationSource()).toBe('Reconciling campaigns');
+            await trackActivity('context-test-child', 'Waiting for files.read', async () => {
+                expect(currentOperationSource()).toBe('Reconciling campaigns');
+            });
+        });
+        await withOperationSource('POST /api/servers/[name]/restart', () =>
+            trackActivity('context-test-request', 'Processing provisioning requests', async () => {
+                expect(currentOperationSource()).toBe('POST /api/servers/[name]/restart');
+            }));
+        expect(currentOperationSource()).toBeNull();
+        expect(workerActivity()).toEqual([]);
+    });
+
+    it('restores context and clears activity when asynchronous work rejects', async () => {
+        await withOperationSource('Manual maintenance check', async () => {
+            await expect(trackActivity('context-test-error', 'Waiting for files.read', async () => {
+                await Promise.resolve();
+                expect(currentOperationSource()).toBe('Manual maintenance check');
+                throw new Error('unavailable');
+            })).rejects.toThrow('unavailable');
+            expect(currentOperationSource()).toBe('Manual maintenance check');
+        });
+        expect(currentOperationSource()).toBeNull();
+        expect(workerActivity()).toEqual([]);
+    });
+});

tests/unit/server-status.test.ts

@@ -30,6 +30,51 @@ async function setup() {
 }
 
 describe('database fleet and container readiness', () => {
+    it('shares routine observations without renewing their timestamps and keeps manual probes fresh', async () => {
+        const { inspect, rcon, fanoutStatus, getServer } = await setup();
+        const { inspectForMonitoring, rconForMonitoring } = await import('../../src/lib/server/fleet/health-observations.js');
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const at = Date.parse('2026-10-05T12:00:00Z');
+        vi.setSystemTime(at);
+        inspect.mockResolvedValue({ ok: true, state: 'running' });
+        rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
+            status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        await inspectForMonitoring('managed');
+        await rconForMonitoring('managed');
+        vi.setSystemTime(at + 10_000);
+        await fanoutStatus();
+        expect(inspect).toHaveBeenCalledTimes(1);
+        expect(rcon).toHaveBeenCalledTimes(1);
+        expect(getServer('managed')).toMatchObject({ container_checked_at: new Date(at).toISOString(), last_seen_at: new Date(at).toISOString() });
+
+        const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
+        rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'manual failure' });
+        await POST({ params: { name: 'managed' } } as never);
+        expect(rcon).toHaveBeenCalledTimes(2);
+        vi.setSystemTime(at + 20_000);
+        await fanoutStatus();
+        expect(rcon).toHaveBeenCalledTimes(3);
+        expect(getServer('managed')).toMatchObject({ last_error: 'timeout:manual failure', last_seen_at: new Date(at).toISOString() });
+    });
+
+    it('does not overwrite a newer manual observation with an earlier background reply', async () => {
+        const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const at = Date.parse('2026-10-05T12:00:00Z');
+        vi.setSystemTime(at);
+        inspect.mockResolvedValue({ ok: true, state: 'running' });
+        let release!: (result: Awaited<ReturnType<typeof rcon>>) => void;
+        rcon.mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
+        const pending = fanoutStatus();
+        await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
+        vi.setSystemTime(at + 10_000);
+        db.prepare("UPDATE servers SET last_checked_at=?,last_error='timeout:manual failure' WHERE name='managed'").run(new Date().toISOString());
+        release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old reply',
+            status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        await pending;
+        expect(getServer('managed')).toMatchObject({ last_error: 'timeout:manual failure', last_seen_at: null, last_checked_at: new Date(at + 10_000).toISOString() });
+    });
+
     it('refreshes a ten-server fleet while one check is slow, without duplicating or exceeding four checks', async () => {
         const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
         const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
@@ -391,6 +436,7 @@ describe('database fleet and container readiness', () => {
 
     it.each(['unknown', undefined, 'throw'])('automatically recovers through RCON when inspection returns %s', async state => {
         const { inspect, rcon, fanoutStatus, getServer } = await setup();
+        vi.useFakeTimers({ toFake: ['Date'] });
         const { GET } = await import('../../src/routes/api/servers/+server.js');
         const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
         if (state === 'throw') inspect.mockRejectedValue(new Error('Inspection timed out'));
@@ -409,6 +455,7 @@ describe('database fleet and container readiness', () => {
         })] }));
 
         rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
+        vi.setSystemTime(Date.now() + 30_000);
         await fanoutStatus();
         expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' });
     });