Fix native health polling and isolate file operation traffic

agent/ec_agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.3"
+VERSION = "1.4.5"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -1092,7 +1092,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1126,11 +1126,20 @@ class Agent:
         bind = saved.get("bindAddress", "0.0.0.0")
         host = "127.0.0.1" if bind in ("0.0.0.0", "") else "::1" if bind == "::" else bind
         family = socket.AF_INET6 if ":" in host else socket.AF_INET
+        # Retry only the read-only occupancy query. A missing UDP reply must
+        # never cause commands with effects to be replayed.
+        for attempt in range(3 if command == "status" else 1):
+            result = self.rcon_attempt(family, host, int(saved["port"]), password, command)
+            if result["ok"] or result.get("reason") not in ("timeout", "unreachable") or attempt == 2 or command != "status":
+                return result
+            time.sleep(0.25 * 2 ** attempt)
+
+    def rcon_attempt(self, family, host, port, password, command):
         chunks = []
         with socket.socket(family, socket.SOCK_DGRAM) as sock:
             sock.settimeout(3)
             try:
-                sock.connect((host, int(saved["port"])))
+                sock.connect((host, port))
                 sock.send(b"\xff\xff\xff\xffrcon " + password.encode() + b" " + command.encode() + b"\n")
                 while len(chunks) < 64 and sum(map(len, chunks)) < 1024 * 1024:
                     chunks.append(sock.recv(65535))
@@ -2016,7 +2025,11 @@ class Agent:
         key = operation.get("instanceId")
         return self.instance_locks.setdefault(key, threading.RLock()) if key else contextlib.nullcontext()
 
-    def read_loop(self):
+    @staticmethod
+    def health_operation(operation):
+        return operation.get("kind") == "instance.inspect" or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
+
+    def read_loop(self, lane="read"):
         # Each lane owns its SQLite connection and never writes executing_id,
         # checkpoints, or current_operation used by the mutation state machine.
         journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
@@ -2026,10 +2039,13 @@ class Agent:
                     if self.update_pending():
                         self.stopping.wait(3)
                         continue
-                    response = self.request("GET", "/api/agent/operations?lane=read&wait=20")
+                    response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20")
+                    handled = False
                     for operation in response.get("operations", []):
                         if self.stopping.is_set() or not self.read_operation(operation):
                             continue
+                        if lane == "health" and not self.health_operation(operation):
+                            continue
                         with self.read_claim_lock:
                             if operation["id"] in self.read_claims:
                                 continue
@@ -2038,19 +2054,26 @@ class Agent:
                             with self.activity(read_only=True) as acquired:
                                 if not acquired or self.update_pending():
                                     continue
-                                with self.read_gate.read(), self.instance_lock(operation):
+                                row, _ = journal.accept(operation)
-                                    row, _ = journal.accept(operation)
+                                if row["state"] in ("accepted", "unknown", "running"):
-                                    if row["state"] in ("accepted", "unknown", "running"):
+                                    self.report(journal.update(operation["id"], "running"))
-                                        self.report(journal.update(operation["id"], "running"))
+                                    with self.read_gate.read(), self.instance_lock(operation):
                                         try:
                                             result = self.dispatch(operation)
                                             row = journal.update(operation["id"], "completed", result)
                                         except Exception as err:
                                             row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
-                                    self.report(row)
+                                # Receipt delivery can take seconds; it must not
+                                # hold the instance lock needed by health checks.
+                                self.report(row)
+                                handled = True
                         finally:
                             with self.read_claim_lock:
                                 self.read_claims.discard(operation["id"])
+                    if response.get("operations") and not handled:
+                        # A legacy controller may return another lane's work or
+                        # a read already claimed locally. Avoid a busy poll loop.
+                        self.stopping.wait(0.25)
                 except (AgentError, OSError, ValueError, KeyError, sqlite3.Error):
                     self.stopping.wait(3)
         finally:
@@ -2109,7 +2132,8 @@ class Agent:
                 self.stopping.wait(3)
         heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
         heartbeats.start()
-        readers = [threading.Thread(target=self.read_loop, daemon=True) for _ in range(2)]
+        # Preserve two concurrent health checks even while files are being read.
+        readers = [threading.Thread(target=self.read_loop, args=(lane,), daemon=True) for lane in ("health", "health", "read")]
         for reader in readers:
             reader.start()
         while not self.stopping.is_set():

agent/tests/test_agent.py

@@ -6,7 +6,7 @@ from pathlib import Path
 import tempfile
 import threading
 import unittest
-from unittest.mock import mock_open, patch
+from unittest.mock import MagicMock, call, mock_open, patch
 
 spec = importlib.util.spec_from_file_location("ec_agent", Path(__file__).resolve().parents[1] / "ec_agent.py")
 module = importlib.util.module_from_spec(spec)
@@ -995,6 +995,64 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(result["players"], 1)
         self.assertTrue(result["ok"])
 
+    @staticmethod
+    def rcon_socket(*replies):
+        sock = MagicMock()
+        sock.__enter__.return_value = sock
+        sock.recv.side_effect = replies
+        return sock
+
+    def test_udp_status_recovers_after_transient_failures(self):
+        self.create()
+        for error in (module.socket.timeout(), OSError("unreachable")):
+            with self.subTest(error=type(error).__name__):
+                failed = self.rcon_socket(error)
+                # Keep all response packets, including occupied player rows.
+                recovered = self.rcon_socket(
+                    b"\xff\xff\xff\xffprint\nmap: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n",
+                    b"\xff\xff\xff\xffprint\n0 42 CNCT abc player\n", module.socket.timeout())
+                with patch.object(module.socket, "socket", side_effect=[failed, recovered]) as sockets, patch.object(module.time, "sleep") as sleep:
+                    result = self.agent.rcon(INSTANCE, 1, "status")
+                self.assertTrue(result["ok"])
+                self.assertEqual(result["players"], 1)
+                self.assertEqual(sockets.call_count, 2)
+                sleep.assert_called_once_with(0.25)
+                failed.__exit__.assert_called_once()
+                recovered.__exit__.assert_called_once()
+
+    def test_udp_status_retries_are_bounded(self):
+        self.create()
+        attempts = [self.rcon_socket(module.socket.timeout()) for _ in range(3)]
+        with patch.object(module.socket, "socket", side_effect=attempts) as sockets, patch.object(module.time, "sleep") as sleep:
+            result = self.agent.rcon(INSTANCE, 1, "status")
+        self.assertFalse(result["ok"])
+        self.assertEqual(result["reason"], "timeout")
+        self.assertEqual(sockets.call_count, 3)
+        self.assertEqual(sleep.call_args_list, [call(0.25), call(0.5)])
+
+    def test_udp_status_does_not_retry_bad_replies_or_partial_tables(self):
+        self.create()
+        for packet in (b"wrong listener", b"\xff\xff\xff\xffprint\nBad rcon_password",
+                       b"\xff\xff\xff\xffprint\nmap: mp_kino\nnum score ping guid name\n"):
+            with self.subTest(packet=packet):
+                sock = self.rcon_socket(packet, module.socket.timeout())
+                with patch.object(module.socket, "socket", return_value=sock) as sockets, patch.object(module.time, "sleep") as sleep:
+                    result = self.agent.rcon(INSTANCE, 1, "status")
+                self.assertIsNone(result["players"])
+                sockets.assert_called_once()
+                sleep.assert_not_called()
+
+    def test_udp_commands_with_effects_are_never_replayed(self):
+        self.create()
+        for command in ("map_restart", "kick 0", "version"):
+            with self.subTest(command=command):
+                sock = self.rcon_socket(module.socket.timeout())
+                with patch.object(module.socket, "socket", return_value=sock) as sockets, patch.object(module.time, "sleep") as sleep:
+                    result = self.agent.rcon(INSTANCE, 1, command)
+                self.assertEqual(result["reason"], "timeout")
+                sockets.assert_called_once()
+                sleep.assert_not_called()
+
     def test_t6_bot_column_allows_empty_restart_but_holds_players_and_partial_replies(self):
         self.create()
         header = "map: zm_nuked\nnum score bot ping guid                             name             lastmsg address               qport  rate\n"

agent/tests/test_update_integration.py

@@ -118,6 +118,44 @@ class UpdateIntegrationTests(unittest.TestCase):
             self.agent.read_loop()
             dispatch.assert_not_called()
 
+    def test_health_lane_runs_while_file_receipt_for_same_instance_is_slow(self):
+        receipt_waiting, release_receipt, health_done = threading.Event(), threading.Event(), threading.Event()
+        file_op = {"id": "file-one", "kind": "files.read", "instanceId": "same", "payload": {}}
+        health_op = {"id": "health-one", "kind": "instance.inspect", "instanceId": "same", "payload": {}}
+        errors = []
+        def request(method, path, body=None):
+            if method == "GET":
+                return {"operations": [health_op if "lane=health" in path else file_op]}
+            if path.endswith("file-one") and body["state"] == "completed":
+                receipt_waiting.set()
+                release_receipt.wait(3)
+            return {}
+        def dispatch(operation):
+            if operation["id"] == "health-one":
+                health_done.set()
+                self.agent.stopping.set()
+            return {"ok": True}
+        def run(lane):
+            try: self.agent.read_loop(lane)
+            except Exception as error: errors.append(error)
+        self.agent.request, self.agent.dispatch = request, dispatch
+        files = threading.Thread(target=run, args=("read",))
+        health = threading.Thread(target=run, args=("health",))
+        files.start()
+        try:
+            self.assertTrue(receipt_waiting.wait(2))
+            health.start()
+            self.assertTrue(health_done.wait(2), "Health waited for a file receipt")
+        finally:
+            self.agent.stopping.set()
+            release_receipt.set()
+            files.join(3)
+            if health.ident is not None: health.join(3)
+        self.assertFalse(files.is_alive())
+        self.assertFalse(health.is_alive())
+        self.assertEqual(errors, [])
+        self.assertEqual(self.agent.journal.get("health-one")["state"], "completed")
+
     def test_instance_mutations_and_global_mutations_exclude_observations(self):
         operation = {"instanceId": "one"}
         entered = threading.Event()

docs/maintenance.md

@@ -126,6 +126,11 @@ Drift includes missing promoted files, changed bytes, and extra files in managed
 script, config, and plugin directories. Config comparison includes the composed
 baseline, overlays, and the selected map. Unmanaged default config is exempt until
 a baseline is promoted. Missing mounts are scan errors, not empty directories.
+Routine drift scans run every five minutes per server, reducing file traffic
+while health observations continue every minute. **Check now**, a replacement
+registration, or a new lifecycle submission bypasses the drift interval. Failed
+file scans retry on the next observer pass; post-rebuild verification still reads
+fresh files.
 
 Drift repair creates a Campaign capturing observed differences and requests a
 Provisioning rebuild. The Campaign observes that rebuild without requesting

docs/native-hosts.md

@@ -5,6 +5,13 @@ instance never uses Komodo or a filesystem mounted into the controller. The
 controller owns desired state, campaigns, collections, safety queues and history;
 the agent owns local execution and its durable operation journal.
 
+Fleet health checks query RCON even when container inspection is unavailable. A
+fresh successful RCON reply can establish game health while container state stays
+unknown; confirmed stopped, unhealthy, and restarting states still take precedence.
+Agent 1.4.4 retries transient failures of the read-only `status` query up to three
+times, with 250 ms and 500 ms delays. Commands that change the game are not retried.
+Status replies still expire normally, and native timeout diagnostics are retained.
+
 ## Install and launch
 
 1. Run the controller with persistent `/data` storage. Set `EC_PUBLIC_URL` to its
@@ -479,9 +486,14 @@ of which task delayed a previous deployment.
 
 Campaign, beta and maintenance observers run independently of provisioning.
 New requests and completed agent commands wake provisioning; an arrival during a
-pass schedules another pass. File drift scans use bounded batches of four reads
+pass schedules another pass. Routine file drift scans run every five minutes;
-per scan to reduce agent polling round trips. Startup and disruption safety gates
+manual checks bypass that interval. Overlapping identical native observations
-remain enforced at command admission.
+share one operation until it completes, without caching results for later checks.
+Agent 1.4.5 reserves two observation workers for instance health and read-only RCON,
+and one for files, logs, and host inventory. Background reads arrive in batches of
+up to four. File receipt uploads release the instance lock before contacting the
+controller. Older agents retain the shared read lane until upgraded. Startup and
+disruption safety gates remain enforced at command admission.
 
 Older completed operations whose progress snapshots were already discarded
 cannot recover missing check results from this change. Their surviving events

src/lib/server-observation.ts

@@ -43,7 +43,8 @@ export function selectServerObservation<T extends ServerObservation>(server: T,
     if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
     const raw = server.rcon_status ?? server;
     const result: T = { ...server, ...raw, telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
-    if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running') || (!server.container_state && server.health === 'offline')) {
+    const containerUnknown = server.container_state === 'unknown';
+    if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) {
         return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null };
     }
     const checked = server.container_checked_at ?? server.last_checked_at ?? raw.last_seen_at;
@@ -69,7 +70,9 @@ export function selectServerObservation<T extends ServerObservation>(server: T,
     }
     const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
     const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !raw.last_error;
-    const containerExpired = !!server.container_checked_at && !containerFresh;
+    // A current authenticated RCON reply can establish game liveness when
+    // inspection is unavailable, without claiming the container was inspected.
+    const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh;
     const age = now - Date.parse(raw.last_seen_at ?? '');
     return { ...result, health: fresh && !containerExpired ? 'online'
         : after && age >= 0 && age <= 120_000 ? 'stale' : 'offline' };

src/lib/server/backends/native.ts

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { db } from '../db/index.js';
 import { getHost, queueHostOperation, waitHostOperation } from '../hosts/index.js';
 import { trackActivity } from '../pipeline/activity.js';
+import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
 
 export interface NativeInstance {
     id: string; server: string; host_id: string; generation: number; library_id: string;
@@ -17,10 +18,27 @@ export function nativeDeployment(id: string): boolean {
     return !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id = ?').get(id);
 }
 
-/** Only the agent receives host paths, Docker access, or game credentials. */
+const observations = new Map<string, Promise<unknown>>();
+const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'logs.read']);
+
+/** Share overlapping observations only, never completed results or mutations. */
 export async function nativeRequest<T = Record<string, unknown>>(instance: NativeInstance, kind: string, payload: Record<string, unknown> = {}, key?: string): Promise<T> {
     const host = getHost(instance.host_id);
     if (!host || host.connectionState !== 'online') throw new Error('Host agent is offline; instance state is unknown.');
+    const readOnly = readKinds.has(kind) || (kind === 'rcon.execute' && ['status', 'getstatus', 'version'].includes(String(payload.command)));
+    if (key || !readOnly) return executeRequest<T>(instance, kind, payload, key);
+    // A probe after a lifecycle submission must not join an old-process read.
+    const identity = JSON.stringify([instance.host_id, instance.id, instance.generation, instance.server,
+        latestGameSubmission(instance.server), kind, payload]);
+    const pending = observations.get(identity);
+    if (pending) return pending as Promise<T>;
+    const request = executeRequest<T>(instance, kind, payload).finally(() => { observations.delete(identity); });
+    observations.set(identity, request);
+    return request;
+}
+
+/** Only the agent receives host paths, Docker access, or game credentials. */
+async function executeRequest<T>(instance: NativeInstance, kind: string, payload: Record<string, unknown>, key?: string): Promise<T> {
     const operation = queueHostOperation(instance.host_id, kind, payload, {
         instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID()
     });

src/lib/server/fleet/health.ts

@@ -11,7 +11,7 @@ const STALE_MAX_MS = 120_000;
  */
 export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
     if (containerState === 'starting' || containerState === 'restarting') return 'starting';
-    if (containerState && containerState !== 'running') return 'offline';
+    if (containerState && containerState !== 'running' && containerState !== 'unknown') return 'offline';
     if (!lastSeenAt) return 'offline';
     const age = Date.now() - new Date(lastSeenAt).getTime();
     if (freshStatusTimestamp(lastSeenAt, Date.now())) {

src/lib/server/fleet/status.ts

@@ -49,7 +49,7 @@ export function serverObservation(server: ServerRow, evidence?: Evidence) {
         last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
     const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
     const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
-    const ready = !provisioning && (!state || state === 'running');
+    const ready = !provisioning && (!state || state === 'running' || state === 'unknown');
     const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
     return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
         status_source: null as 'telemetry' | 'rcon' | null,

src/lib/server/hosts/index.ts

@@ -348,7 +348,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
     queueMicrotask(() => workEvents.emit(hostId));
     return operation;
 }
-export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'write' = 'all'): HostOperation[] {
+export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'health' | 'write' = 'all'): HostOperation[] {
     const host = requireHost(hostId);
     if (host.mode === 'disabled' || !host.connected) return [];
     // Older agents hold activity.lock across the entire delivered batch, including
@@ -362,7 +362,13 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'wr
     // consults its durable journal before acknowledging or executing a replay.
     const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
         OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
-    const laneFilter = lane === 'all' ? '' : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}`;
+    const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
+    const healthLane = host.inventory.capabilities?.includes('operations.health-lane');
+    const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}`
+        : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;
+    // Health never queues behind a delivered file batch. Keep background reads
+    // in small batches to amortize delivery without handing out a long backlog.
+    const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25;
     return database().transaction(() => {
         const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
             ${laneFilter}
@@ -370,7 +376,7 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'wr
             AND (kind!='instance.cancel-create' OR ?)
             AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
                 AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
-            ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`)
+            ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
             .all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationRow[];
         const deliveredAt = now();
         for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);

src/lib/server/integrations/rcon.ts

@@ -382,7 +382,19 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
         const endpoint = { host: `agent:${native.host_id}`, port: native.port };
         const request = (async (): Promise<RconStatusResult> => {
             try {
-                const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'status' });
+                const reply = await nativeRequest<{ ok?: boolean; reason?: string; text: string }>(native, 'rcon.execute', { command: 'status' });
+                if (reply.ok === false) {
+                    const failures: Record<string, { reason: RconFailure; detail: string }> = {
+                        unconfigured: { reason: 'no-config', detail: 'Agent RCON password is not configured.' },
+                        unreachable: { reason: 'send-error', detail: 'Agent could not reach the local RCON endpoint.' },
+                        timeout: { reason: 'timeout', detail: 'Agent received no reply from the local RCON endpoint.' },
+                        authentication: { reason: 'bad-reply', detail: 'wrong RCON password' },
+                        'invalid-reply': { reason: 'bad-reply', detail: 'Agent RCON reply is missing the 0xff prefix.' }
+                    };
+                    return { ok: false, endpoint, ...(failures[reply.reason ?? ''] ?? {
+                        reason: 'bad-reply' as const, detail: 'Agent RCON query failed.'
+                    }) };
+                }
                 const players = parsePlayerCount(reply.text);
                 if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: reply.text.slice(0, 4000) };
                 const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text);

src/lib/server/maintenance/recovery.ts

@@ -16,6 +16,7 @@ import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident
 import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
 
 export const WATCHDOG_SCAN_MS = 60_000;
+export const WATCHDOG_DRIFT_SCAN_MS = 5 * 60_000;
 export const WATCHDOG_FAILURE_MS = 120_000;
 export const WATCHDOG_RECOVERY_MS = 300_000;
 const iso = () => new Date().toISOString();
@@ -240,7 +241,7 @@ async function observe(server: string, operationId?: string): Promise<Observatio
     return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
         reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
 }
-async function scanServer(server: string): Promise<void> {
+async function scanServer(server: string, forceScan: boolean): Promise<void> {
     const row = getServer(server);
     if (!row || row.desired_state === 'stopped') return;
     if (scanSuppressed(server) || protectedWork(server)) return;
@@ -265,8 +266,19 @@ async function scanServer(server: string): Promise<void> {
     }
     // Unknown container health is an observation failure, not proof that a rebuild is safe.
     if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
+    // File trees change far less often than liveness. Persist the last successful
+    // scan per registration/process so health checks do not rescan every minute.
+    // Manual checks and new lifecycle submissions always get fresh file evidence.
+    const driftKey = `watchdog_drift_scan:${server}`;
+    const instance = nativeInstance(server);
+    const identity = JSON.stringify([row.created_at, instance?.id, instance?.generation, latestGameSubmission(server)]);
+    let previous: { identity?: string; at?: string } = {};
+    try { previous = JSON.parse(setting(driftKey) ?? '{}'); } catch { /* rescan legacy/invalid state */ }
+    const age = Date.now() - Date.parse(previous?.at ?? '');
+    if (!forceScan && previous?.identity === identity && age >= 0 && age < WATCHDOG_DRIFT_SCAN_MS) return;
     const prepared = await prepareDriftCampaign(server);
     if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
+    setSetting(driftKey, JSON.stringify({ identity, at: iso() }));
     if (!prepared.plan.files.length) {
         const current = active(server, 'drift');
         if (current) transition(current.id, 'resolved', 'Server files match the promoted setup');
@@ -305,7 +317,7 @@ async function run(forceScan: boolean): Promise<void> {
     async function pump() {
         while (cursor < servers.length) {
             const server = servers[cursor++];
-            try { await scanServer(server.name); }
+            try { await scanServer(server.name, forceScan); }
             catch (err) {
                 if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`);
             }

src/lib/server/pipeline/worker.ts

@@ -150,10 +150,14 @@ async function refreshServerStatus(server: string): Promise<number | null> {
         if (unchanged()) {
             updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown';
             updates.container_checked_at = new Date().toISOString();
-            if (!container.ok) {
+            // An unavailable inspection is not evidence that the game stopped.
+            // Probe it just as a manual RCON check would; explicit lifecycle or
+            // unhealthy states still take precedence over game replies.
+            const unavailable = updates.container_state === 'unknown';
+            if (!container.ok && !unavailable) {
                 updates.last_error = null;
                 updates.last_reply_snippet = null;
-            } else if (!telemetryEvidence(srv.name)) {
+            } else if (unavailable || !telemetryEvidence(srv.name)) {
                 flush();
                 const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
                     reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));

src/routes/api/agent/operations/+server.ts

@@ -6,7 +6,7 @@ export async function GET({ request }: { request: Request }) {
     const host = authenticateAgent(request);
     reconcileHostPreparation(host.id);
     const params = new URL(request.url).searchParams;
-    const lane = params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all';
+    const lane = params.get('lane') === 'health' ? 'health' : params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all';
     let operations = pendingHostOperations(host.id, lane);
     if (!operations.length && params.get('wait') === '20') {
         await waitForHostWork(host.id, request.signal);

static/agent/ec-agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.3"
+VERSION = "1.4.5"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -1092,7 +1092,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1126,11 +1126,20 @@ class Agent:
         bind = saved.get("bindAddress", "0.0.0.0")
         host = "127.0.0.1" if bind in ("0.0.0.0", "") else "::1" if bind == "::" else bind
         family = socket.AF_INET6 if ":" in host else socket.AF_INET
+        # Retry only the read-only occupancy query. A missing UDP reply must
+        # never cause commands with effects to be replayed.
+        for attempt in range(3 if command == "status" else 1):
+            result = self.rcon_attempt(family, host, int(saved["port"]), password, command)
+            if result["ok"] or result.get("reason") not in ("timeout", "unreachable") or attempt == 2 or command != "status":
+                return result
+            time.sleep(0.25 * 2 ** attempt)
+
+    def rcon_attempt(self, family, host, port, password, command):
         chunks = []
         with socket.socket(family, socket.SOCK_DGRAM) as sock:
             sock.settimeout(3)
             try:
-                sock.connect((host, int(saved["port"])))
+                sock.connect((host, port))
                 sock.send(b"\xff\xff\xff\xffrcon " + password.encode() + b" " + command.encode() + b"\n")
                 while len(chunks) < 64 and sum(map(len, chunks)) < 1024 * 1024:
                     chunks.append(sock.recv(65535))
@@ -2016,7 +2025,11 @@ class Agent:
         key = operation.get("instanceId")
         return self.instance_locks.setdefault(key, threading.RLock()) if key else contextlib.nullcontext()
 
-    def read_loop(self):
+    @staticmethod
+    def health_operation(operation):
+        return operation.get("kind") == "instance.inspect" or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
+
+    def read_loop(self, lane="read"):
         # Each lane owns its SQLite connection and never writes executing_id,
         # checkpoints, or current_operation used by the mutation state machine.
         journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
@@ -2026,10 +2039,13 @@ class Agent:
                     if self.update_pending():
                         self.stopping.wait(3)
                         continue
-                    response = self.request("GET", "/api/agent/operations?lane=read&wait=20")
+                    response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20")
+                    handled = False
                     for operation in response.get("operations", []):
                         if self.stopping.is_set() or not self.read_operation(operation):
                             continue
+                        if lane == "health" and not self.health_operation(operation):
+                            continue
                         with self.read_claim_lock:
                             if operation["id"] in self.read_claims:
                                 continue
@@ -2038,19 +2054,26 @@ class Agent:
                             with self.activity(read_only=True) as acquired:
                                 if not acquired or self.update_pending():
                                     continue
-                                with self.read_gate.read(), self.instance_lock(operation):
+                                row, _ = journal.accept(operation)
-                                    row, _ = journal.accept(operation)
+                                if row["state"] in ("accepted", "unknown", "running"):
-                                    if row["state"] in ("accepted", "unknown", "running"):
+                                    self.report(journal.update(operation["id"], "running"))
-                                        self.report(journal.update(operation["id"], "running"))
+                                    with self.read_gate.read(), self.instance_lock(operation):
                                         try:
                                             result = self.dispatch(operation)
                                             row = journal.update(operation["id"], "completed", result)
                                         except Exception as err:
                                             row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
-                                    self.report(row)
+                                # Receipt delivery can take seconds; it must not
+                                # hold the instance lock needed by health checks.
+                                self.report(row)
+                                handled = True
                         finally:
                             with self.read_claim_lock:
                                 self.read_claims.discard(operation["id"])
+                    if response.get("operations") and not handled:
+                        # A legacy controller may return another lane's work or
+                        # a read already claimed locally. Avoid a busy poll loop.
+                        self.stopping.wait(0.25)
                 except (AgentError, OSError, ValueError, KeyError, sqlite3.Error):
                     self.stopping.wait(3)
         finally:
@@ -2109,7 +2132,8 @@ class Agent:
                 self.stopping.wait(3)
         heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
         heartbeats.start()
-        readers = [threading.Thread(target=self.read_loop, daemon=True) for _ in range(2)]
+        # Preserve two concurrent health checks even while files are being read.
+        readers = [threading.Thread(target=self.read_loop, args=(lane,), daemon=True) for lane in ("health", "health", "read")]
         for reader in readers:
             reader.start()
         while not self.stopping.is_set():

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.4.3",
+  "version": "1.4.5",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "9ca9bff1a42ab838b175329c6a8458e18053429efe2f050f477a9ad9c598410f",
+  "sha256": "bd4afa2ec3ab6a3022dba99d0580b4b6b1ba5622ae4138035e3e3862885a97bd",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/native-hosts.test.ts

@@ -15,6 +15,24 @@ async function enrolled() {
 }
 
 describe('native host enrollment and scoped authentication', () => {
+    it('reserves health delivery while a file backlog is pending and bounds background batches', async () => {
+        const api = await enrolled();
+        api.heartbeat(api.host.id, { ...inventory, capabilities: ['operations.health-lane'] });
+        const files = Array.from({ length: 30 }, (_, i) => api.queueHostOperation(api.host.id, 'files.read', { path: `file-${i}` }));
+        const status = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'status' });
+        const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {});
+        const mutation = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'map_restart' });
+        expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([status.id]);
+        api.reportHostOperation(api.host.id, status.id, { state: 'completed', result: {} });
+        expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([inspect.id]);
+        expect(api.pendingHostOperations(api.host.id, 'read').map(op => op.id)).toEqual(files.slice(0, 4).map(op => op.id));
+        expect(api.pendingHostOperations(api.host.id, 'write').map(op => op.id)).toEqual([mutation.id]);
+        // Rolling upgrades: older agents still receive health through read.
+        api.heartbeat(api.host.id, inventory);
+        files.forEach(file => api.reportHostOperation(api.host.id, file.id, { state: 'completed', result: {} }));
+        expect(api.pendingHostOperations(api.host.id, 'read').map(op => op.id)).toEqual([inspect.id]);
+    });
+
     it('separates observation and mutation delivery and wakes long polls on newly queued work', async () => {
         const api = await enrolled();
         const inspect = api.queueHostOperation(api.host.id, 'host.inspect', {});

tests/unit/native-observations.test.ts

@@ -0,0 +1,76 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { await fixture.cleanup(); });
+
+async function setup() {
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    const host = hosts.createHost({ name: 'Observation host' });
+    hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 });
+    const { db } = await import('../../src/lib/server/db/index.js');
+    db().prepare("INSERT INTO servers(name,komodo_name) VALUES('test','')").run();
+    db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
+        VALUES('instance','test',?,'library','image',28960,?)`).run(host.id, new Date().toISOString());
+    const native = await import('../../src/lib/server/backends/native.js');
+    const instance = native.nativeInstance('test')!;
+    return { ...hosts, ...native, db: db(), host, instance };
+}
+
+describe('overlapping native observations', () => {
+    it.each(['files.read', 'files.list', 'files.stat', 'logs.read', 'instance.inspect'])('shares %s only until completion', async kind => {
+        const api = await setup();
+        const requests = Array.from({ length: 4 }, () => api.nativeRequest(api.instance, kind, { path: 'same' }));
+        const operations = api.listHostOperations(api.host.id);
+        expect(operations).toHaveLength(1);
+        api.reportHostOperation(api.host.id, operations[0].id, { state: 'completed', result: { value: 'first' } });
+        expect(await Promise.all(requests)).toEqual(Array(4).fill({ value: 'first' }));
+        const fresh = api.nativeRequest(api.instance, kind, { path: 'same' });
+        const next = api.listHostOperations(api.host.id)[0];
+        expect(next.id).not.toBe(operations[0].id);
+        api.reportHostOperation(api.host.id, next.id, { state: 'completed', result: { value: 'changed' } });
+        expect(await fresh).toEqual({ value: 'changed' });
+    });
+
+    it('does not merge different paths, mutations, or reads across lifecycle submissions', async () => {
+        const api = await setup();
+        const requests = [api.nativeRequest(api.instance, 'files.read', { path: 'one' }),
+            api.nativeRequest(api.instance, 'files.read', { path: 'two' }),
+            api.nativeRequest(api.instance, 'rcon.execute', { command: 'map_restart' }),
+            api.nativeRequest(api.instance, 'rcon.execute', { command: 'map_restart' })];
+        api.db.prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('test',?)").run(new Date().toISOString());
+        requests.push(api.nativeRequest(api.instance, 'files.read', { path: 'one' }));
+        const operations = api.listHostOperations(api.host.id);
+        expect(operations).toHaveLength(5);
+        for (const operation of operations) api.reportHostOperation(api.host.id, operation.id, { state: 'completed', result: {} });
+        await Promise.all(requests);
+    });
+
+    it('releases failed observations so the next request can recover', async () => {
+        const api = await setup();
+        const first = api.nativeRequest(api.instance, 'files.read', { path: 'one' });
+        const rejected = expect(first).rejects.toThrow('unavailable');
+        api.reportHostOperation(api.host.id, api.listHostOperations(api.host.id)[0].id, { state: 'failed', error: 'unavailable' });
+        await rejected;
+        const next = api.nativeRequest(api.instance, 'files.read', { path: 'one' });
+        expect(api.listHostOperations(api.host.id)).toHaveLength(2);
+        api.reportHostOperation(api.host.id, api.listHostOperations(api.host.id)[0].id, { state: 'completed', result: {} });
+        await next;
+    });
+
+    it('keeps replacement generations separate and rejects the old result', async () => {
+        const api = await setup();
+        const old = api.nativeRequest(api.instance, 'files.read', { path: 'one' });
+        const rejected = expect(old).rejects.toThrow('Instance was replaced');
+        api.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run();
+        const current = api.nativeRequest(api.nativeInstance('test')!, 'files.read', { path: 'one' });
+        const operations = api.listHostOperations(api.host.id);
+        expect(operations).toHaveLength(2);
+        for (const operation of operations) api.reportHostOperation(api.host.id, operation.id, {
+            state: 'completed', result: { generation: operation.generation }
+        });
+        await rejected;
+        expect(await current).toEqual({ generation: 2 });
+    });
+});

tests/unit/rcon-status.test.ts

@@ -5,6 +5,7 @@ import { komodoConfigured, rconConfigured } from '../../src/lib/server/config.js
 import { getServer, type ServerRow } from '../../src/lib/server/fleet/servers.js';
 import { latestGameSubmission } from '../../src/lib/server/fleet/lifecycle-evidence.js';
 import { rconGetPlayerCount, rconPlutoniumVersion, rconStatusResult } from '../../src/lib/server/integrations/rcon.js';
+import { nativeInstance, nativeRequest } from '../../src/lib/server/backends/native.js';
 
 vi.mock('node:dgram', () => ({ createSocket: vi.fn() }));
 vi.mock('../../src/lib/server/config.js', () => ({
@@ -14,6 +15,7 @@ vi.mock('../../src/lib/server/config.js', () => ({
 vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn(), setServerField: vi.fn() }));
 vi.mock('../../src/lib/server/fleet/lifecycle-evidence.js', () => ({ latestGameSubmission: vi.fn() }));
 vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoResolveGamePort: vi.fn() }));
+vi.mock('../../src/lib/server/backends/native.js', () => ({ nativeInstance: vi.fn(), nativeRequest: vi.fn() }));
 
 class FakeSocket extends EventEmitter {
     send = vi.fn();
@@ -47,6 +49,28 @@ afterEach(() => {
 });
 
 describe('RCON status transport', () => {
+    it.each([
+        ['timeout', 'timeout', 'Agent received no reply'],
+        ['unreachable', 'send-error', 'Agent could not reach'],
+        ['unconfigured', 'no-config', 'Agent RCON password'],
+        ['authentication', 'bad-reply', 'wrong RCON password'],
+        ['invalid-reply', 'bad-reply', 'Agent RCON reply is missing']
+    ])('preserves the native agent %s diagnosis', async (agentReason, reason, detail) => {
+        server.native_instance_id = 'instance';
+        vi.mocked(nativeInstance).mockReturnValue({ id: 'instance', generation: 1, host_id: 'host', port: 28960 } as never);
+        vi.mocked(nativeRequest).mockResolvedValue({ ok: false, reason: agentReason, text: '' });
+        expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason, detail: expect.stringContaining(detail) });
+        expect(nativeRequest).toHaveBeenCalledOnce();
+        expect(sockets).toHaveLength(0);
+    });
+
+    it('uses the successful native reply after agent-side recovery', async () => {
+        server.native_instance_id = 'instance';
+        vi.mocked(nativeInstance).mockReturnValue({ id: 'instance', generation: 1, host_id: 'host', port: 28960 } as never);
+        vi.mocked(nativeRequest).mockResolvedValue({ ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' });
+        expect(await rconStatusResult(server.name)).toMatchObject({ ok: true, status: { players: 0, map: 'mp_kino' } });
+    });
+
     it('shares overlapping health and occupancy probes and probes again after completion', async () => {
         const health = rconStatusResult(server.name);
         const occupancy = rconGetPlayerCount(server.name);

tests/unit/server-observation.test.ts

@@ -30,13 +30,22 @@ describe('shared server observation', () => {
         expect(fallback).toMatchObject({ status_source: 'rcon', health: 'online', last_players: 4, last_seen_at: at(40_000) });
     });
 
-    it.each(['starting', 'restarting', 'unhealthy', 'exited', 'unknown'])('telemetry cannot override container state %s', state => {
+    it.each(['starting', 'restarting', 'unhealthy', 'exited'])('telemetry cannot override container state %s', state => {
         const selected = selectServerObservation(server({ health: state === 'starting' || state === 'restarting' ? 'starting' : 'offline', container_state: state }), report(), now);
         expect(selected.health).not.toBe('online');
         expect(selected.last_players).toBeNull();
         expect(selected.roster).toBeNull();
     });
 
+    it('uses fresh RCON with unknown container state, but does not substitute telemetry or extend freshness', () => {
+        const raw = server({ container_state: 'unknown', container_checked_at: at(-60_000) });
+        expect(selectServerObservation(raw, report(), now)).toMatchObject({ health: 'online', last_players: 4, status_source: 'rcon' });
+        expect(selectServerObservation(raw, report(46_000), now + 46_000).health).toBe('stale');
+        expect(selectServerObservation(raw, report(121_000), now + 121_000).health).toBe('offline');
+        expect(selectServerObservation({ ...raw, telemetry_after: at() }, report(), now).health).toBe('offline');
+        expect(selectServerObservation({ ...raw, last_seen_at: null }, report(), now).health).toBe('offline');
+    });
+
     it('requires fresh container evidence and reports after the latest lifecycle submission', () => {
         expect(selectServerObservation(server({ container_checked_at: at(-46_000) }), report(), now).health).not.toBe('online');
         expect(selectServerObservation(server({ health: 'provisioning' }), report(), now).health).toBe('provisioning');

tests/unit/server-status.test.ts

@@ -379,7 +379,7 @@ describe('database fleet and container readiness', () => {
         expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null });
     });
 
-    it.each(['unhealthy', 'exited', undefined])('does not poll RCON with container state %s', async (state) => {
+    it.each(['unhealthy', 'exited', 'stopped', 'missing'])('does not poll RCON with container state %s', async (state) => {
         const { inspect, rcon, fanoutStatus } = await setup();
         inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' });
         await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
@@ -389,6 +389,30 @@ describe('database fleet and container readiness', () => {
         expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'offline', last_players: null });
     });
 
+    it.each(['unknown', undefined, 'throw'])('automatically recovers through RCON when inspection returns %s', async state => {
+        const { inspect, rcon, fanoutStatus, getServer } = await setup();
+        const { GET } = await import('../../src/routes/api/servers/+server.js');
+        const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
+        if (state === 'throw') inspect.mockRejectedValue(new Error('Inspection timed out'));
+        else inspect.mockResolvedValue({ ok: false, state, reason: 'Inspection timed out' });
+        await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
+        rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
+            status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        const events: unknown[] = [];
+        const unsubscribe = subscribe(event => events.push(event));
+        try { await fanoutStatus(); } finally { unsubscribe(); }
+        expect(rcon).toHaveBeenCalledOnce();
+        expect(getServer('managed')?.container_state).toBe('unknown');
+        expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_players: 2, status_source: 'rcon' });
+        expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
+            health: 'online', players: 2, containerState: 'unknown'
+        })] }));
+
+        rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
+        await fanoutStatus();
+        expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' });
+    });
+
     it('waits for an EC restart request to settle before trusting the old healthy container', async () => {
         const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
         db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString());

tests/unit/watchdog.test.ts

@@ -68,6 +68,41 @@ async function sustainedFailure() {
 }
 
 describe('Watchdog detection and ownership', () => {
+    it('scans files every five minutes while retaining minute health checks and immediate manual scans', async () => {
+        const drift = await import('../../src/lib/server/fleet/drift.js');
+        const scan = vi.spyOn(drift, 'prepareDriftCampaign');
+        await watchdog.watchdogTick();
+        expect(scan).toHaveBeenCalledTimes(1);
+        for (let minute = 1; minute < 5; minute++) {
+            advance(watchdog.WATCHDOG_SCAN_MS);
+            await watchdog.watchdogTick();
+        }
+        expect(scan).toHaveBeenCalledTimes(1);
+        expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(5);
+        advance(watchdog.WATCHDOG_SCAN_MS);
+        await watchdog.watchdogTick();
+        expect(scan).toHaveBeenCalledTimes(2);
+        await watchdog.watchdogTick(true);
+        expect(scan).toHaveBeenCalledTimes(3);
+        advance(watchdog.WATCHDOG_SCAN_MS);
+        database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server);
+        await watchdog.watchdogTick();
+        expect(scan).toHaveBeenCalledTimes(4);
+        scan.mockRestore();
+    });
+
+    it('retries failed file scans on the next health scan instead of retaining successful freshness', async () => {
+        const drift = await import('../../src/lib/server/fleet/drift.js');
+        const scan = vi.spyOn(drift, 'prepareDriftCampaign').mockRejectedValueOnce(new Error('Agent unavailable'));
+        await watchdog.watchdogTick();
+        expect(watchdog.watchdogOverview().scanError).toContain('Agent unavailable');
+        advance(watchdog.WATCHDOG_SCAN_MS);
+        await watchdog.watchdogTick();
+        expect(scan).toHaveBeenCalledTimes(2);
+        expect(watchdog.watchdogOverview().scanError).toBeNull();
+        scan.mockRestore();
+    });
+
     it('defaults to automatic recovery and debounces manual scans without accelerating the failure window', async () => {
         expect(watchdog.watchdogOverview().enabled).toBe(true);
         down();