Fix native health polling and isolate file operation traffic
This commit is contained in:
+35
-11
@@ -32,7 +32,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.4.3"
|
||||
VERSION = "1.4.5"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
LIBRARY_DOWNLOAD_WORKERS = 4
|
||||
@@ -1092,7 +1092,7 @@ class Agent:
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
update = self.update_inventory()
|
||||
capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
capabilities = ["instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation):
|
||||
@@ -1126,11 +1126,20 @@ class Agent:
|
||||
bind = saved.get("bindAddress", "0.0.0.0")
|
||||
host = "127.0.0.1" if bind in ("0.0.0.0", "") else "::1" if bind == "::" else bind
|
||||
family = socket.AF_INET6 if ":" in host else socket.AF_INET
|
||||
# Retry only the read-only occupancy query. A missing UDP reply must
|
||||
# never cause commands with effects to be replayed.
|
||||
for attempt in range(3 if command == "status" else 1):
|
||||
result = self.rcon_attempt(family, host, int(saved["port"]), password, command)
|
||||
if result["ok"] or result.get("reason") not in ("timeout", "unreachable") or attempt == 2 or command != "status":
|
||||
return result
|
||||
time.sleep(0.25 * 2 ** attempt)
|
||||
|
||||
def rcon_attempt(self, family, host, port, password, command):
|
||||
chunks = []
|
||||
with socket.socket(family, socket.SOCK_DGRAM) as sock:
|
||||
sock.settimeout(3)
|
||||
try:
|
||||
sock.connect((host, int(saved["port"])))
|
||||
sock.connect((host, port))
|
||||
sock.send(b"\xff\xff\xff\xffrcon " + password.encode() + b" " + command.encode() + b"\n")
|
||||
while len(chunks) < 64 and sum(map(len, chunks)) < 1024 * 1024:
|
||||
chunks.append(sock.recv(65535))
|
||||
@@ -2016,7 +2025,11 @@ class Agent:
|
||||
key = operation.get("instanceId")
|
||||
return self.instance_locks.setdefault(key, threading.RLock()) if key else contextlib.nullcontext()
|
||||
|
||||
def read_loop(self):
|
||||
@staticmethod
|
||||
def health_operation(operation):
|
||||
return operation.get("kind") == "instance.inspect" or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
|
||||
|
||||
def read_loop(self, lane="read"):
|
||||
# Each lane owns its SQLite connection and never writes executing_id,
|
||||
# checkpoints, or current_operation used by the mutation state machine.
|
||||
journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
|
||||
@@ -2026,10 +2039,13 @@ class Agent:
|
||||
if self.update_pending():
|
||||
self.stopping.wait(3)
|
||||
continue
|
||||
response = self.request("GET", "/api/agent/operations?lane=read&wait=20")
|
||||
response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20")
|
||||
handled = False
|
||||
for operation in response.get("operations", []):
|
||||
if self.stopping.is_set() or not self.read_operation(operation):
|
||||
continue
|
||||
if lane == "health" and not self.health_operation(operation):
|
||||
continue
|
||||
with self.read_claim_lock:
|
||||
if operation["id"] in self.read_claims:
|
||||
continue
|
||||
@@ -2038,19 +2054,26 @@ class Agent:
|
||||
with self.activity(read_only=True) as acquired:
|
||||
if not acquired or self.update_pending():
|
||||
continue
|
||||
with self.read_gate.read(), self.instance_lock(operation):
|
||||
row, _ = journal.accept(operation)
|
||||
if row["state"] in ("accepted", "unknown", "running"):
|
||||
self.report(journal.update(operation["id"], "running"))
|
||||
row, _ = journal.accept(operation)
|
||||
if row["state"] in ("accepted", "unknown", "running"):
|
||||
self.report(journal.update(operation["id"], "running"))
|
||||
with self.read_gate.read(), self.instance_lock(operation):
|
||||
try:
|
||||
result = self.dispatch(operation)
|
||||
row = journal.update(operation["id"], "completed", result)
|
||||
except Exception as err:
|
||||
row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
|
||||
self.report(row)
|
||||
# Receipt delivery can take seconds; it must not
|
||||
# hold the instance lock needed by health checks.
|
||||
self.report(row)
|
||||
handled = True
|
||||
finally:
|
||||
with self.read_claim_lock:
|
||||
self.read_claims.discard(operation["id"])
|
||||
if response.get("operations") and not handled:
|
||||
# A legacy controller may return another lane's work or
|
||||
# a read already claimed locally. Avoid a busy poll loop.
|
||||
self.stopping.wait(0.25)
|
||||
except (AgentError, OSError, ValueError, KeyError, sqlite3.Error):
|
||||
self.stopping.wait(3)
|
||||
finally:
|
||||
@@ -2109,7 +2132,8 @@ class Agent:
|
||||
self.stopping.wait(3)
|
||||
heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
|
||||
heartbeats.start()
|
||||
readers = [threading.Thread(target=self.read_loop, daemon=True) for _ in range(2)]
|
||||
# Preserve two concurrent health checks even while files are being read.
|
||||
readers = [threading.Thread(target=self.read_loop, args=(lane,), daemon=True) for lane in ("health", "health", "read")]
|
||||
for reader in readers:
|
||||
reader.start()
|
||||
while not self.stopping.is_set():
|
||||
|
||||
@@ -6,7 +6,7 @@ from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import mock_open, patch
|
||||
from unittest.mock import MagicMock, call, mock_open, patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("ec_agent", Path(__file__).resolve().parents[1] / "ec_agent.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
@@ -995,6 +995,64 @@ class AgentTests(unittest.TestCase):
|
||||
self.assertEqual(result["players"], 1)
|
||||
self.assertTrue(result["ok"])
|
||||
|
||||
@staticmethod
|
||||
def rcon_socket(*replies):
|
||||
sock = MagicMock()
|
||||
sock.__enter__.return_value = sock
|
||||
sock.recv.side_effect = replies
|
||||
return sock
|
||||
|
||||
def test_udp_status_recovers_after_transient_failures(self):
|
||||
self.create()
|
||||
for error in (module.socket.timeout(), OSError("unreachable")):
|
||||
with self.subTest(error=type(error).__name__):
|
||||
failed = self.rcon_socket(error)
|
||||
# Keep all response packets, including occupied player rows.
|
||||
recovered = self.rcon_socket(
|
||||
b"\xff\xff\xff\xffprint\nmap: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n",
|
||||
b"\xff\xff\xff\xffprint\n0 42 CNCT abc player\n", module.socket.timeout())
|
||||
with patch.object(module.socket, "socket", side_effect=[failed, recovered]) as sockets, patch.object(module.time, "sleep") as sleep:
|
||||
result = self.agent.rcon(INSTANCE, 1, "status")
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertEqual(result["players"], 1)
|
||||
self.assertEqual(sockets.call_count, 2)
|
||||
sleep.assert_called_once_with(0.25)
|
||||
failed.__exit__.assert_called_once()
|
||||
recovered.__exit__.assert_called_once()
|
||||
|
||||
def test_udp_status_retries_are_bounded(self):
|
||||
self.create()
|
||||
attempts = [self.rcon_socket(module.socket.timeout()) for _ in range(3)]
|
||||
with patch.object(module.socket, "socket", side_effect=attempts) as sockets, patch.object(module.time, "sleep") as sleep:
|
||||
result = self.agent.rcon(INSTANCE, 1, "status")
|
||||
self.assertFalse(result["ok"])
|
||||
self.assertEqual(result["reason"], "timeout")
|
||||
self.assertEqual(sockets.call_count, 3)
|
||||
self.assertEqual(sleep.call_args_list, [call(0.25), call(0.5)])
|
||||
|
||||
def test_udp_status_does_not_retry_bad_replies_or_partial_tables(self):
|
||||
self.create()
|
||||
for packet in (b"wrong listener", b"\xff\xff\xff\xffprint\nBad rcon_password",
|
||||
b"\xff\xff\xff\xffprint\nmap: mp_kino\nnum score ping guid name\n"):
|
||||
with self.subTest(packet=packet):
|
||||
sock = self.rcon_socket(packet, module.socket.timeout())
|
||||
with patch.object(module.socket, "socket", return_value=sock) as sockets, patch.object(module.time, "sleep") as sleep:
|
||||
result = self.agent.rcon(INSTANCE, 1, "status")
|
||||
self.assertIsNone(result["players"])
|
||||
sockets.assert_called_once()
|
||||
sleep.assert_not_called()
|
||||
|
||||
def test_udp_commands_with_effects_are_never_replayed(self):
|
||||
self.create()
|
||||
for command in ("map_restart", "kick 0", "version"):
|
||||
with self.subTest(command=command):
|
||||
sock = self.rcon_socket(module.socket.timeout())
|
||||
with patch.object(module.socket, "socket", return_value=sock) as sockets, patch.object(module.time, "sleep") as sleep:
|
||||
result = self.agent.rcon(INSTANCE, 1, command)
|
||||
self.assertEqual(result["reason"], "timeout")
|
||||
sockets.assert_called_once()
|
||||
sleep.assert_not_called()
|
||||
|
||||
def test_t6_bot_column_allows_empty_restart_but_holds_players_and_partial_replies(self):
|
||||
self.create()
|
||||
header = "map: zm_nuked\nnum score bot ping guid name lastmsg address qport rate\n"
|
||||
|
||||
@@ -118,6 +118,44 @@ class UpdateIntegrationTests(unittest.TestCase):
|
||||
self.agent.read_loop()
|
||||
dispatch.assert_not_called()
|
||||
|
||||
def test_health_lane_runs_while_file_receipt_for_same_instance_is_slow(self):
|
||||
receipt_waiting, release_receipt, health_done = threading.Event(), threading.Event(), threading.Event()
|
||||
file_op = {"id": "file-one", "kind": "files.read", "instanceId": "same", "payload": {}}
|
||||
health_op = {"id": "health-one", "kind": "instance.inspect", "instanceId": "same", "payload": {}}
|
||||
errors = []
|
||||
def request(method, path, body=None):
|
||||
if method == "GET":
|
||||
return {"operations": [health_op if "lane=health" in path else file_op]}
|
||||
if path.endswith("file-one") and body["state"] == "completed":
|
||||
receipt_waiting.set()
|
||||
release_receipt.wait(3)
|
||||
return {}
|
||||
def dispatch(operation):
|
||||
if operation["id"] == "health-one":
|
||||
health_done.set()
|
||||
self.agent.stopping.set()
|
||||
return {"ok": True}
|
||||
def run(lane):
|
||||
try: self.agent.read_loop(lane)
|
||||
except Exception as error: errors.append(error)
|
||||
self.agent.request, self.agent.dispatch = request, dispatch
|
||||
files = threading.Thread(target=run, args=("read",))
|
||||
health = threading.Thread(target=run, args=("health",))
|
||||
files.start()
|
||||
try:
|
||||
self.assertTrue(receipt_waiting.wait(2))
|
||||
health.start()
|
||||
self.assertTrue(health_done.wait(2), "Health waited for a file receipt")
|
||||
finally:
|
||||
self.agent.stopping.set()
|
||||
release_receipt.set()
|
||||
files.join(3)
|
||||
if health.ident is not None: health.join(3)
|
||||
self.assertFalse(files.is_alive())
|
||||
self.assertFalse(health.is_alive())
|
||||
self.assertEqual(errors, [])
|
||||
self.assertEqual(self.agent.journal.get("health-one")["state"], "completed")
|
||||
|
||||
def test_instance_mutations_and_global_mutations_exclude_observations(self):
|
||||
operation = {"instanceId": "one"}
|
||||
entered = threading.Event()
|
||||
|
||||
Reference in New Issue
Block a user