josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve server status when batch probes miss the deadline

Browse Source
This commit is contained in:
josh
2026-10-05 16:17:29 -04:00
parent ad63b3957d
commit ae703617af
7 changed files with 105 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/ec_agent.py
+8 -7
View File
@@ -32,7 +32,7 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.4.12"
VERSION = "1.4.13"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000
@@ -1368,7 +1368,7 @@ class Agent:
def probe(identity):
instance_id, generation = identity
target = {"instanceId": instance_id, "generation": generation}
reply = {"ok": False, "reason": "timeout", "text": ""}
reply = {"ok": False, "reason": "deadline", "text": ""}
# A batch does not hold every instance lock while one server waits.
# Bound concurrency across overlapping batches as well as within one.
lock = self.instance_lock(target)
@@ -1382,7 +1382,7 @@ class Agent:
if current.get("generation", generation) != generation or current.get("removed"):
reply = {"ok": False, "reason": "superseded", "text": ""}
elif time.monotonic() < deadline:
reply = self.rcon(instance_id, generation, "status", deadline=deadline)
reply = self.rcon(instance_id, generation, "status", deadline=deadline, status_attempts=1)
except (AgentError, OSError, ValueError, KeyError):
reply = {"ok": False, "reason": "unreachable", "text": ""}
finally:
@@ -1395,7 +1395,7 @@ class Agent:
results = list(executor.map(probe, identities))
return {"ok": all(row["reply"].get("ok") for row in results), "results": results}
def rcon(self, instance_id, generation, command, deadline=None):
def rcon(self, instance_id, generation, command, deadline=None, status_attempts=3):
if not isinstance(command, str) or not command or len(command) > 2048 or any(ch in command for ch in ("\x00", "\r", "\n")):
raise AgentError("Invalid RCON command")
saved = self.spec(instance_id, generation)
@@ -1407,11 +1407,12 @@ class Agent:
family = socket.AF_INET6 if ":" in host else socket.AF_INET
# Retry only the read-only occupancy query. A missing UDP reply must
# never cause commands with effects to be replayed.
for attempt in range(3 if command == "status" else 1):
attempts = status_attempts if command == "status" else 1
for attempt in range(attempts):
if deadline is not None and time.monotonic() >= deadline:
return {"ok": False, "reason": "timeout", "players": None, "text": ""}
return {"ok": False, "reason": "deadline" if attempt == 0 else "timeout", "players": None, "text": ""}
result = self.rcon_attempt(family, host, int(saved["port"]), password, command, **({"deadline": deadline} if deadline is not None else {}))
if result["ok"] or result.get("reason") not in ("timeout", "unreachable") or attempt == 2 or command != "status":
if result["ok"] or result.get("reason") not in ("timeout", "unreachable") or attempt == attempts - 1:
return result
time.sleep(min(0.25 * 2 ** attempt, max(0, deadline - time.monotonic())) if deadline is not None else 0.25 * 2 ** attempt)
agent/tests/test_agent.py
+40 -2
View File
@@ -613,7 +613,7 @@ class AgentTests(unittest.TestCase):
lock = threading.Lock()
active, peak = 0, 0
def rcon(instance_id, generation, command, deadline):
def rcon(instance_id, generation, command, deadline, status_attempts):
nonlocal active, peak
with lock:
active += 1
@@ -622,6 +622,7 @@ class AgentTests(unittest.TestCase):
ready.set()
try:
self.assertEqual(command, "status")
self.assertEqual(status_attempts, 1)
self.assertTrue(release.wait(2))
if instance_id == "00000000-0000-4000-8000-000000000002":
raise AgentError("local failure")
@@ -647,6 +648,43 @@ class AgentTests(unittest.TestCase):
self.assertEqual(results[0]["results"][2]["reply"]["reason"], "unreachable")
self.assertTrue(results[0]["results"][4]["reply"]["ok"])
def test_batch_uses_one_attempt_but_individual_status_still_retries(self):
saved = {"rconPassword": "secret", "port": 28960}
timeout = {"ok": False, "reason": "timeout", "text": ""}
operation = {"kind": "rcon.status-batch", "payload": {"instances": [{"instanceId": INSTANCE, "generation": 1}]}}
with patch.object(self.agent, "spec", return_value=saved), patch.object(self.agent, "rcon_attempt", return_value=timeout) as attempt, patch.object(module.time, "sleep") as sleep:
self.assertEqual(self.agent.dispatch(operation)["results"][0]["reply"], timeout)
attempt.assert_called_once()
sleep.assert_not_called()
attempt.reset_mock()
self.assertEqual(self.agent.rcon(INSTANCE, 1, "status"), timeout)
self.assertEqual(attempt.call_count, 3)
self.assertEqual(sleep.call_count, 2)
attempt.reset_mock()
self.assertEqual(self.agent.rcon(INSTANCE, 1, "status", deadline=0, status_attempts=1)["reason"], "deadline")
attempt.assert_not_called()
def test_batch_reaches_healthy_servers_after_four_unresponsive_servers(self):
clock = [0]
checked = []
targets = [{"instanceId": f"00000000-0000-4000-8000-{i:012d}", "generation": 1} for i in range(16)]
def attempt(family, host, port, password, command, deadline):
checked.append(port)
if port < 28964:
clock[0] += 3
return {"ok": False, "reason": "timeout", "text": ""}
return {"ok": True, "text": "status"}
# Serial execution is a conservative budget test; concurrency is tested separately.
with patch.object(module, "ThreadPoolExecutor") as pool, patch.object(module.time, "monotonic", side_effect=lambda: clock[0]), patch.object(self.agent, "spec", side_effect=lambda instance, generation: {"port": 28960 + int(instance[-12:]), "rconPassword": "secret"}), patch.object(self.agent, "rcon_attempt", side_effect=attempt), patch.object(module.time, "sleep") as sleep:
pool.return_value.__enter__.return_value.map.side_effect = map
result = self.agent.rcon_status_batch({"payload": {"instances": targets}})
self.assertEqual(len(checked), 16)
self.assertEqual(sum(row["reply"]["ok"] for row in result["results"]), 12)
self.assertEqual(clock[0], 12)
sleep.assert_not_called()
def test_status_batch_rejects_old_generations_and_bounds_expired_work(self):
marker = self.agent.instance_dir(INSTANCE, 1).parent / "current.json"
module.atomic_json(marker, {"generation": 2})
@@ -658,7 +696,7 @@ class AgentTests(unittest.TestCase):
marker.unlink()
with patch.object(module.time, "monotonic", side_effect=[0] + [21] * 20):
result = self.agent.dispatch(operation)
self.assertEqual(result["results"][0]["reply"]["reason"], "timeout")
self.assertEqual(result["results"][0]["reply"]["reason"], "deadline")
probe.assert_not_called()
for targets in ([], [None], [{"instanceId": INSTANCE, "generation": 1}] * 2):
with self.assertRaises(AgentError):