josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix native health polling and isolate file operation traffic

Browse Source
This commit is contained in:
josh
2026-10-04 21:50:29 -04:00
parent 8a31de92fa
commit 81f088739c
22 changed files with 446 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/agent/ec-agent.py
+35 -11
View File
@@ -32,7 +32,7 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.4.3"
VERSION = "1.4.5"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
LIBRARY_DOWNLOAD_WORKERS = 4
@@ -1092,7 +1092,7 @@ class Agent:
if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory()
capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
capabilities = ["instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
def safe_to_disrupt(self, instance_id, generation):
@@ -1126,11 +1126,20 @@ class Agent:
bind = saved.get("bindAddress", "0.0.0.0")
host = "127.0.0.1" if bind in ("0.0.0.0", "") else "::1" if bind == "::" else bind
family = socket.AF_INET6 if ":" in host else socket.AF_INET
# Retry only the read-only occupancy query. A missing UDP reply must
# never cause commands with effects to be replayed.
for attempt in range(3 if command == "status" else 1):
result = self.rcon_attempt(family, host, int(saved["port"]), password, command)
if result["ok"] or result.get("reason") not in ("timeout", "unreachable") or attempt == 2 or command != "status":
return result
time.sleep(0.25 * 2 ** attempt)
def rcon_attempt(self, family, host, port, password, command):
chunks = []
with socket.socket(family, socket.SOCK_DGRAM) as sock:
sock.settimeout(3)
try:
sock.connect((host, int(saved["port"])))
sock.connect((host, port))
sock.send(b"\xff\xff\xff\xffrcon " + password.encode() + b" " + command.encode() + b"\n")
while len(chunks) < 64 and sum(map(len, chunks)) < 1024 * 1024:
chunks.append(sock.recv(65535))
@@ -2016,7 +2025,11 @@ class Agent:
key = operation.get("instanceId")
return self.instance_locks.setdefault(key, threading.RLock()) if key else contextlib.nullcontext()
def read_loop(self):
@staticmethod
def health_operation(operation):
return operation.get("kind") == "instance.inspect" or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
def read_loop(self, lane="read"):
# Each lane owns its SQLite connection and never writes executing_id,
# checkpoints, or current_operation used by the mutation state machine.
journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
@@ -2026,10 +2039,13 @@ class Agent:
if self.update_pending():
self.stopping.wait(3)
continue
response = self.request("GET", "/api/agent/operations?lane=read&wait=20")
response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20")
handled = False
for operation in response.get("operations", []):
if self.stopping.is_set() or not self.read_operation(operation):
continue
if lane == "health" and not self.health_operation(operation):
continue
with self.read_claim_lock:
if operation["id"] in self.read_claims:
continue
@@ -2038,19 +2054,26 @@ class Agent:
with self.activity(read_only=True) as acquired:
if not acquired or self.update_pending():
continue
with self.read_gate.read(), self.instance_lock(operation):
row, _ = journal.accept(operation)
if row["state"] in ("accepted", "unknown", "running"):
self.report(journal.update(operation["id"], "running"))
row, _ = journal.accept(operation)
if row["state"] in ("accepted", "unknown", "running"):
self.report(journal.update(operation["id"], "running"))
with self.read_gate.read(), self.instance_lock(operation):
try:
result = self.dispatch(operation)
row = journal.update(operation["id"], "completed", result)
except Exception as err:
row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
self.report(row)
# Receipt delivery can take seconds; it must not
# hold the instance lock needed by health checks.
self.report(row)
handled = True
finally:
with self.read_claim_lock:
self.read_claims.discard(operation["id"])
if response.get("operations") and not handled:
# A legacy controller may return another lane's work or
# a read already claimed locally. Avoid a busy poll loop.
self.stopping.wait(0.25)
except (AgentError, OSError, ValueError, KeyError, sqlite3.Error):
self.stopping.wait(3)
finally:
@@ -2109,7 +2132,8 @@ class Agent:
self.stopping.wait(3)
heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
heartbeats.start()
readers = [threading.Thread(target=self.read_loop, daemon=True) for _ in range(2)]
# Preserve two concurrent health checks even while files are being read.
readers = [threading.Thread(target=self.read_loop, args=(lane,), daemon=True) for lane in ("health", "health", "read")]
for reader in readers:
reader.start()
while not self.stopping.is_set():
static/agent/manifest.json
+2 -2
View File
@@ -1,8 +1,8 @@
{
"version": "1.4.3",
"version": "1.4.5",
"protocolVersion": 1,
"platform": "debian-amd64",
"sha256": "9ca9bff1a42ab838b175329c6a8458e18053429efe2f050f477a9ad9c598410f",
"sha256": "bd4afa2ec3ab6a3022dba99d0580b4b6b1ba5622ae4138035e3e3862885a97bd",
"updaterProtocolVersion": 1,
"updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
}