josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Reuse heartbeat container state and batch host RCON status

Browse Source
This commit is contained in:
josh
2026-10-05 15:14:44 -04:00
parent 6586809b35
commit 239eceb779
12 changed files with 517 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/ec_agent.py
+78 -14
View File
@@ -32,7 +32,7 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.4.9"
VERSION = "1.4.10"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000
@@ -519,7 +519,7 @@ class ReadWriteGate:
def read_operation(operation):
if not isinstance(operation, dict):
return False
if operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read"):
if operation.get("kind") in ("host.inspect", "instance.inspect", "rcon.status-batch", "files.read", "files.list", "files.stat", "files.manifest", "logs.read"):
return True
payload = operation.get("payload")
return operation.get("kind") == "rcon.execute" and isinstance(payload, dict) and payload.get("command") in ("status", "getstatus", "version")
@@ -913,6 +913,7 @@ class Agent:
self.read_gate = ReadWriteGate()
self.read_claim_lock = threading.Lock()
self.read_claims = set()
self.health_probe_gate = threading.BoundedSemaphore(2)
self.cancelled_creations = frozenset()
self.observation_keep_days = 0 # Older controllers do not authorize cleanup.
self.library_progress = {}
@@ -1127,18 +1128,19 @@ class Agent:
return None
return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
def instance_resources(self):
def instance_resources(self, observations=None):
"""One bounded Docker stats batch; unavailable samples are never zero load."""
ids = self.docker.call("container", "ls", "--filter", "label=" + LABEL + ".managed=true",
ids = self.docker.call("container", "ls", "--all", "--filter", "label=" + LABEL + ".managed=true",
"--no-trunc", "--format", "{{.ID}}", timeout=10).splitlines()[:1000]
if not ids:
return []
observed_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds")
containers = json.loads(self.docker.call("container", "inspect", *ids, timeout=10))
owned = {}
for info in containers:
try:
labels = info.get("Config", {}).get("Labels", {}) or {}
if labels.get(LABEL + ".managed") != "true" or not info.get("State", {}).get("Running"):
if labels.get(LABEL + ".managed") != "true":
continue
instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
name = self.container_name(instance_id, generation)
@@ -1147,6 +1149,16 @@ class Agent:
expected = str(self.instance_dir(instance_id, generation) / "app")
if not any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])):
continue
state = info.get("State", {})
healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy"
status = state.get("Status", "unknown")
if status == "running" and not healthy:
status = state.get("Health", {}).get("Status", "unknown")
if observations is not None:
observations.append({"instanceId": instance_id, "generation": generation,
"state": status, "healthy": healthy, "observedAt": observed_at})
if not state.get("Running"):
continue
started = datetime.fromisoformat(info["State"]["StartedAt"].replace("Z", "+00:00"))
uptime = max(0, int((datetime.now(timezone.utc) - started).total_seconds()))
owned[name] = {"instanceId": instance_id, "generation": generation, "uptimeSeconds": uptime}
@@ -1280,17 +1292,21 @@ class Agent:
images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
resources = {}
if docker_running:
observations = []
try:
resources["instanceResources"] = self.instance_resources()
resources["instanceResources"] = self.instance_resources(observations)
except (AgentError, ValueError, TypeError, KeyError):
resources["instanceResourcesError"] = "Container resource sampling failed. Check Docker access and stats availability on this host."
# A failed stats call does not discard a successful inspection.
# Missing entries remain unknown; they are never inferred stopped.
resources["instanceStates"] = observations
os_name = platform.system()
with contextlib.suppress(OSError):
for line in Path("/etc/os-release").read_text().splitlines():
if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory()
capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
def safe_to_disrupt(self, instance_id, generation):
@@ -1314,7 +1330,46 @@ class Agent:
if result["players"] != 0:
raise Held("Players are present; disruption held")
def rcon(self, instance_id, generation, command):
def rcon_status_batch(self, operation):
targets = operation.get("payload", {}).get("instances")
if not isinstance(targets, list) or not 1 <= len(targets) <= 1000 or any(not isinstance(target, dict) for target in targets):
raise AgentError("Invalid status batch")
identities = [instance_identity(target.get("instanceId"), target.get("generation")) for target in targets]
if len(set(identities)) != len(identities):
raise AgentError("Duplicate status batch target")
deadline = time.monotonic() + 20
def probe(identity):
instance_id, generation = identity
target = {"instanceId": instance_id, "generation": generation}
reply = {"ok": False, "reason": "timeout", "text": ""}
# A batch does not hold every instance lock while one server waits.
# Bound concurrency across overlapping batches as well as within one.
lock = self.instance_lock(target)
# Use the same lock order as individual health-lane operations.
if lock.acquire(timeout=max(0, deadline - time.monotonic())):
try:
if self.health_probe_gate.acquire(timeout=max(0, deadline - time.monotonic())):
try:
marker = self.instance_dir(instance_id, generation).parent / "current.json"
current = json.loads(marker.read_text()) if marker.exists() else {}
if current.get("generation", generation) != generation or current.get("removed"):
reply = {"ok": False, "reason": "superseded", "text": ""}
elif time.monotonic() < deadline:
reply = self.rcon(instance_id, generation, "status", deadline=deadline)
except (AgentError, OSError, ValueError, KeyError):
reply = {"ok": False, "reason": "unreachable", "text": ""}
finally:
self.health_probe_gate.release()
finally:
lock.release()
return {**target, "reply": reply}
with ThreadPoolExecutor(max_workers=2) as executor:
results = list(executor.map(probe, identities))
return {"ok": all(row["reply"].get("ok") for row in results), "results": results}
def rcon(self, instance_id, generation, command, deadline=None):
if not isinstance(command, str) or not command or len(command) > 2048 or any(ch in command for ch in ("\x00", "\r", "\n")):
raise AgentError("Invalid RCON command")
saved = self.spec(instance_id, generation)
@@ -1327,21 +1382,25 @@ class Agent:
# Retry only the read-only occupancy query. A missing UDP reply must
# never cause commands with effects to be replayed.
for attempt in range(3 if command == "status" else 1):
result = self.rcon_attempt(family, host, int(saved["port"]), password, command)
if deadline is not None and time.monotonic() >= deadline:
return {"ok": False, "reason": "timeout", "players": None, "text": ""}
result = self.rcon_attempt(family, host, int(saved["port"]), password, command, **({"deadline": deadline} if deadline is not None else {}))
if result["ok"] or result.get("reason") not in ("timeout", "unreachable") or attempt == 2 or command != "status":
return result
time.sleep(0.25 * 2 ** attempt)
time.sleep(min(0.25 * 2 ** attempt, max(0, deadline - time.monotonic())) if deadline is not None else 0.25 * 2 ** attempt)
def rcon_attempt(self, family, host, port, password, command):
def rcon_attempt(self, family, host, port, password, command, deadline=None):
chunks = []
with socket.socket(family, socket.SOCK_DGRAM) as sock:
sock.settimeout(3)
sock.settimeout(max(0.001, min(3, deadline - time.monotonic())) if deadline is not None else 3)
try:
sock.connect((host, port))
sock.send(b"\xff\xff\xff\xffrcon " + password.encode() + b" " + command.encode() + b"\n")
while len(chunks) < 64 and sum(map(len, chunks)) < 1024 * 1024:
if deadline is not None and time.monotonic() >= deadline:
break
chunks.append(sock.recv(65535))
sock.settimeout(0.15)
sock.settimeout(max(0.001, min(0.15, deadline - time.monotonic())) if deadline is not None else 0.15)
except socket.timeout:
pass
except OSError:
@@ -2028,6 +2087,8 @@ class Agent:
return {"image": image, "ready": True}
if kind == "host.inspect":
return self.inventory()
if kind == "rcon.status-batch":
return self.rcon_status_batch(operation)
if kind == "library.download":
return self.library_download(payload)
if kind == "library.remove":
@@ -2049,6 +2110,9 @@ class Agent:
if kind == "rcon.execute":
if payload["command"] not in ("status", "getstatus", "version"):
self.checkpoint({"phase": "rcon-command"})
else:
with self.health_probe_gate:
return self.rcon(instance_id, generation, payload["command"])
return self.rcon(instance_id, generation, payload["command"])
raise AgentError("Unsupported operation kind")
@@ -2266,7 +2330,7 @@ class Agent:
@staticmethod
def health_operation(operation):
return operation.get("kind") == "instance.inspect" or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
return operation.get("kind") in ("instance.inspect", "rcon.status-batch") or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
def read_loop(self, lane="read"):
# Each lane owns its SQLite connection and never writes executing_id,