josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Automatically place servers using learned resource capacity

Browse Source
This commit is contained in:
josh
2026-10-04 21:04:09 -04:00
parent a1b957392f
commit e6cdf9eb7f
25 changed files with 927 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/ec_agent.py
+60 -2
View File
@@ -9,6 +9,8 @@ import argparse
import base64
from concurrent.futures import ThreadPoolExecutor, as_completed
import contextlib
from datetime import datetime, timezone
import math
import hashlib
import fcntl
import json
@@ -30,7 +32,7 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.4.2"
VERSION = "1.4.3"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
LIBRARY_DOWNLOAD_WORKERS = 4
@@ -929,6 +931,58 @@ class Agent:
return None
return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
def instance_resources(self):
"""One bounded Docker stats batch; unavailable samples are never zero load."""
ids = self.docker.call("container", "ls", "--filter", LABEL + ".managed=true",
"--no-trunc", "--format", "{{.ID}}", timeout=10).splitlines()[:1000]
if not ids:
return []
containers = json.loads(self.docker.call("container", "inspect", *ids, timeout=10))
owned = {}
for info in containers:
try:
labels = info.get("Config", {}).get("Labels", {}) or {}
if labels.get(LABEL + ".managed") != "true" or not info.get("State", {}).get("Running"):
continue
instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
name = self.container_name(instance_id, generation)
if info.get("Name", "").lstrip("/") != name:
continue
expected = str(self.instance_dir(instance_id, generation) / "app")
if not any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])):
continue
started = datetime.fromisoformat(info["State"]["StartedAt"].replace("Z", "+00:00"))
uptime = max(0, int((datetime.now(timezone.utc) - started).total_seconds()))
owned[name] = {"instanceId": instance_id, "generation": generation, "uptimeSeconds": uptime}
except (AgentError, ValueError, KeyError, TypeError):
continue
if not owned:
return []
rows = self.docker.call("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", *owned, timeout=15)
samples = []
units = {"B": 1, "KiB": 1024, "MiB": 1024 ** 2, "GiB": 1024 ** 3,
"TiB": 1024 ** 4, "kB": 1000, "MB": 1000 ** 2, "GB": 1000 ** 3}
for line in rows.splitlines():
try:
row = json.loads(line)
identity = owned.get(row.get("Name"))
if not identity or int(row["PIDs"]) <= 0:
continue
# Docker Linux CPU percent is core-based: 150% means 1.5 cores.
percent = row["CPUPerc"]
if not percent.endswith("%"):
continue
cores = float(percent[:-1]) / 100
used = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)", row["MemUsage"].split("/", 1)[0].strip())
if not used or used[2] not in units or not math.isfinite(cores) or cores < 0:
continue
memory = math.ceil(float(used[1]) * units[used[2]])
if memory > 0:
samples.append({**identity, "cpuCores": cores, "memoryBytes": memory})
except (ValueError, KeyError, TypeError, AttributeError, OverflowError):
continue
return samples
def inventory(self):
cpu_usage = self.cpu_usage_percent()
usage = shutil.disk_usage(self.root)
@@ -1028,6 +1082,10 @@ class Agent:
if aliases.exists():
with contextlib.suppress(OSError, ValueError):
images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
resources = {}
if docker_running:
with contextlib.suppress(AgentError, ValueError, TypeError, KeyError):
resources["instanceResources"] = self.instance_resources()
os_name = platform.system()
with contextlib.suppress(OSError):
for line in Path("/etc/os-release").read_text().splitlines():
@@ -1035,7 +1093,7 @@ class Agent:
os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory()
capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
def safe_to_disrupt(self, instance_id, generation):
health = self.inspect(instance_id, generation)
agent/tests/test_agent.py
+44
View File
@@ -107,6 +107,50 @@ class AgentTests(unittest.TestCase):
else:
self.assertEqual(inventory["cpuUsagePercent"], usage)
def resource_container(self):
return {"Name": self.agent.container_name(INSTANCE, 1),
"Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}},
"State": {"Running": True, "StartedAt": "2000-01-01T00:00:00.000000000Z"},
"Mounts": [{"Type": "bind", "Source": str(self.agent.instance_dir(INSTANCE, 1) / "app"), "Destination": "/home/plutainer/app"}]}
def test_instance_resources_batches_owned_containers_and_normalizes_core_cpu_and_memory(self):
name = self.agent.container_name(INSTANCE, 1)
rows = json.dumps({"Name": name, "CPUPerc": "150.00%", "MemUsage": "384.5MiB / 8GiB", "PIDs": "12"})
with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), rows]) as call:
samples = self.agent.instance_resources()
self.assertEqual(len(samples), 1)
self.assertEqual(samples[0]["instanceId"], INSTANCE)
self.assertEqual(samples[0]["generation"], 1)
self.assertEqual(samples[0]["cpuCores"], 1.5)
self.assertEqual(samples[0]["memoryBytes"], int(384.5 * 1024 ** 2))
self.assertGreater(samples[0]["uptimeSeconds"], 300)
self.assertEqual(call.call_args_list[-1].args, ("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", name))
def test_instance_resources_ignores_stopped_foreign_and_mismatched_containers(self):
for change in ("stopped", "foreign", "generation", "name"):
info = self.resource_container()
if change == "stopped":
info["State"]["Running"] = False
elif change == "foreign":
info["Mounts"][0]["Source"] = "/another-installation/app"
elif change == "generation":
info["Config"]["Labels"][module.LABEL + ".generation"] = "2"
else:
info["Name"] = "unrelated"
with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([info])]) as call:
self.assertEqual(self.agent.instance_resources(), [])
self.assertEqual(call.call_count, 2)
def test_instance_resources_skips_unavailable_stats_instead_of_reporting_zero_usage(self):
good = {"Name": self.agent.container_name(INSTANCE, 1), "CPUPerc": "0.00%", "MemUsage": "200MiB / 8GiB", "PIDs": "10"}
for override in ({"CPUPerc": "NaN%"}, {"CPUPerc": "-1%"}, {"CPUPerc": "--"}, {"MemUsage": "bad"}, {"MemUsage": "0B / 0B"}, {"PIDs": "0"}, {"Name": "foreign"}):
with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), json.dumps({**good, **override})]):
self.assertEqual(self.agent.instance_resources(), [])
with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), json.dumps(good)]):
self.assertEqual(self.agent.instance_resources()[0]["cpuCores"], 0)
with patch.object(self.agent, "instance_resources", side_effect=AgentError("timed out")):
self.assertNotIn("instanceResources", self.agent.inventory())
def operation(self, kind, payload=None, op_id="operation-1", generation=1):
return {"id": op_id, "kind": kind, "instanceId": INSTANCE, "generation": generation, "payload": payload or {}}