josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add configurable retention for successful routine operations

Browse Source
This commit is contained in:
josh
2026-10-05 13:09:44 -04:00
parent 2d069907f2
commit e6912ceb57
14 changed files with 590 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/agent/ec-agent.py
+69 -9
View File
@@ -32,7 +32,7 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.4.8"
VERSION = "1.4.9"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000
@@ -516,6 +516,19 @@ class ReadWriteGate:
self.condition.notify_all()
def read_operation(operation):
if not isinstance(operation, dict):
return False
if operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read"):
return True
payload = operation.get("payload")
return operation.get("kind") == "rcon.execute" and isinstance(payload, dict) and payload.get("command") in ("status", "getstatus", "version")
def valid_observation_keep_days(value):
return type(value) is int and 0 <= value <= 36500
class Journal:
def __init__(self, path, recover=True):
self.db = sqlite3.connect(path)
@@ -525,8 +538,14 @@ class Journal:
self.db.execute("PRAGMA journal_mode=WAL")
self.db.execute("PRAGMA synchronous=FULL")
self.db.execute("CREATE TABLE IF NOT EXISTS operations (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)")
if "checkpoint" not in {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}:
columns = {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}
if "checkpoint" not in columns:
self.db.execute("ALTER TABLE operations ADD COLUMN checkpoint TEXT")
if "reported_at" not in columns:
self.db.execute("ALTER TABLE operations ADD COLUMN reported_at REAL")
if "retention_eligible" not in columns:
self.db.execute("ALTER TABLE operations ADD COLUMN retention_eligible INTEGER NOT NULL DEFAULT 0")
self.db.execute("CREATE INDEX IF NOT EXISTS operations_observation_retention ON operations(updated,id) WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1")
self.db.execute("UPDATE operations SET state='unknown', error='Agent restarted during execution; inspect before reconciliation', updated=? WHERE state='running'", (time.time(),))
self.db.commit()
@@ -551,10 +570,37 @@ class Journal:
return dict(row)
def update(self, op_id, state, result=None, error=None):
self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=? WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id))
self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=?, reported_at=NULL, retention_eligible=0 WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id))
self.db.commit()
return self.get(op_id)
def confirm_completed(self, row):
"""Only an acknowledged, unchanged completion can become disposable."""
if row["state"] != "completed":
return
eligible = False
try:
result = json.loads(row["result"]) if row.get("result") is not None else None
eligible = read_operation(json.loads(row["request"])) and not row.get("error") and not (
isinstance(result, dict) and (result.get("ok") is False or result.get("healthy") is False or result.get("error") not in (None, ""))
)
except (ValueError, TypeError, KeyError):
pass # Malformed historical records are retained for inspection.
self.db.execute("""UPDATE operations SET reported_at=COALESCE(reported_at,?), retention_eligible=?
WHERE id=? AND state='completed' AND updated=? AND result IS ? AND error IS ?""",
(time.time(), int(bool(eligible)), row["id"], row["updated"], row.get("result"), row.get("error")))
self.db.commit()
def prune_observations(self, keep_days):
if not valid_observation_keep_days(keep_days) or keep_days == 0:
return 0
cutoff = time.time() - keep_days * 86400
removed = self.db.execute("""DELETE FROM operations WHERE id IN (
SELECT id FROM operations WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1
AND updated<? ORDER BY updated,id LIMIT 500)""", (cutoff,)).rowcount
self.db.commit()
return removed
def close(self):
self.db.close()
@@ -868,6 +914,7 @@ class Agent:
self.read_claim_lock = threading.Lock()
self.read_claims = set()
self.cancelled_creations = frozenset()
self.observation_keep_days = 0 # Older controllers do not authorize cleanup.
self.library_progress = {}
self.library_transfers = {}
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
@@ -2118,7 +2165,7 @@ class Agent:
return self.journal.update(operation["id"], "failed", error="Interrupted library download could not be resumed")
if kind == "files.stage":
return self.journal.update(operation["id"], "completed", self.stage(instance_id, generation, payload, operation["id"]))
if kind in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read") or kind == "rcon.execute" and payload.get("command") in ("status", "getstatus", "version"):
if self.read_operation(operation):
return self.journal.update(operation["id"], "completed", self.dispatch(operation))
if kind in ("files.activate", "files.rollback") and checkpoint:
stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json"
@@ -2172,7 +2219,7 @@ class Agent:
pass
return row
def report(self, row):
def report(self, row, journal=None):
payload = {"state": row["state"]}
if row.get("result"):
payload["result"] = json.loads(row["result"])
@@ -2182,13 +2229,19 @@ class Agent:
payload["state"] = "running"
if row.get("error"):
payload["error"] = row["error"]
return self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload)
response = self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload)
confirmed = response.get("operation") if isinstance(response, dict) else None
if row["state"] == "completed" and isinstance(confirmed, dict) and confirmed.get("id") == row["id"] and confirmed.get("state") == "completed":
(journal if journal is not None else self.journal).confirm_completed(row)
return response
def heartbeat(self):
response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation})
# Only the heartbeat thread replaces this immutable snapshot; it never
# mutates the operation journal or races Docker cleanup against create.
self.cancelled_creations = frozenset(value for value in response.get("cancelledCreations", []) if isinstance(value, str))
keep_days = response.get("observationKeepDays")
self.observation_keep_days = keep_days if valid_observation_keep_days(keep_days) else 0
host = response.get("host", {})
if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}
@@ -2205,7 +2258,7 @@ class Agent:
@staticmethod
def read_operation(operation):
return operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read") or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
return read_operation(operation)
def instance_lock(self, operation):
key = operation.get("instanceId")
@@ -2219,12 +2272,19 @@ class Agent:
# Each lane owns its SQLite connection and never writes executing_id,
# checkpoints, or current_operation used by the mutation state machine.
journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
next_prune = time.monotonic()
try:
while not self.stopping.is_set():
try:
if self.update_pending():
self.stopping.wait(3)
continue
if lane == "read" and time.monotonic() >= next_prune:
next_prune = time.monotonic() + 60
with self.activity(read_only=True) as acquired:
if acquired and not self.update_pending():
if journal.prune_observations(self.observation_keep_days) == 500:
next_prune = time.monotonic() + 15
response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20")
handled = False
for operation in response.get("operations", []):
@@ -2242,7 +2302,7 @@ class Agent:
continue
row, _ = journal.accept(operation)
if row["state"] in ("accepted", "unknown", "running"):
self.report(journal.update(operation["id"], "running"))
self.report(journal.update(operation["id"], "running"), journal=journal)
with self.read_gate.read(), self.instance_lock(operation):
try:
result = self.dispatch(operation)
@@ -2251,7 +2311,7 @@ class Agent:
row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
# Receipt delivery can take seconds; it must not
# hold the instance lock needed by health checks.
self.report(row)
self.report(row, journal=journal)
handled = True
finally:
with self.read_claim_lock:
static/agent/manifest.json
+2 -2
View File
@@ -1,8 +1,8 @@
{
"version": "1.4.8",
"version": "1.4.9",
"protocolVersion": 1,
"platform": "debian-amd64",
"sha256": "45eb0ef619fb0ceef2ac843657fd7fdaf8cfa610951f0ddfabdbf5942abc6e8c",
"sha256": "c0a02509a8307b2abb75e5450bf3f39596252f26e041ed938d662bc7127bec62",
"updaterProtocolVersion": 1,
"updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
}