Add configurable retention for successful routine operations
This commit is contained in:
+69
-9
@@ -32,7 +32,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.4.8"
|
||||
VERSION = "1.4.9"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
MANIFEST_MAX_ENTRIES = 10000
|
||||
@@ -516,6 +516,19 @@ class ReadWriteGate:
|
||||
self.condition.notify_all()
|
||||
|
||||
|
||||
def read_operation(operation):
|
||||
if not isinstance(operation, dict):
|
||||
return False
|
||||
if operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read"):
|
||||
return True
|
||||
payload = operation.get("payload")
|
||||
return operation.get("kind") == "rcon.execute" and isinstance(payload, dict) and payload.get("command") in ("status", "getstatus", "version")
|
||||
|
||||
|
||||
def valid_observation_keep_days(value):
|
||||
return type(value) is int and 0 <= value <= 36500
|
||||
|
||||
|
||||
class Journal:
|
||||
def __init__(self, path, recover=True):
|
||||
self.db = sqlite3.connect(path)
|
||||
@@ -525,8 +538,14 @@ class Journal:
|
||||
self.db.execute("PRAGMA journal_mode=WAL")
|
||||
self.db.execute("PRAGMA synchronous=FULL")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS operations (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)")
|
||||
if "checkpoint" not in {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}:
|
||||
columns = {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}
|
||||
if "checkpoint" not in columns:
|
||||
self.db.execute("ALTER TABLE operations ADD COLUMN checkpoint TEXT")
|
||||
if "reported_at" not in columns:
|
||||
self.db.execute("ALTER TABLE operations ADD COLUMN reported_at REAL")
|
||||
if "retention_eligible" not in columns:
|
||||
self.db.execute("ALTER TABLE operations ADD COLUMN retention_eligible INTEGER NOT NULL DEFAULT 0")
|
||||
self.db.execute("CREATE INDEX IF NOT EXISTS operations_observation_retention ON operations(updated,id) WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1")
|
||||
self.db.execute("UPDATE operations SET state='unknown', error='Agent restarted during execution; inspect before reconciliation', updated=? WHERE state='running'", (time.time(),))
|
||||
self.db.commit()
|
||||
|
||||
@@ -551,10 +570,37 @@ class Journal:
|
||||
return dict(row)
|
||||
|
||||
def update(self, op_id, state, result=None, error=None):
|
||||
self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=? WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id))
|
||||
self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=?, reported_at=NULL, retention_eligible=0 WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id))
|
||||
self.db.commit()
|
||||
return self.get(op_id)
|
||||
|
||||
def confirm_completed(self, row):
|
||||
"""Only an acknowledged, unchanged completion can become disposable."""
|
||||
if row["state"] != "completed":
|
||||
return
|
||||
eligible = False
|
||||
try:
|
||||
result = json.loads(row["result"]) if row.get("result") is not None else None
|
||||
eligible = read_operation(json.loads(row["request"])) and not row.get("error") and not (
|
||||
isinstance(result, dict) and (result.get("ok") is False or result.get("healthy") is False or result.get("error") not in (None, ""))
|
||||
)
|
||||
except (ValueError, TypeError, KeyError):
|
||||
pass # Malformed historical records are retained for inspection.
|
||||
self.db.execute("""UPDATE operations SET reported_at=COALESCE(reported_at,?), retention_eligible=?
|
||||
WHERE id=? AND state='completed' AND updated=? AND result IS ? AND error IS ?""",
|
||||
(time.time(), int(bool(eligible)), row["id"], row["updated"], row.get("result"), row.get("error")))
|
||||
self.db.commit()
|
||||
|
||||
def prune_observations(self, keep_days):
|
||||
if not valid_observation_keep_days(keep_days) or keep_days == 0:
|
||||
return 0
|
||||
cutoff = time.time() - keep_days * 86400
|
||||
removed = self.db.execute("""DELETE FROM operations WHERE id IN (
|
||||
SELECT id FROM operations WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1
|
||||
AND updated<? ORDER BY updated,id LIMIT 500)""", (cutoff,)).rowcount
|
||||
self.db.commit()
|
||||
return removed
|
||||
|
||||
def close(self):
|
||||
self.db.close()
|
||||
|
||||
@@ -868,6 +914,7 @@ class Agent:
|
||||
self.read_claim_lock = threading.Lock()
|
||||
self.read_claims = set()
|
||||
self.cancelled_creations = frozenset()
|
||||
self.observation_keep_days = 0 # Older controllers do not authorize cleanup.
|
||||
self.library_progress = {}
|
||||
self.library_transfers = {}
|
||||
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
|
||||
@@ -2118,7 +2165,7 @@ class Agent:
|
||||
return self.journal.update(operation["id"], "failed", error="Interrupted library download could not be resumed")
|
||||
if kind == "files.stage":
|
||||
return self.journal.update(operation["id"], "completed", self.stage(instance_id, generation, payload, operation["id"]))
|
||||
if kind in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read") or kind == "rcon.execute" and payload.get("command") in ("status", "getstatus", "version"):
|
||||
if self.read_operation(operation):
|
||||
return self.journal.update(operation["id"], "completed", self.dispatch(operation))
|
||||
if kind in ("files.activate", "files.rollback") and checkpoint:
|
||||
stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json"
|
||||
@@ -2172,7 +2219,7 @@ class Agent:
|
||||
pass
|
||||
return row
|
||||
|
||||
def report(self, row):
|
||||
def report(self, row, journal=None):
|
||||
payload = {"state": row["state"]}
|
||||
if row.get("result"):
|
||||
payload["result"] = json.loads(row["result"])
|
||||
@@ -2182,13 +2229,19 @@ class Agent:
|
||||
payload["state"] = "running"
|
||||
if row.get("error"):
|
||||
payload["error"] = row["error"]
|
||||
return self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload)
|
||||
response = self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload)
|
||||
confirmed = response.get("operation") if isinstance(response, dict) else None
|
||||
if row["state"] == "completed" and isinstance(confirmed, dict) and confirmed.get("id") == row["id"] and confirmed.get("state") == "completed":
|
||||
(journal if journal is not None else self.journal).confirm_completed(row)
|
||||
return response
|
||||
|
||||
def heartbeat(self):
|
||||
response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation})
|
||||
# Only the heartbeat thread replaces this immutable snapshot; it never
|
||||
# mutates the operation journal or races Docker cleanup against create.
|
||||
self.cancelled_creations = frozenset(value for value in response.get("cancelledCreations", []) if isinstance(value, str))
|
||||
keep_days = response.get("observationKeepDays")
|
||||
self.observation_keep_days = keep_days if valid_observation_keep_days(keep_days) else 0
|
||||
host = response.get("host", {})
|
||||
if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
|
||||
self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}
|
||||
@@ -2205,7 +2258,7 @@ class Agent:
|
||||
|
||||
@staticmethod
|
||||
def read_operation(operation):
|
||||
return operation.get("kind") in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read") or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version"))
|
||||
return read_operation(operation)
|
||||
|
||||
def instance_lock(self, operation):
|
||||
key = operation.get("instanceId")
|
||||
@@ -2219,12 +2272,19 @@ class Agent:
|
||||
# Each lane owns its SQLite connection and never writes executing_id,
|
||||
# checkpoints, or current_operation used by the mutation state machine.
|
||||
journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False)
|
||||
next_prune = time.monotonic()
|
||||
try:
|
||||
while not self.stopping.is_set():
|
||||
try:
|
||||
if self.update_pending():
|
||||
self.stopping.wait(3)
|
||||
continue
|
||||
if lane == "read" and time.monotonic() >= next_prune:
|
||||
next_prune = time.monotonic() + 60
|
||||
with self.activity(read_only=True) as acquired:
|
||||
if acquired and not self.update_pending():
|
||||
if journal.prune_observations(self.observation_keep_days) == 500:
|
||||
next_prune = time.monotonic() + 15
|
||||
response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20")
|
||||
handled = False
|
||||
for operation in response.get("operations", []):
|
||||
@@ -2242,7 +2302,7 @@ class Agent:
|
||||
continue
|
||||
row, _ = journal.accept(operation)
|
||||
if row["state"] in ("accepted", "unknown", "running"):
|
||||
self.report(journal.update(operation["id"], "running"))
|
||||
self.report(journal.update(operation["id"], "running"), journal=journal)
|
||||
with self.read_gate.read(), self.instance_lock(operation):
|
||||
try:
|
||||
result = self.dispatch(operation)
|
||||
@@ -2251,7 +2311,7 @@ class Agent:
|
||||
row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable")
|
||||
# Receipt delivery can take seconds; it must not
|
||||
# hold the instance lock needed by health checks.
|
||||
self.report(row)
|
||||
self.report(row, journal=journal)
|
||||
handled = True
|
||||
finally:
|
||||
with self.read_claim_lock:
|
||||
|
||||
Reference in New Issue
Block a user