josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add automatic agent updates with verified rollback

Browse Source
This commit is contained in:
josh
2026-10-04 10:40:45 -04:00
parent 020894b816
commit 8d1331c67d
26 changed files with 2568 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/tests/test_installer.py
+169
View File
@@ -1,10 +1,13 @@
"""Exercise installer guards without installing software or touching services."""
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import re
import stat
import subprocess
import sys
import tempfile
import unittest
@@ -15,6 +18,11 @@ SOURCE = (Path(__file__).resolve().parents[1] / "install.sh").read_text()
BLOCKS = re.findall(r"<<'PY'\n(.*?)\nPY", SOURCE, re.DOTALL)
PREFLIGHT = next(block for block in BLOCKS if "# INSTALLATION_PREFLIGHT:" in block)
OWNERSHIP = next(block for block in BLOCKS if "# INSTALLATION_OWNERSHIP:" in block)
RELEASE = next(block for block in BLOCKS if "# INSTALLATION_RELEASE:" in block)
CLEANUP = re.search(r"installation_cleanup\(\) \{.*?\n\}", SOURCE, re.DOTALL).group()
SPEC = importlib.util.spec_from_file_location("package_release", Path(__file__).resolve().parents[1] / "package_release.py")
PACKAGER = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(PACKAGER)
class InstallerTests(unittest.TestCase):
@@ -163,6 +171,47 @@ class InstallerTests(unittest.TestCase):
with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
self.preflight()
def test_pending_update_recovery_is_preserved_and_blocks_manual_replacement(self):
self.legacy_installation()
update = self.config.parent / "update"
update.mkdir()
journal = update / "journal.json"
saved = '{"phase":"swapping","targetVersion":"1.2.0"}'
journal.write_text(saved)
with self.assertRaisesRegex(SystemExit, "automatic update needs recovery"):
self.preflight()
self.assertEqual(journal.read_text(), saved)
journal.unlink()
journal.symlink_to(self.base / "missing")
with self.assertRaisesRegex(SystemExit, "automatic update needs recovery"):
self.preflight()
def test_update_directory_and_lock_cannot_follow_symlinks(self):
self.legacy_installation()
elsewhere = self.base / "unrelated"
elsewhere.mkdir()
update = self.config.parent / "update"
update.symlink_to(elsewhere, target_is_directory=True)
with self.assertRaisesRegex(SystemExit, "not symlinks"):
self.preflight()
update.unlink()
lock = self.config.with_name("update.lock")
lock.symlink_to(elsewhere / "sentinel")
with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
self.preflight()
self.assertFalse((elsewhere / "sentinel").exists())
lock.unlink()
activity = self.root / "agent" / "activity.lock"
activity.symlink_to(elsewhere / "activity-sentinel")
with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
self.preflight()
self.assertFalse((elsewhere / "activity-sentinel").exists())
def test_data_root_cannot_contain_the_updater_or_service_installation(self):
for path in ("/etc/systemd", "/etc/systemd/system", "/usr/local/libexec/electric-cherry-update.py", "/usr/local/libexec/electric-cherry-agent.py.update", "/usr/local/libexec/electric-cherry-uninstall.py.update"):
with self.subTest(path=path), self.assertRaisesRegex(SystemExit, "outside the agent executable and service"):
self.preflight(path)
def test_mounted_parent_is_allowed_but_data_root_and_nested_mounts_are_rejected(self):
mountinfo = Path('/proc/1/mountinfo')
original_exists, original_read = Path.exists, Path.read_text
@@ -181,5 +230,125 @@ class InstallerTests(unittest.TestCase):
self.preflight()
class ReleaseTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.source = Path(self.temporary.name) / "source"
self.target = Path(self.temporary.name) / "release"
self.source.mkdir()
(self.source / "ec_agent.py").write_text('VERSION = "3.12.7"\nraise RuntimeError("must never execute during packaging")\n')
(self.source / "ec_agent_update.py").write_text('raise RuntimeError("must never execute during verification")\n')
(self.source / "install.sh").write_text("#!/bin/bash\n")
PACKAGER.package(self.source, self.target)
def tearDown(self):
self.temporary.cleanup()
def verify(self):
argv = ["-", str(self.target / "manifest.json"), str(self.target / "ec-agent.py"), str(self.target / "ec-agent-update.py")]
with patch.object(sys, "argv", argv):
exec(compile(RELEASE, "installer-release", "exec"), {})
def manifest(self, **changes):
path = self.target / "manifest.json"
value = json.loads(path.read_text())
value.update(changes)
path.write_text(json.dumps(value))
return value
def test_package_derives_version_and_hashes_both_sources_without_executing_them(self):
manifest = self.manifest()
self.assertEqual(manifest["version"], "3.12.7")
self.assertEqual(manifest["updaterProtocolVersion"], 1)
self.assertEqual(manifest["sha256"], hashlib.sha256((self.source / "ec_agent.py").read_bytes()).hexdigest())
self.assertEqual(manifest["updaterSha256"], hashlib.sha256((self.source / "ec_agent_update.py").read_bytes()).hexdigest())
self.assertEqual((self.target / "ec-agent-update.py").read_bytes(), (self.source / "ec_agent_update.py").read_bytes())
self.verify()
def test_changed_agent_and_updater_downloads_are_rejected(self):
for filename, error in (("ec-agent.py", "Agent release checksum"), ("ec-agent-update.py", "Updater release checksum")):
with self.subTest(filename=filename):
path = self.target / filename
original = path.read_bytes()
path.write_bytes(original + b"# changed\n")
with self.assertRaisesRegex(SystemExit, error):
self.verify()
path.write_bytes(original)
def test_installer_rejects_missing_or_unsupported_updater_protocol(self):
for protocol in (None, 0, 2, "1"):
self.manifest(updaterProtocolVersion=protocol)
with self.subTest(protocol=protocol), self.assertRaisesRegex(SystemExit, "Unsupported agent or updater"):
self.verify()
def test_installer_rejects_manifest_version_mismatch_and_invalid_code(self):
self.manifest(version="3.12.8")
with self.assertRaisesRegex(SystemExit, "version does not match"):
self.verify()
malformed = b"def malformed(:\n"
(self.target / "ec-agent-update.py").write_bytes(malformed)
self.manifest(version="3.12.7", updaterSha256=hashlib.sha256(malformed).hexdigest())
with self.assertRaises(SyntaxError):
self.verify()
def test_packaging_requires_one_literal_semantic_agent_version(self):
for source in ('VERSION = "bad"', 'VERSION = "01.2.3"', 'VERSION = str(3)', 'VERSION = "1.2.3"\nVERSION = "1.2.4"'):
(self.source / "ec_agent.py").write_text(source)
with self.subTest(source=source), self.assertRaisesRegex(ValueError, "literal semantic VERSION"):
PACKAGER.package(self.source, self.target)
def test_installer_serializes_changes_and_enables_update_timer_only_after_agent_starts(self):
locked = SOURCE.index("flock --exclusive 9")
self.assertLess(SOURCE.index("systemctl stop electric-cherry-update.timer"), locked)
self.assertLess(locked, SOURCE.index("installation_preflight", locked))
self.assertLess(SOURCE.index("installation_preflight", locked), SOURCE.index('install -m 0755 "$temporary_dir/ec-agent.py"'))
self.assertLess(locked, SOURCE.index("flock --exclusive 8"))
self.assertLess(SOURCE.index("flock --exclusive 8"), SOURCE.index("systemctl stop electric-cherry-agent.service"))
self.assertLess(SOURCE.index("flock --unlock 8", SOURCE.index("systemctl daemon-reload")), SOURCE.index("systemctl restart electric-cherry-agent"))
self.assertLess(SOURCE.index("systemctl is-active --quiet electric-cherry-agent"), SOURCE.index("systemctl enable --now electric-cherry-update.timer"))
self.assertIn("ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py", SOURCE)
self.assertIn("OnUnitInactiveSec=5min", SOURCE)
self.assertIn("RandomizedDelaySec=30s", SOURCE)
self.assertIn("AccuracySec=1s", SOURCE)
self.assertNotIn("Persistent=true", SOURCE)
def test_failed_install_restores_only_previously_active_timer_after_releasing_locks(self):
for previous, finished, code in ((True, False, 37), (False, False, 37), (True, True, 0)):
with self.subTest(previous=previous, finished=finished):
scratch = Path(self.temporary.name) / "scratch"
scratch.mkdir(exist_ok=True)
log = Path(self.temporary.name) / "cleanup.log"
log.unlink(missing_ok=True)
script = CLEANUP + r'''
flock() { printf 'unlock:%s\n' "$2" >> "$EC_TEST_LOG"; }
systemctl() {
if { : >&8; } 2>/dev/null || { : >&9; } 2>/dev/null; then
printf 'lock-still-open\n' >> "$EC_TEST_LOG"
return 1
fi
printf '%s\n' "$*" >> "$EC_TEST_LOG"
}
temporary_dir="$EC_TEST_TEMP"
updater_timer_was_active="$EC_TEST_PREVIOUS"
installer_finished="$EC_TEST_FINISHED"
update_lock_held=true
activity_lock_held=true
exec 8>"$EC_TEST_LOCK"
exec 9>"$EC_TEST_LOCK"
trap installation_cleanup EXIT
exit "$EC_TEST_EXIT"
'''
environment = {**os.environ, "EC_TEST_LOG": str(log), "EC_TEST_TEMP": str(scratch),
"EC_TEST_PREVIOUS": str(previous).lower(), "EC_TEST_FINISHED": str(finished).lower(),
"EC_TEST_LOCK": str(Path(self.temporary.name) / "test.lock"), "EC_TEST_EXIT": str(code)}
result = subprocess.run(["bash", "-c", script], env=environment, capture_output=True)
self.assertEqual(result.returncode, code, result.stderr)
expected = ["unlock:8", "unlock:9"]
if previous and not finished:
expected.append("start electric-cherry-update.timer")
self.assertEqual(log.read_text().splitlines(), expected)
self.assertFalse(scratch.exists())
if __name__ == "__main__":
unittest.main()