josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add automatic agent updates with verified rollback

Browse Source
This commit is contained in:
josh
2026-10-04 10:40:45 -04:00
parent 020894b816
commit 8d1331c67d
26 changed files with 2568 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/agent/install.sh
+146 -22
View File
@@ -8,6 +8,8 @@ controller=''
enrollment_token=''
data_dir='/var/lib/electric-cherry'
reenroll=''
existing_enrollment=false
existing_data_root=false
while [ "$#" -gt 0 ]; do
case "$1" in
--controller) controller="${2:?Missing controller URL}"; shift 2 ;;
@@ -31,7 +33,16 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y --no-install-recommends ca-certificates curl python3
apt-get install -y --no-install-recommends ca-certificates curl python3 util-linux
installation_preflight() {
if [ "$existing_enrollment" = true ] && [ ! -f /etc/electric-cherry/agent.json ]; then
echo 'The existing enrollment disappeared during installation. Stop and inspect host deletion before trying again.' >&2
return 1
fi
if [ "$existing_data_root" = true ] && [ ! -d "$data_dir" ]; then
echo 'The existing data directory disappeared during installation. Stop and inspect host deletion before trying again.' >&2
return 1
fi
python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
# INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
import json, pathlib, re, stat, sys
@@ -51,9 +62,14 @@ if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
installed_paths = [config,
pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
pathlib.Path('/usr/local/libexec/electric-cherry-agent.py.update'),
pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py.update'),
pathlib.Path('/usr/local/libexec/electric-cherry-update.py'),
pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service'),
pathlib.Path('/etc/systemd/system/electric-cherry-update.service'),
pathlib.Path('/etc/systemd/system/electric-cherry-update.timer')]
if any(root == path or root in path.parents for path in installed_paths):
raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
@@ -66,10 +82,13 @@ def trusted_directories(path):
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
def trusted_json(path):
def trusted_file(path):
info = path.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
def trusted_json(path):
trusted_file(path)
value = json.loads(path.read_text())
if not isinstance(value, dict):
raise SystemExit('Existing installation metadata is invalid.')
@@ -89,6 +108,13 @@ if mountinfo.exists():
request = config.with_name('decommission.json')
if request.exists() or request.is_symlink():
raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
trusted_directories(config.parent / 'update')
journal = config.parent / 'update' / 'journal.json'
if journal.exists() or journal.is_symlink():
raise SystemExit('An automatic update needs recovery. Let electric-cherry-update.service finish or repair that update before rerunning the installer.')
update_lock = config.with_name('update.lock')
if update_lock.exists() or update_lock.is_symlink():
trusted_file(update_lock)
saved = None
if config.exists() or config.is_symlink():
saved = trusted_json(config)
@@ -108,6 +134,62 @@ elif root.exists() and any(root.iterdir()):
raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
for directory in ('agent', 'instances', 'libraries'):
trusted_directories(root / directory)
activity_lock = root / 'agent' / 'activity.lock'
if activity_lock.exists() or activity_lock.is_symlink():
trusted_file(activity_lock)
PY
}
if [ -f /etc/electric-cherry/agent.json ]; then existing_enrollment=true; fi
if [ -d "$data_dir" ]; then existing_data_root=true; fi
installation_preflight
temporary_dir=$(mktemp -d)
updater_timer_was_active=false
update_lock_held=false
activity_lock_held=false
installer_finished=false
installation_cleanup() {
local ec_install_status=$?
trap - EXIT
if [ "$activity_lock_held" = true ]; then
flock --unlock 8 || true
exec 8>&-
fi
if [ "$update_lock_held" = true ]; then
flock --unlock 9 || true
exec 9>&-
fi
# Recovery must remain scheduled when an upgrade or its second preflight
# fails. Release the worker lock before restarting the prior active timer.
if [ "$installer_finished" != true ] && [ "$updater_timer_was_active" = true ]; then
systemctl start electric-cherry-update.timer >/dev/null 2>&1 || true
fi
rm -rf "$temporary_dir" || true
exit "$ec_install_status"
}
trap installation_cleanup EXIT
controller="${controller%/}"
curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent-update.py" -o "$temporary_dir/ec-agent-update.py"
python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" "$temporary_dir/ec-agent-update.py" <<'PY'
# INSTALLATION_RELEASE: check both downloads before installing or executing either.
import ast, hashlib, json, pathlib, re, sys
manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
agent = pathlib.Path(sys.argv[2]).read_bytes()
updater = pathlib.Path(sys.argv[3]).read_bytes()
if hashlib.sha256(agent).hexdigest() != manifest.get('sha256'):
raise SystemExit('Agent release checksum verification failed.')
if hashlib.sha256(updater).hexdigest() != manifest.get('updaterSha256'):
raise SystemExit('Updater release checksum verification failed.')
if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1 or manifest.get('updaterProtocolVersion') != 1:
raise SystemExit('Unsupported agent or updater release manifest.')
versions = [node.value.value for node in ast.parse(agent).body
if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == 'VERSION' for target in node.targets)
and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)]
if len(versions) != 1 or versions[0] != manifest.get('version') or not re.fullmatch(r'(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)', versions[0]):
raise SystemExit('Agent release version does not match its manifest.')
compile(agent, 'ec-agent.py', 'exec')
compile(updater, 'ec-agent-update.py', 'exec')
PY
if ! command -v docker >/dev/null; then
if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
@@ -118,29 +200,37 @@ if ! command -v docker >/dev/null; then
systemctl enable --now docker
fi
docker info >/dev/null 2>&1 || { echo 'Docker is installed but unavailable. Start or repair it, then rerun.' >&2; exit 1; }
install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
install -d -m 0700 "$data_dir/agent" /etc/electric-cherry
available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
temporary_dir=$(mktemp -d)
trap 'rm -rf "$temporary_dir"' EXIT
controller="${controller%/}"
curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" <<'PY'
import hashlib, json, pathlib, sys
manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
actual = hashlib.sha256(pathlib.Path(sys.argv[2]).read_bytes()).hexdigest()
if actual != manifest.get('sha256'):
raise SystemExit('Agent release checksum verification failed.')
if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1:
raise SystemExit('Unsupported agent release manifest.')
PY
install -d -m 0700 /etc/electric-cherry
# Wait for an already-running updater to finish; never interrupt its rollback.
# The same lock also prevents a timer firing while files are being installed.
if systemctl cat electric-cherry-update.timer >/dev/null 2>&1; then
if systemctl is-active --quiet electric-cherry-update.timer; then
updater_timer_was_active=true
fi
systemctl stop electric-cherry-update.timer
fi
exec 9>/etc/electric-cherry/update.lock
flock --exclusive 9
update_lock_held=true
installation_preflight
# Serialize with running operations, especially a deletion request that could
# otherwise start between preflight and stopping the main agent.
if [ -d "$data_dir/agent" ]; then
exec 8>"$data_dir/agent/activity.lock"
flock --exclusive 8
activity_lock_held=true
installation_preflight
fi
# Stop only the agent while replacing its own executable/journal owner. Existing
# game containers keep running under Docker throughout installation and updates.
if systemctl cat electric-cherry-agent.service >/dev/null 2>&1; then
systemctl stop electric-cherry-agent.service
fi
installation_preflight
install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
install -d -m 0700 "$data_dir/agent"
available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py 2>/dev/null || {
install -d -m 0755 /usr/local/libexec
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
@@ -148,6 +238,7 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
# A separate worker survives stopping/removing the agent service. Its only
# entry point is a fixed, authenticated deletion request prepared by the agent.
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
install -m 0755 "$temporary_dir/ec-agent-update.py" /usr/local/libexec/electric-cherry-update.py
printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
unset enrollment_token
python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
@@ -215,8 +306,41 @@ UMask=0077
[Install]
WantedBy=multi-user.target
EOF
cat > /etc/systemd/system/electric-cherry-update.service <<'EOF'
[Unit]
Description=Update the Electric Cherry Host Agent
After=network-online.target docker.service
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py
TimeoutStartSec=0
UMask=0077
EOF
cat > /etc/systemd/system/electric-cherry-update.timer <<'EOF'
[Unit]
Description=Check for Electric Cherry Host Agent Updates
[Timer]
OnBootSec=2min
OnUnitInactiveSec=5min
RandomizedDelaySec=30s
AccuracySec=1s
Unit=electric-cherry-update.service
[Install]
WantedBy=timers.target
EOF
systemctl daemon-reload
systemctl enable electric-cherry-agent
if [ "$activity_lock_held" = true ]; then
flock --unlock 8
exec 8>&-
activity_lock_held=false
fi
systemctl restart electric-cherry-agent
systemctl is-active --quiet electric-cherry-agent || { echo 'Agent service did not start. Inspect journalctl -u electric-cherry-agent.' >&2; exit 1; }
echo 'Electric Cherry host agent installed and enrolled. Host readiness is shown in Hosts.'
systemctl enable --now electric-cherry-update.timer
installer_finished=true
echo 'Electric Cherry host agent installed and enrolled. Automatic agent updates are enabled; host readiness is shown in Hosts.'