diff --git a/.dockerignore b/.dockerignore index 4fa244e..bf24014 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,3 +10,7 @@ archive *.log test-fixtures data +# Offline release signing material must never enter the build context. +**/.gnupg +**/private-keys-v1.d +**/secring.gpg diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 46b3c92..f07cf9e 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: - name: Install Python and native build tools run: | apt-get update - apt-get install -y --no-install-recommends python3 make g++ + apt-get install -y --no-install-recommends python3 make g++ gpg gpg-agent gpgv - name: Install dependencies run: npm ci diff --git a/agent/ec_agent_update.py b/agent/ec_agent_update.py index eb66a1c..a43c037 100644 --- a/agent/ec_agent_update.py +++ b/agent/ec_agent_update.py @@ -19,13 +19,15 @@ import re import stat import subprocess import sys +import tempfile import time import urllib.parse import urllib.request import uuid -UPDATER_PROTOCOL_VERSION = 1 +UPDATER_PROTOCOL_VERSION = 2 AGENT_CONFIG = Path("/etc/electric-cherry/agent.json") +RELEASE_KEYRING = Path("/etc/electric-cherry/release-signing.gpg") AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py") UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py") UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json") @@ -36,6 +38,7 @@ UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json") AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json") AGENT_SERVICE = "electric-cherry-agent.service" MAX_MANIFEST = 16 * 1024 +MAX_SIGNATURE = 16 * 1024 MAX_SOURCE = 8 * 1024 * 1024 HEALTH_TIMEOUT = 90 VERSION_RE = re.compile(r"^(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})$") @@ -150,6 +153,32 @@ def source_version(source): return declarations[0] +def verify_release_signature(manifest, signature): + # Trust is provisioned by the host operator, never fetched from the + # controller. Check every path component before exposing it to gpgv. + try: + keyring = trusted(RELEASE_KEYRING) + except (UpdateError, OSError): + raise IntegrityError("The host release-signing keyring is missing or unsafe; provision a trusted root-owned keyring") from None + if not signature: + raise IntegrityError("The controller release requires a detached manifest signature") + trusted(UPDATE_DIR, directory=True) + # An isolated home excludes any ambient GnuPG keyring or configuration. + # The exact downloaded bytes are verified before manifest JSON is parsed. + with tempfile.TemporaryDirectory(prefix="verify-", dir=UPDATE_DIR) as temporary: + home = Path(temporary) + manifest_path = home / "manifest.json" + signature_path = home / "manifest.json.sig" + manifest_path.write_bytes(manifest) + signature_path.write_bytes(signature) + try: + result = subprocess.run(["gpgv", "--homedir", str(home), "--keyring", str(keyring), str(signature_path), str(manifest_path)], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30, check=False) + except (OSError, subprocess.TimeoutExpired): + raise IntegrityError("Release signature verification is unavailable; install or repair gpgv on this host") from None + if result.returncode: + raise IntegrityError("The release manifest signature is invalid or is not signed by a trusted release key") + + @contextlib.contextmanager def exclusive_lock(path): trusted(path, missing=True) @@ -242,7 +271,7 @@ class Updater: return state def fetch(self, path, limit): - request = urllib.request.Request(self.installation["controller"] + path, headers={"User-Agent": "electric-cherry-updater/1"}) + request = urllib.request.Request(self.installation["controller"] + path, headers={"User-Agent": "electric-cherry-updater/2"}) class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): raise ContactError("Agent release redirects are not allowed") @@ -274,8 +303,15 @@ class Updater: raise ContactError("The controller release is unavailable; the running agent was left unchanged") from None def manifest(self): + manifest = self.fetch("/api/agent/release/manifest.json", MAX_MANIFEST) try: - value = json.loads(self.fetch("/api/agent/release/manifest.json", MAX_MANIFEST)) + signature = self.fetch("/api/agent/release/manifest.json.sig", MAX_SIGNATURE) + except ContactError: + raise IntegrityError("The controller release signature is unavailable; a signed release is required") from None + self.ensure_directory() + verify_release_signature(manifest, signature) + try: + value = json.loads(manifest) except (ValueError, UnicodeError): raise IntegrityError("The controller release manifest is invalid") from None if not isinstance(value, dict) or type(value.get("protocolVersion")) is not int or value["protocolVersion"] != 1 or value.get("platform") != "debian-amd64": diff --git a/agent/install.sh b/agent/install.sh index 5e15831..b201458 100644 --- a/agent/install.sh +++ b/agent/install.sh @@ -33,7 +33,7 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; } export DEBIAN_FRONTEND=noninteractive apt-get update -qq -apt-get install -y --no-install-recommends ca-certificates curl python3 util-linux +apt-get install -y --no-install-recommends ca-certificates curl python3 util-linux gpgv installation_preflight() { if [ "$existing_enrollment" = true ] && [ ! -f /etc/electric-cherry/agent.json ]; then echo 'The existing enrollment disappeared during installation. Stop and inspect host deletion before trying again.' >&2 @@ -96,6 +96,11 @@ def trusted_json(path): trusted_directories(root) trusted_directories(config.parent) +keyring = config.with_name('release-signing.gpg') +try: + trusted_file(keyring) +except FileNotFoundError: + raise SystemExit('Provision the publisher public key at /etc/electric-cherry/release-signing.gpg through a trusted channel before installing. Never obtain the trust key from the controller.') from None mountinfo = pathlib.Path('/proc/1/mountinfo') if mountinfo.exists(): for line in mountinfo.read_text().splitlines(): @@ -169,19 +174,43 @@ installation_cleanup() { trap installation_cleanup EXIT controller="${controller%/}" curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json" +curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json.sig" -o "$temporary_dir/manifest.json.sig" curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py" curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent-update.py" -o "$temporary_dir/ec-agent-update.py" -python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" "$temporary_dir/ec-agent-update.py" <<'PY' +python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" "$temporary_dir/ec-agent-update.py" /etc/electric-cherry/release-signing.gpg <<'PY' # INSTALLATION_RELEASE: check both downloads before installing or executing either. -import ast, hashlib, json, pathlib, re, sys -manifest = json.loads(pathlib.Path(sys.argv[1]).read_text()) +import ast, hashlib, json, pathlib, re, stat, subprocess, sys, tempfile +manifest_path = pathlib.Path(sys.argv[1]) +manifest_bytes = manifest_path.read_bytes() +signature = manifest_path.with_name('manifest.json.sig') +keyring = pathlib.Path(sys.argv[4]) +for path in (*reversed(keyring.parents), keyring): + info = path.lstat() + expected_type = stat.S_ISREG if path == keyring else stat.S_ISDIR + if not expected_type(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022: + raise SystemExit('Release signing key must be a regular root-owned file in protected directories.') +if len(manifest_bytes) > 16 * 1024 or signature.stat().st_size > 16 * 1024: + raise SystemExit('Release signing metadata exceeds its size limit.') +try: + with tempfile.TemporaryDirectory(prefix='gpgv-', dir=manifest_path.parent) as home: + verified = subprocess.run(['gpgv', '--homedir', home, '--keyring', str(keyring), + str(signature), '-'], input=manifest_bytes, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=30, check=False) +except (OSError, subprocess.TimeoutExpired): + raise SystemExit('Release signature verification requires a working gpgv installation.') from None +if verified.returncode: + raise SystemExit('Release signature verification failed. The installed agent was left unchanged.') +manifest = json.loads(manifest_bytes) +if not isinstance(manifest, dict): + raise SystemExit('Unsupported agent or updater release manifest.') agent = pathlib.Path(sys.argv[2]).read_bytes() updater = pathlib.Path(sys.argv[3]).read_bytes() if hashlib.sha256(agent).hexdigest() != manifest.get('sha256'): raise SystemExit('Agent release checksum verification failed.') if hashlib.sha256(updater).hexdigest() != manifest.get('updaterSha256'): raise SystemExit('Updater release checksum verification failed.') -if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1 or manifest.get('updaterProtocolVersion') != 1: +if manifest.get('platform') != 'debian-amd64' or type(manifest.get('protocolVersion')) is not int or manifest['protocolVersion'] != 1 or type(manifest.get('updaterProtocolVersion')) is not int or manifest['updaterProtocolVersion'] != 2: raise SystemExit('Unsupported agent or updater release manifest.') versions = [node.value.value for node in ast.parse(agent).body if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == 'VERSION' for target in node.targets) diff --git a/agent/package_release.py b/agent/package_release.py index 65312a8..d0f5c71 100644 --- a/agent/package_release.py +++ b/agent/package_release.py @@ -5,14 +5,11 @@ import hashlib import json from pathlib import Path import re -import shutil -def package(source, target): - """Read version metadata without importing or executing release code.""" - source, target = Path(source), Path(target) - agent = (source / "ec_agent.py").read_bytes() - updater = (source / "ec_agent_update.py").read_bytes() +def manifest_bytes(agent, updater, installer): + """Return the deterministic bytes signed by the offline release signer.""" + # Inspect source without importing or executing release code. versions = [node.value.value for node in ast.parse(agent).body if isinstance(node, ast.Assign) and any(isinstance(name, ast.Name) and name.id == "VERSION" for name in node.targets) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)] @@ -21,14 +18,33 @@ def package(source, target): compile(agent, "ec_agent.py", "exec") compile(updater, "ec_agent_update.py", "exec") manifest = {"version": versions[0], "protocolVersion": 1, "platform": "debian-amd64", - "sha256": hashlib.sha256(agent).hexdigest(), "updaterProtocolVersion": 1, - "updaterSha256": hashlib.sha256(updater).hexdigest()} + "sha256": hashlib.sha256(agent).hexdigest(), "updaterProtocolVersion": 2, + "updaterSha256": hashlib.sha256(updater).hexdigest(), + "installerSha256": hashlib.sha256(installer).hexdigest()} + return (json.dumps(manifest, indent=2) + "\n").encode("utf-8") + + +def package(source, target): + """Read version metadata without importing or executing release code.""" + source, target = Path(source), Path(target) + agent = (source / "ec_agent.py").read_bytes() + updater = (source / "ec_agent_update.py").read_bytes() + installer = (source / "install.sh").read_bytes() + manifest = manifest_bytes(agent, updater, installer) target.mkdir(parents=True, exist_ok=True) + try: + previous_manifest = (target / "manifest.json").read_bytes() + except FileNotFoundError: + previous_manifest = None + if previous_manifest != manifest: + # Builds never sign. Keep an offline signature only across identical + # rebuilds; changed releases remain unsigned until reviewed and signed. + (target / "manifest.json.sig").unlink(missing_ok=True) (target / "ec-agent.py").write_bytes(agent) (target / "ec-agent-update.py").write_bytes(updater) - shutil.copyfile(source / "install.sh", target / "install.sh") + (target / "install.sh").write_bytes(installer) temporary_manifest = target / ".manifest.json.tmp" - temporary_manifest.write_text(json.dumps(manifest, indent=2) + "\n") + temporary_manifest.write_bytes(manifest) temporary_manifest.replace(target / "manifest.json") diff --git a/agent/sign_release.py b/agent/sign_release.py new file mode 100644 index 0000000..1677d76 --- /dev/null +++ b/agent/sign_release.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Sign a reviewed, packaged release using a separate offline GnuPG home. + +This command is never invoked by controller builds or runtime. Keep +the signing key and its GnuPG home outside this checkout and Docker context. +""" +import argparse +from pathlib import Path +import re +import subprocess +import tempfile + +from package_release import manifest_bytes + + +def reviewed_manifest(release): + manifest = (release / "manifest.json").read_bytes() + expected = manifest_bytes((release / "ec-agent.py").read_bytes(), + (release / "ec-agent-update.py").read_bytes(), + (release / "install.sh").read_bytes()) + if manifest != expected: + raise ValueError("Release manifest does not match packaged artifacts; package and review the release again") + return manifest + + +def sign_release(release, gnupghome, fingerprint): + """Sign exact manifest bytes, without importing any private key material.""" + release, gnupghome = Path(release).resolve(), Path(gnupghome).resolve() + repository = Path(__file__).resolve().parent.parent + if not re.fullmatch(r"(?:[A-Fa-f0-9]{40}|[A-Fa-f0-9]{64})", fingerprint): + raise ValueError("Signing requires a full OpenPGP key fingerprint (40 or 64 hexadecimal characters)") + if gnupghome.is_relative_to(repository) or gnupghome.is_relative_to(release): + raise ValueError("Keep the signing GnuPG home outside the repository and release directory") + if not gnupghome.is_dir(): + raise ValueError("The offline signing GnuPG home must already exist") + manifest = reviewed_manifest(release) + with tempfile.TemporaryDirectory(prefix=".sign-release-", dir=release) as directory: + staged_manifest = Path(directory) / "manifest.json" + signature = Path(directory) / "manifest.json.sig" + staged_manifest.write_bytes(manifest) + subprocess.run([ + "gpg", "--no-options", "--homedir", str(gnupghome), "--batch", "--yes", + "--armor", "--detach-sign", "--digest-algo", "SHA512", + "--local-user", fingerprint.upper() + "!", "--output", str(signature), + str(staged_manifest), + ], check=True) + if not signature.read_bytes().startswith(b"-----BEGIN PGP SIGNATURE-----\n"): + raise ValueError("GnuPG did not produce an ASCII-armored detached signature") + if reviewed_manifest(release) != manifest: + raise ValueError("Release changed during signing; review and sign the release again") + signature.chmod(0o644) + signature.replace(release / "manifest.json.sig") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--release-dir", type=Path, + default=Path(__file__).resolve().parent.parent / "static" / "agent") + parser.add_argument("--gnupghome", required=True, type=Path, + help="Existing offline GnuPG home outside the checkout/build context") + parser.add_argument("--fingerprint", required=True, + help="Full fingerprint of the exact signing key (not a key ID or email)") + args = parser.parse_args() + try: + sign_release(args.release_dir, args.gnupghome, args.fingerprint) + except (OSError, ValueError, SyntaxError, subprocess.CalledProcessError) as cause: + parser.exit(1, f"Release signing failed: {cause}\n") + print(f"Signed {args.release_dir / 'manifest.json'}") + + +if __name__ == "__main__": + main() diff --git a/agent/tests/test_installer.py b/agent/tests/test_installer.py index c4937ca..0127b04 100644 --- a/agent/tests/test_installer.py +++ b/agent/tests/test_installer.py @@ -11,7 +11,7 @@ import subprocess import sys import tempfile import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch SOURCE = (Path(__file__).resolve().parents[1] / "install.sh").read_text() @@ -33,6 +33,8 @@ class InstallerTests(unittest.TestCase): self.root = self.base / "managed" self.config = self.base / "configuration" / "agent.json" self.config.parent.mkdir() + self.keyring = self.config.with_name('release-signing.gpg') + self.keyring.write_bytes(b'publisher public key') self.controller = "https://controller.example" self.saved = {"root": str(self.root), "controller": self.controller, "hostId": "host-one", "credential": "private"} @@ -80,6 +82,17 @@ class InstallerTests(unittest.TestCase): self.preflight() self.assertEqual((self.root / "unrelated.txt").read_text(), "keep") + def test_trust_key_must_be_preprovisioned_and_protected(self): + self.keyring.unlink() + with self.assertRaisesRegex(SystemExit, 'Provision the publisher public key'): + self.preflight() + self.keyring.write_bytes(b'publisher public key') + for override in ({'uid': 1000}, {'mode': stat.S_IFREG | 0o666}, {'mode': stat.S_IFLNK | 0o777}): + with self.subTest(override=override): + self.overrides[str(self.keyring)] = override + with self.assertRaisesRegex(SystemExit, 'regular, root-owned'): + self.preflight() + def test_known_legacy_layout_can_upgrade_and_marker_binds_exact_enrollment(self): self.legacy_installation() original_config = self.config.read_bytes() @@ -337,20 +350,31 @@ docker() { class ReleaseTests(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() - self.source = Path(self.temporary.name) / "source" - self.target = Path(self.temporary.name) / "release" + self.source = Path(self.temporary.name).resolve() / "source" + self.target = Path(self.temporary.name).resolve() / "release" self.source.mkdir() (self.source / "ec_agent.py").write_text('VERSION = "3.12.7"\nraise RuntimeError("must never execute during packaging")\n') (self.source / "ec_agent_update.py").write_text('raise RuntimeError("must never execute during verification")\n') (self.source / "install.sh").write_text("#!/bin/bash\n") PACKAGER.package(self.source, self.target) + self.keyring = self.source.parent / 'release-signing.gpg' + self.keyring.write_bytes(b'publisher public key') + (self.target / 'manifest.json.sig').write_bytes(b'detached signature') + self.gpgv = Mock(return_value=subprocess.CompletedProcess([], 0)) def tearDown(self): self.temporary.cleanup() def verify(self): - argv = ["-", str(self.target / "manifest.json"), str(self.target / "ec-agent.py"), str(self.target / "ec-agent-update.py")] - with patch.object(sys, "argv", argv): + argv = ["-", str(self.target / "manifest.json"), str(self.target / "ec-agent.py"), str(self.target / "ec-agent-update.py"), str(self.keyring)] + original_lstat = Path.lstat + def root_owned(path): + fields = list(original_lstat(path)) + fields[4] = 0 + if stat.S_ISDIR(fields[0]): + fields[0] &= ~0o022 + return os.stat_result(fields) + with patch.object(sys, "argv", argv), patch.object(Path, 'lstat', root_owned), patch.object(subprocess, 'run', self.gpgv): exec(compile(RELEASE, "installer-release", "exec"), {}) def manifest(self, **changes): @@ -363,7 +387,7 @@ class ReleaseTests(unittest.TestCase): def test_package_derives_version_and_hashes_both_sources_without_executing_them(self): manifest = self.manifest() self.assertEqual(manifest["version"], "3.12.7") - self.assertEqual(manifest["updaterProtocolVersion"], 1) + self.assertEqual(manifest["updaterProtocolVersion"], 2) self.assertEqual(manifest["sha256"], hashlib.sha256((self.source / "ec_agent.py").read_bytes()).hexdigest()) self.assertEqual(manifest["updaterSha256"], hashlib.sha256((self.source / "ec_agent_update.py").read_bytes()).hexdigest()) self.assertEqual((self.target / "ec-agent-update.py").read_bytes(), (self.source / "ec_agent_update.py").read_bytes()) @@ -380,11 +404,38 @@ class ReleaseTests(unittest.TestCase): path.write_bytes(original) def test_installer_rejects_missing_or_unsupported_updater_protocol(self): - for protocol in (None, 0, 2, "1"): + for protocol in (None, 0, 1, True, "2"): self.manifest(updaterProtocolVersion=protocol) with self.subTest(protocol=protocol), self.assertRaisesRegex(SystemExit, "Unsupported agent or updater"): self.verify() + def test_signature_is_verified_against_pinned_key_before_manifest_parsing(self): + self.verify() + args, kwargs = self.gpgv.call_args + self.assertEqual(args[0][0], 'gpgv') + self.assertEqual(args[0][-4:], ['--keyring', str(self.keyring), str(self.target / 'manifest.json.sig'), '-']) + self.assertEqual(kwargs['input'], (self.target / 'manifest.json').read_bytes()) + self.assertEqual(kwargs['timeout'], 30) + self.assertFalse(Path(args[0][2]).exists()) + (self.target / 'manifest.json').write_bytes(b'not even JSON') + self.gpgv.return_value = subprocess.CompletedProcess([], 1) + with self.assertRaisesRegex(SystemExit, 'signature verification failed'): + self.verify() + + def test_missing_signature_key_or_gpgv_prevents_installation(self): + for path in (self.keyring, self.target / 'manifest.json.sig'): + original = path.read_bytes() + path.unlink() + self.gpgv.reset_mock() + with self.subTest(path=path), self.assertRaises(FileNotFoundError): + self.verify() + self.gpgv.assert_not_called() + path.write_bytes(original) + for failure in (FileNotFoundError(), subprocess.TimeoutExpired('gpgv', 30)): + self.gpgv.side_effect = failure + with self.subTest(failure=failure), self.assertRaisesRegex(SystemExit, 'working gpgv'): + self.verify() + def test_installer_rejects_manifest_version_mismatch_and_invalid_code(self): self.manifest(version="3.12.8") with self.assertRaisesRegex(SystemExit, "version does not match"): diff --git a/agent/tests/test_release_signing.py b/agent/tests/test_release_signing.py new file mode 100644 index 0000000..8701f82 --- /dev/null +++ b/agent/tests/test_release_signing.py @@ -0,0 +1,244 @@ +"""Release packaging is unsigned; only the explicit offline command may sign.""" +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + + +AGENT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("package_release", AGENT / "package_release.py") +PACKAGER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PACKAGER) +SPEC = importlib.util.spec_from_file_location("sign_release", AGENT / "sign_release.py") +SIGNER = importlib.util.module_from_spec(SPEC) +with patch.dict(sys.modules, {"package_release": PACKAGER}): + SPEC.loader.exec_module(SIGNER) +SPEC = importlib.util.spec_from_file_location("signed_pipeline_updater", AGENT / "ec_agent_update.py") +UPDATER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(UPDATER) + + +class ReleaseFixture: + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.base = Path(self.temporary.name).resolve() + self.source = self.base / "source" + self.source.mkdir() + self.target = self.base / "release" + self.gnupghome = self.base / "offline-keyring" + self.gnupghome.mkdir(mode=0o700) + self.fingerprint = "A" * 40 + (self.source / "ec_agent.py").write_text('VERSION = "4.2.0"\nraise RuntimeError("Do not run")\n') + (self.source / "ec_agent_update.py").write_text('raise RuntimeError("Do not run")\n') + (self.source / "install.sh").write_text("#!/bin/bash\nexit 1\n") + PACKAGER.package(self.source, self.target) + + def tearDown(self): + self.temporary.cleanup() + + +class PackagedReleaseTests(ReleaseFixture, unittest.TestCase): + def test_manifest_is_deterministic_and_covers_every_executable(self): + first = (self.target / "manifest.json").read_bytes() + PACKAGER.package(self.source, self.target) + self.assertEqual(first, (self.target / "manifest.json").read_bytes()) + manifest = json.loads(first) + self.assertEqual(manifest["updaterProtocolVersion"], 2) + for field, filename in (("sha256", "ec-agent.py"), + ("updaterSha256", "ec-agent-update.py"), + ("installerSha256", "install.sh")): + self.assertEqual(manifest[field], hashlib.sha256((self.target / filename).read_bytes()).hexdigest()) + self.assertFalse((self.target / "manifest.json.sig").exists()) + + def test_packaging_preserves_signature_bytes_only_for_identical_manifest(self): + signature = self.target / "manifest.json.sig" + for filename in ("ec_agent.py", "ec_agent_update.py", "install.sh"): + with self.subTest(filename=filename): + signature.write_bytes(b"offline signature") + with patch.object(subprocess, "run", side_effect=AssertionError("Builds must never sign")): + PACKAGER.package(self.source, self.target) + self.assertEqual(signature.read_bytes(), b"offline signature") + source = self.source / filename + source.write_bytes(source.read_bytes() + b"# reviewed change\n") + PACKAGER.package(self.source, self.target) + self.assertFalse(signature.exists()) + + def test_orphaned_signature_is_removed_when_manifest_was_missing(self): + (self.target / "manifest.json").unlink() + (self.target / "manifest.json.sig").write_bytes(b"orphaned signature") + PACKAGER.package(self.source, self.target) + self.assertFalse((self.target / "manifest.json.sig").exists()) + + def test_signer_requires_full_fingerprint_before_invoking_gpg(self): + for fingerprint in ("", "release@example.com", "A" * 16, "Z" * 40, "A" * 41): + with self.subTest(fingerprint=fingerprint), patch.object(subprocess, "run") as run: + with self.assertRaisesRegex(ValueError, "full OpenPGP key fingerprint"): + SIGNER.sign_release(self.target, self.gnupghome, fingerprint) + run.assert_not_called() + + def test_signer_rejects_private_key_homes_in_repository_or_release(self): + for home in (AGENT, self.target, self.target / "private"): + with self.subTest(home=home), patch.object(subprocess, "run") as run: + with self.assertRaisesRegex(ValueError, "outside the repository and release"): + SIGNER.sign_release(self.target, home, self.fingerprint) + run.assert_not_called() + + def test_signer_refuses_unreviewed_or_inconsistent_release_bytes(self): + for filename in ("ec-agent.py", "ec-agent-update.py", "install.sh", "manifest.json"): + with self.subTest(filename=filename), patch.object(subprocess, "run") as run: + path = self.target / filename + original = path.read_bytes() + path.write_bytes(original + b"\n") + with self.assertRaisesRegex(ValueError, "does not match packaged artifacts"): + SIGNER.sign_release(self.target, self.gnupghome, self.fingerprint) + run.assert_not_called() + path.write_bytes(original) + + def test_failed_signing_does_not_replace_existing_signature(self): + signature = self.target / "manifest.json.sig" + signature.write_bytes(b"previous valid signature") + with patch.object(subprocess, "run", side_effect=subprocess.CalledProcessError(2, "gpg")): + with self.assertRaises(subprocess.CalledProcessError): + SIGNER.sign_release(self.target, self.gnupghome, self.fingerprint) + self.assertEqual(signature.read_bytes(), b"previous valid signature") + self.assertEqual(list(self.target.glob(".sign-release-*")), []) + + def test_release_mutation_during_signing_never_publishes_a_signature(self): + def mutate(command, **kwargs): + Path(command[command.index("--output") + 1]).write_bytes(b"-----BEGIN PGP SIGNATURE-----\nstaged\n") + (self.target / "ec-agent.py").write_bytes(b'VERSION = "9.0.0"\n') + with patch.object(subprocess, "run", side_effect=mutate): + with self.assertRaisesRegex(ValueError, "does not match packaged artifacts"): + SIGNER.sign_release(self.target, self.gnupghome, self.fingerprint) + self.assertFalse((self.target / "manifest.json.sig").exists()) + + +@unittest.skipUnless(shutil.which("gpg") and shutil.which("gpgv"), "GnuPG required for real offline signing tests") +class OfflineSigningTests(ReleaseFixture, unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.keys = tempfile.TemporaryDirectory(prefix="ec-sign-") + cls.keyhome = Path(cls.keys.name) + cls.keyhome.chmod(0o700) + command = ["gpg", "--no-options", "--homedir", str(cls.keyhome), "--batch"] + subprocess.run(command + ["--pinentry-mode", "loopback", "--passphrase", "", + "--quick-generate-key", "Ephemeral release test ", + "ed25519", "sign", "0"], check=True, capture_output=True) + listing = subprocess.run(command + ["--with-colons", "--list-secret-keys"], + check=True, capture_output=True, text=True).stdout + cls.key_fingerprint = next(line.split(":")[9] for line in listing.splitlines() if line.startswith("fpr:")) + cls.public_key = subprocess.run(command + ["--export", cls.key_fingerprint], + check=True, capture_output=True).stdout + + @classmethod + def tearDownClass(cls): + if shutil.which("gpgconf"): + subprocess.run(["gpgconf", "--homedir", str(cls.keyhome), "--kill", "gpg-agent"], + check=False, capture_output=True) + cls.keys.cleanup() + + def setUp(self): + super().setUp() + self.keyring = self.base / "release-signing.gpg" + self.keyring.write_bytes(self.public_key) + self.update_dir = self.base / "update" + self.update_dir.mkdir(mode=0o700) + original_lstat = Path.lstat + protected_paths = {self.keyring, *self.keyring.parents, self.update_dir} + def root_owned(path): + info = original_lstat(path) + if path not in protected_paths: + return info + fields = list(info) + fields[4] = 0 + if stat.S_ISDIR(fields[0]): + fields[0] &= ~0o022 + return os.stat_result(fields) + self.root_owned = root_owned + + def verify_installer(self): + blocks = re.findall(r"<<'PY'\n(.*?)\nPY", (AGENT / "install.sh").read_text(), re.DOTALL) + release = next(block for block in blocks if "# INSTALLATION_RELEASE:" in block) + argv = ["-", str(self.target / "manifest.json"), str(self.target / "ec-agent.py"), + str(self.target / "ec-agent-update.py"), str(self.keyring)] + # Only filesystem ownership is simulated; real GnuPG verifies the + # separately exported public key and the offline signer's exact bytes. + with patch.object(sys, "argv", argv), patch.object(Path, "lstat", self.root_owned): + exec(compile(release, "installer-release", "exec"), {}) + + def verify_updater(self): + with patch.object(UPDATER, "RELEASE_KEYRING", self.keyring), \ + patch.object(UPDATER, "UPDATE_DIR", self.update_dir), \ + patch.object(Path, "lstat", self.root_owned): + UPDATER.verify_release_signature((self.target / "manifest.json").read_bytes(), + (self.target / "manifest.json.sig").read_bytes()) + + def verify(self): + return subprocess.run([ + "gpg", "--no-options", "--homedir", str(self.keyhome), "--batch", "--verify", + str(self.target / "manifest.json.sig"), str(self.target / "manifest.json"), + ], capture_output=True) + + def test_offline_signature_verifies_exact_bytes_and_survives_identical_build(self): + SIGNER.sign_release(self.target, self.keyhome, self.key_fingerprint) + signature = (self.target / "manifest.json.sig").read_bytes() + self.assertTrue(signature.startswith(b"-----BEGIN PGP SIGNATURE-----\n")) + self.assertEqual(self.verify().returncode, 0) + PACKAGER.package(self.source, self.target) + self.assertEqual((self.target / "manifest.json.sig").read_bytes(), signature) + self.assertEqual(self.verify().returncode, 0) + with (self.target / "manifest.json").open("ab") as manifest: + manifest.write(b" ") + self.assertNotEqual(self.verify().returncode, 0) + + def test_unknown_signing_key_fails_without_publishing_a_signature(self): + with self.assertRaises(subprocess.CalledProcessError): + SIGNER.sign_release(self.target, self.keyhome, "F" * 40) + self.assertFalse((self.target / "manifest.json.sig").exists()) + + def test_offline_signed_release_is_accepted_by_installer_and_updater(self): + SIGNER.sign_release(self.target, self.keyhome, self.key_fingerprint) + self.verify_installer() + self.verify_updater() + + def test_controller_cannot_authorize_changed_code_by_rehashing_manifest(self): + SIGNER.sign_release(self.target, self.keyhome, self.key_fingerprint) + for filename in ("ec-agent.py", "ec-agent-update.py"): + with self.subTest(filename=filename): + path = self.target / filename + original = path.read_bytes() + path.write_bytes(original + b"# controller's unauthorized change\n") + (self.target / "manifest.json").write_bytes(PACKAGER.manifest_bytes( + (self.target / "ec-agent.py").read_bytes(), + (self.target / "ec-agent-update.py").read_bytes(), + (self.target / "install.sh").read_bytes())) + with self.assertRaisesRegex(SystemExit, "signature verification failed"): + self.verify_installer() + with self.assertRaisesRegex(UPDATER.IntegrityError, "signature is invalid"): + self.verify_updater() + path.write_bytes(original) + + def test_signed_manifest_rejects_changed_agent_and_updater_downloads(self): + SIGNER.sign_release(self.target, self.keyhome, self.key_fingerprint) + for filename, message in (("ec-agent.py", "Agent release checksum"), + ("ec-agent-update.py", "Updater release checksum")): + with self.subTest(filename=filename): + path = self.target / filename + original = path.read_bytes() + path.write_bytes(original + b"# controller's unauthorized change\n") + with self.assertRaisesRegex(SystemExit, message): + self.verify_installer() + path.write_bytes(original) + + +if __name__ == "__main__": + unittest.main() diff --git a/agent/tests/test_updates.py b/agent/tests/test_updates.py index f5b306b..1503c5b 100644 --- a/agent/tests/test_updates.py +++ b/agent/tests/test_updates.py @@ -5,7 +5,9 @@ import io import json import os from pathlib import Path +import shutil import stat +import subprocess import tempfile import time import unittest @@ -41,6 +43,7 @@ class UpdateTests(unittest.TestCase): directory.mkdir(parents=True) paths = { "AGENT_CONFIG": self.config_dir / "agent.json", + "RELEASE_KEYRING": self.config_dir / "release-signing.gpg", "AGENT_EXECUTABLE": self.bin_dir / "electric-cherry-agent.py", "UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py", "UNINSTALL_REQUEST": self.config_dir / "decommission.json", @@ -62,9 +65,12 @@ class UpdateTests(unittest.TestCase): if directory and not path.is_dir() or not directory and not path.is_file(): raise module.UpdateError("Wrong file type") return path - self.patches.extend([patch.object(module, "trusted", side_effect=trusted_for_test), patch.object(module, "sync_parent"), patch.object(module, "systemctl", side_effect=self.systemctl)]) + self.signature_patch = patch.object(module, "verify_release_signature") + self.patches.extend([patch.object(module, "trusted", side_effect=trusted_for_test), patch.object(module, "sync_parent"), patch.object(module, "systemctl", side_effect=self.systemctl), self.signature_patch]) for item in self.patches: - item.start() + mocked = item.start() + if item is self.signature_patch: + self.signature_verifier = mocked self.config = {"hostId": "host-one", "root": str(self.root), "controller": "https://controller.example", "credential": "secret-enrollment-credential"} self.config_bytes = json.dumps(self.config).encode() module.AGENT_CONFIG.write_bytes(self.config_bytes) @@ -78,7 +84,8 @@ class UpdateTests(unittest.TestCase): self.write_health = True self.downloads = [] self.source = CANDIDATE - self.release = {"version": "1.1.0", "sha256": module.sha256(CANDIDATE), "protocolVersion": 1, "updaterProtocolVersion": 1, "platform": "debian-amd64"} + self.signature = b"detached signature" + self.release = {"version": "1.1.0", "sha256": module.sha256(CANDIDATE), "protocolVersion": 1, "updaterProtocolVersion": 2, "platform": "debian-amd64"} def tearDown(self): for item in reversed(self.patches): @@ -108,6 +115,10 @@ class UpdateTests(unittest.TestCase): self.downloads.append(path) if path.endswith("manifest.json"): return json.dumps(self.release).encode() + if path.endswith("manifest.json.sig"): + if isinstance(self.signature, Exception): + raise self.signature + return self.signature if path.endswith("ec-agent.py"): return self.source self.fail("Unexpected download " + path) @@ -128,6 +139,7 @@ class UpdateTests(unittest.TestCase): def test_verified_upgrade_waits_for_authenticated_candidate_health(self): self.assertEqual(self.updater().run(), "updated") + self.signature_verifier.assert_called_once_with(json.dumps(self.release).encode(), self.signature) self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE) self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), CANDIDATE) self.assertEqual(module.AGENT_CONFIG.read_bytes(), self.config_bytes) @@ -137,6 +149,36 @@ class UpdateTests(unittest.TestCase): self.assertFalse((module.UPDATE_DIR / "agent.backup.py").exists()) self.assertEqual(next(command for command in self.commands if command[0] != "show"), ("stop", module.AGENT_SERVICE)) + def test_invalid_signature_rejects_attacker_source_and_matching_manifest_checksum(self): + self.source = b'VERSION = "1.1.0"\nraise RuntimeError("attacker code")\n' + self.release["sha256"] = module.sha256(self.source) + self.signature_verifier.side_effect = module.IntegrityError("Invalid release signature") + self.assertEqual(self.updater().run(), "failed") + self.assert_unchanged() + self.assertEqual(self.commands, []) + self.assertEqual(self.downloads, ["/api/agent/release/manifest.json", "/api/agent/release/manifest.json.sig"]) + self.assertFalse(module.UPDATE_JOURNAL.exists()) + self.assertFalse((module.UPDATE_DIR / "failed-release.json").exists()) + self.assertNotIn("targetVersion", self.status()) + + def test_signature_is_verified_before_manifest_json_is_parsed(self): + updater = self.updater() + updater.fetch = lambda path, limit: b"untrusted invalid JSON" if path.endswith("manifest.json") else self.signature + self.signature_verifier.side_effect = module.IntegrityError("Invalid release signature") + with patch.object(module.json, "loads", side_effect=AssertionError("Parsed unsigned metadata")): + with self.assertRaisesRegex(module.IntegrityError, "Invalid release signature"): + updater.manifest() + self.signature_verifier.assert_called_once_with(b"untrusted invalid JSON", self.signature) + + def test_missing_signature_never_downloads_code_or_stops_agent(self): + self.signature = module.ContactError("No signature is published") + self.assertEqual(self.updater().run(), "failed") + self.assertIn("signature", self.status()["error"]) + self.assert_unchanged() + self.assertEqual(self.commands, []) + self.assertFalse(any(path.endswith("ec-agent.py") for path in self.downloads)) + self.signature_verifier.assert_not_called() + def test_equal_or_older_versions_do_not_download_or_restart(self): for target in ("1.0.0", "0.9.9"): with self.subTest(target=target): @@ -207,7 +249,7 @@ class UpdateTests(unittest.TestCase): self.downloads.clear() self.source = CANDIDATE self.assertEqual(self.updater().run(), "failed") - self.assertEqual(self.downloads, ["/api/agent/release/manifest.json"]) + self.assertEqual(self.downloads, ["/api/agent/release/manifest.json", "/api/agent/release/manifest.json.sig"]) def test_invalid_or_mismatched_python_version_is_never_installed(self): for source in (b'VERSION = "1.1.0"\ninvalid syntax ?\n', b'VERSION = "5.0.0"\n'): @@ -220,7 +262,7 @@ class UpdateTests(unittest.TestCase): def test_incompatible_protocol_platform_or_version_never_downloads_code(self): original = dict(self.release) - for key, value in (("protocolVersion", 2), ("protocolVersion", True), ("platform", "linux-arm64"), ("version", "1.1.0-rc1"), ("version", "01.1.0"), ("updaterProtocolVersion", 2)): + for key, value in (("protocolVersion", 2), ("protocolVersion", True), ("platform", "linux-arm64"), ("version", "1.1.0-rc1"), ("version", "01.1.0"), ("updaterProtocolVersion", 1), ("updaterProtocolVersion", 3)): with self.subTest(key=key, value=value): self.release = {**original, key: value} self.assertIn(self.updater().run(), ("failed", "deferred")) @@ -239,7 +281,7 @@ class UpdateTests(unittest.TestCase): self.assertEqual(sum(command == ("start", module.AGENT_SERVICE) for command in self.commands), 2) self.downloads.clear() self.assertEqual(self.updater().run(), "failed") - self.assertEqual(self.downloads, ["/api/agent/release/manifest.json"]) + self.assertEqual(self.downloads, ["/api/agent/release/manifest.json", "/api/agent/release/manifest.json.sig"]) def test_changed_artifact_of_failed_target_can_be_retried(self): updater = self.updater() @@ -266,9 +308,12 @@ class UpdateTests(unittest.TestCase): def test_crash_between_executable_replacements_recovers_from_backups(self): self.crash_during_second_replacement() self.downloads.clear() + self.signature_verifier.reset_mock() + self.signature_verifier.side_effect = module.IntegrityError("Signing keyring unavailable during recovery") self.assertEqual(self.updater().run(), "failed") self.assert_unchanged() self.assertEqual(self.downloads, []) + self.signature_verifier.assert_not_called() self.assertTrue(self.active) self.assertFalse(module.UPDATE_JOURNAL.exists()) @@ -413,6 +458,108 @@ class UpdateSecurityTests(unittest.TestCase): with self.subTest(value=value), self.assertRaises(module.IntegrityError): module.version(value) + def test_release_keyring_rejects_untrusted_ownership_permissions_and_file_types(self): + for mode, uid in ((stat.S_IFREG | 0o644, 1000), (stat.S_IFREG | 0o664, 0), (stat.S_IFREG | 0o646, 0), (stat.S_IFLNK | 0o777, 0), (stat.S_IFDIR | 0o755, 0)): + def file_info(path): + selected_mode, selected_uid = (mode, uid) if path == module.RELEASE_KEYRING else (stat.S_IFDIR | 0o755, 0) + return os.stat_result((selected_mode, 1, 1, 1, selected_uid, 0, 0, 0, 0, 0)) + with self.subTest(mode=mode, uid=uid), patch.object(Path, "lstat", autospec=True, side_effect=file_info), patch.object(module.subprocess, "run") as run: + with self.assertRaisesRegex(module.IntegrityError, "keyring is missing or unsafe"): + module.verify_release_signature(b"manifest", b"signature") + run.assert_not_called() + + +class SignatureVerificationFixture: + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.base = Path(self.temporary.name) + self.keyring = self.base / "release-signing.gpg" + self.keyring.write_bytes(b"trusted host public keys") + self.update_dir = self.base / "update" + self.update_dir.mkdir(mode=0o700) + for name, value in (("RELEASE_KEYRING", self.keyring), ("UPDATE_DIR", self.update_dir)): + entry = patch.object(module, name, value) + entry.start() + self.addCleanup(entry.stop) + # Test fixtures are owned by the test runner rather than system root. + # Root ownership and unsafe modes are checked independently above. + entry = patch.object(module, "trusted", side_effect=lambda path, **_: Path(path)) + self.trusted = entry.start() + self.addCleanup(entry.stop) + + +class SignatureVerificationTests(SignatureVerificationFixture, unittest.TestCase): + def test_gpgv_uses_only_host_keyring_and_exact_bytes_with_an_isolated_home(self): + manifest = b'{"version": "1.2.3"}\n' + signature = b"binary detached signature\x00\xff" + def verify(command, **kwargs): + home = Path(command[2]) + self.assertEqual(command[:2], ["gpgv", "--homedir"]) + self.assertEqual(command[3:5], ["--keyring", str(self.keyring)]) + self.assertEqual(home.parent, self.update_dir) + self.assertEqual(stat.S_IMODE(home.stat().st_mode), 0o700) + self.assertEqual(Path(command[5]).read_bytes(), signature) + self.assertEqual(Path(command[6]).read_bytes(), manifest) + self.assertEqual(kwargs["timeout"], 30) + self.assertEqual(kwargs["stdin"], subprocess.DEVNULL) + return subprocess.CompletedProcess(command, 0) + with patch.object(module.subprocess, "run", side_effect=verify) as run: + module.verify_release_signature(manifest, signature) + run.assert_called_once() + self.assertEqual(self.trusted.call_args_list[0].args, (self.keyring,)) + self.assertEqual(list(self.update_dir.iterdir()), []) + + def test_missing_keyring_or_signature_never_runs_a_verifier(self): + with patch.object(module.subprocess, "run") as run: + self.trusted.side_effect = module.UpdateError("Missing keyring") + with self.assertRaisesRegex(module.IntegrityError, "keyring"): + module.verify_release_signature(b"manifest", b"signature") + self.trusted.side_effect = lambda path, **_: Path(path) + with self.assertRaisesRegex(module.IntegrityError, "detached manifest signature"): + module.verify_release_signature(b"manifest", b"") + run.assert_not_called() + + def test_bad_unknown_or_unavailable_verifier_fails_closed_and_cleans_temporary_files(self): + for result in (subprocess.CompletedProcess([], 1), subprocess.CompletedProcess([], 2), FileNotFoundError("gpgv missing"), subprocess.TimeoutExpired("gpgv", 30)): + with self.subTest(result=result): + kwargs = {"side_effect": result} if isinstance(result, Exception) else {"return_value": result} + with patch.object(module.subprocess, "run", **kwargs): + with self.assertRaises(module.IntegrityError): + module.verify_release_signature(b"manifest", b"signature") + self.assertEqual(list(self.update_dir.iterdir()), []) + + +@unittest.skipUnless(shutil.which("gpg") and shutil.which("gpgv"), "GnuPG signing and verification tools are required") +class RealSignatureVerificationTests(SignatureVerificationFixture, unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.keys = tempfile.TemporaryDirectory() + cls.addClassCleanup(cls.keys.cleanup) + cls.homes = [] + for index in range(2): + home = Path(cls.keys.name) / str(index) + home.mkdir(mode=0o700) + cls.homes.append(home) + subprocess.run(["gpg", "--homedir", str(home), "--batch", "--pinentry-mode", "loopback", "--passphrase", "", "--quick-generate-key", f"Release Test {index} ", "ed25519", "sign", "0"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + if shutil.which("gpgconf"): + cls.addClassCleanup(subprocess.run, ["gpgconf", "--homedir", str(home), "--kill", "gpg-agent"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + cls.public_key = subprocess.run(["gpg", "--homedir", str(cls.homes[0]), "--batch", "--export"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout + + def sign(self, manifest, index=0): + return subprocess.run(["gpg", "--homedir", str(self.homes[index]), "--batch", "--pinentry-mode", "loopback", "--passphrase", "", "--digest-algo", "SHA512", "--detach-sign"], input=manifest, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout + + def test_real_signatures_authenticate_exact_manifest_and_reject_unknown_keys(self): + self.keyring.write_bytes(self.public_key) + manifest = b'{"version":"1.1.0","sha256":"trusted-code-checksum"}\n' + signature = self.sign(manifest) + module.verify_release_signature(manifest, signature) + for changed_manifest, changed_signature in ((manifest + b" ", signature), (manifest.replace(b"trusted-code-checksum", b"attacker-code-checksum"), signature), (manifest, signature[:-1]), (manifest, self.sign(manifest, 1))): + with self.subTest(manifest=changed_manifest, signature=changed_signature): + with self.assertRaises(module.IntegrityError): + module.verify_release_signature(changed_manifest, changed_signature) + self.assertEqual(list(self.update_dir.iterdir()), []) + if __name__ == "__main__": unittest.main() diff --git a/docs/development/README.md b/docs/development/README.md index 4325744..72f77bb 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -2,7 +2,8 @@ The controller uses SvelteKit, TypeScript and SQLite (`better-sqlite3`), with the Node adapter and server-sent events. The host agent uses Python 3's standard -library and runs as a systemd service; hosts do not need Node.js. +library and runs as a systemd service; release verification calls `gpgv`. +Hosts do not need Node.js. Agents connect outbound to the controller over HTTPS and manage Docker/Plutainer and local RCON on each host. The controller owns fleet metadata and queues; it @@ -10,7 +11,9 @@ does not require a Docker socket or direct access to host filesystems. ## Run locally -Use Node.js 22.13+ (the container uses Node.js 22), npm and Python 3. +Use Node.js 22.13+ (the container uses Node.js 22), npm and Python 3. Install +GnuPG's `gpg`, `gpg-agent` and `gpgv` to run real release-signing integration tests; local test +runs skip those tests when either executable is absent. ```sh npm ci @@ -89,8 +92,10 @@ npm run build The [Gitea Actions workflow](../../.gitea/workflows/ci.yml) runs these checks on every push and pull request after `npm ci`. It uses a Node.js 22 Debian container -with Python 3 and native build tools for `better-sqlite3`. The runner must support -Docker jobs with the `ubuntu-latest` label. `npm run check` runs before the tests +with Python 3, `gpg`, `gpg-agent`, `gpgv` and native build tools for `better-sqlite3`. Signing +tests create disposable keys in temporary directories; CI never receives release +private keys. The runner must support Docker jobs with the `ubuntu-latest` label. +`npm run check` runs before the tests to generate the SvelteKit configuration needed in a fresh checkout. Application tests use two workers in CI to bound resource use for database-heavy fixtures. The build step runs the same `npm run build` command used by the Dockerfile, @@ -118,7 +123,23 @@ Dependency security checks must include development dependencies: use `npm audit without `--omit=dev`. `npm run dev` and `npm run build` regenerate the downloadable agent, installer, -standalone updater and checksum manifest. Builds also bundle report workers +standalone updater and checksum manifest. Packaging preserves an existing +`static/agent/manifest.json.sig` only when its manifest bytes remain identical; +changes to the agent, updater or installer remove the stale signature. Builds +never sign, and hosts reject unsigned releases while retaining installed code. +Finalize and review release code, run `npm run agent:package`, and explicitly +sign it on the offline publisher machine: + +```sh +python3 agent/sign_release.py --gnupghome /secure/electric-cherry-signing --fingerprint FULL_SIGNING_KEY_FINGERPRINT +``` + +Commit or distribute the signed public artifacts before building the controller +image from the same source revision. Keep the signing home outside the repository, Docker context, controller +and CI. Follow [Publishing signed releases](../guides/native-hosts.md#publishing-signed-releases) +for key creation, host provisioning and rotation. + +Builds also bundle report workers under `build/reports` and launch preparation under `build/launch`. Development and test commands compile those workers automatically. Restart `npm run dev` after changing agent or worker code, or run `npm run agent:package` to refresh diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index a967547..89a7a2f 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -9,6 +9,8 @@ HTTPS. Running game containers continue through controller or agent outages. - A controller machine with Docker Compose and an HTTPS reverse proxy. - Debian 12 or 13 amd64 game hosts with storage for game files and instances. +- A signed agent release, plus its publisher's public key and reviewed installer + obtained through a trusted channel independent of the controller. - Your complete T5 or T6 game folders, including the DLC you intend to host. - Plutonium server keys for each game, with one unused key per new concurrent lobby. @@ -37,6 +39,13 @@ For an existing installation, retain its original data directory. Compose defaults to `/root/zombies/.electric-cherry`; choose a dedicated directory for a new controller. See [.env.example](../../.env.example) for the other settings. +Before building for game hosts, obtain the publisher's signed `static/agent` +artifacts and matching source revision, or follow +[Publishing signed releases](native-hosts.md#publishing-signed-releases) +for your own build. Normal builds preserve signatures for identical artifacts; +source changes require a new offline signature. An unsigned controller can run, +but hosts reject its agent installation and updates. + ```sh docker compose up -d --build ``` @@ -60,15 +69,24 @@ password recovery. 1. Open **Releases → Game files** and upload the complete local folder for T5 or T6, including DLC. Uploads can run before any host is enrolled. The latest verified upload becomes that game's default files. -2. Open **Fleet → Hosts → Add Host**, configure the host's game-port range and - bind address, and run the generated enrollment command on the host. -3. Wait for automatic setup to prepare the Plutainer runtime and available game +2. [Provision release trust](native-hosts.md#provision-release-trust): independently + verify the publisher key's full fingerprint, install its binary public key at + `/etc/electric-cherry/release-signing.gpg`, and copy the reviewed installer to + `./electric-cherry-install.sh` on the host. Both come from the trusted publisher. +3. Open **Fleet → Hosts → Add Host**, configure the host's game-port range and + bind address, and run the generated enrollment command from the directory + containing that local installer. Do not pipe controller-downloaded code into + a root shell. +4. Wait for automatic setup to prepare the Plutainer runtime and available game files. The host page reports download, verification, and readiness progress. The installer sets up the Python agent as a systemd service and enables automatic agent updates. It installs Docker if needed. Multiple instances share read-only base game files; each instance has its own writable data. See [Native hosts](native-hosts.md) for storage layout, network access, and recovery. +Existing hosts using updater protocol 1 need a +[trusted installer migration](native-hosts.md#automatic-agent-updates) before +controller-independent release verification takes effect. ## Launch your first server diff --git a/docs/guides/native-hosts.md b/docs/guides/native-hosts.md index 708b9d1..196ffb3 100644 --- a/docs/guides/native-hosts.md +++ b/docs/guides/native-hosts.md @@ -12,9 +12,11 @@ the agent owns local execution and its durable operation journal. and roles; agent endpoints authenticate their own host credentials. See [Users and access](authentication.md) for setup and password recovery. 2. Open **Hosts → Add Host**, choose a name, local game-port range and bind address, - and generate an enrollment. Run the displayed command on Debian 12/13 amd64. -3. The installer validates the platform, installs Python 3 and an absent Docker - installation, verifies the downloaded agent and updater SHA-256 over HTTPS, + and generate an enrollment. First [provision release trust](#provision-release-trust) + and copy the reviewed installer to `./electric-cherry-install.sh` on the Debian + 12/13 amd64 host. Then run the displayed command from that directory. +3. The installer validates the platform, installs Python 3, `gpgv` and an absent Docker + installation, verifies the release signature and agent/updater SHA-256 checksums, enrolls it and enables `electric-cherry-agent.service` and its automatic update timer. It needs no Git, Node.js or local build. 4. In **Game Files**, upload your complete local folder under T5 or T6. Include @@ -41,6 +43,37 @@ the agent owns local execution and its durable operation journal. A fresh destination host downloads centrally stored game files without manual copying or per-container setup. EC does not supply the game content itself. +### Provision release trust + +Obtain the release publisher's public key and reviewed installer through a +trusted channel independent of the controller. Verify the full public-key +fingerprint with the publisher through that independent channel. On a trusted +administrator workstation with GnuPG, inspect the supplied binary public key: + +```sh +gpg --show-keys --with-fingerprint ./release-signing.gpg +``` + +Transfer that verified public key and installer to the host, with the installer +named `electric-cherry-install.sh`. Provision the key before running enrollment: + +```sh +sudo install -d -o root -g root -m 0700 /etc/electric-cherry +sudo install -o root -g root -m 0644 ./release-signing.gpg /etc/electric-cherry/release-signing.gpg +``` + +The keyring and every parent directory must be owned by root, must not be +symlinks, and must not be writable by other users. Install only the publisher +keys you intend to trust. The generated enrollment command runs the local +installer with `sudo bash ./electric-cherry-install.sh`; never replace this with +a controller download piped into a root shell. An installer supplied by a +compromised controller could bypass its own verification. The host never obtains +or replaces its trust key through controller APIs. + +If you build your own controller releases, act as the publisher using the +[signing workflow below](#publishing-signed-releases). Hosts reject unsigned +releases, including ordinary development builds that have not been signed. + ### Docker installation The installer uses Debian's Docker packages: `docker.io` on Debian 12, plus @@ -169,15 +202,25 @@ directory, while file writes reject symlinks and paths outside that directory. New installations enable automatic agent updates by default. Each host checks its controller's release over HTTPS about every five minutes, beginning about two minutes after boot, with up to 30 seconds of jitter. Updating the controller -therefore makes its compatible, newer agent release available to enrolled hosts. +therefore makes its compatible, newer signed agent release available to enrolled hosts. There is no controller command to schedule and no update button to press. -**Existing hosts need the current installer once.** From the host's **Agent -access**, generate an installer command and rerun it on the same machine, with -the same controller and data directory. Do not use `--reenroll` for this upgrade. -The installer preserves host identity, credentials, game files and running -containers. Agent version 1.2.0 includes automatic update support; the host page -shows **Setup required** until a fresh heartbeat confirms the updater is installed. +**Existing hosts must migrate to updater protocol 2 with the trusted installer.** +Protocol 1 bootstraps trust controller-provided checksums and remain exposed to +controller-forged updates until this migration is complete. Publishing a signed +release does not upgrade those bootstraps: they request an installer rerun when +they see protocol 2, but a compromised controller could still serve protocol 1. + +First provision release trust as described above. From the host's **Agent +access**, generate a command and run it using the reviewed local installer, with +the same controller and data directory. Preserve a custom root with `--data-dir`; +do not use `--reenroll` for this upgrade. To pause old automatic checks during +migration, run `sudo systemctl stop electric-cherry-update.timer`. This does not +interrupt a running update worker; let any active update or journal recovery +finish before installing. The installer preserves host identity, credentials, +game files and running containers, replaces the bootstrap, and enables its timer +on success. Hosts without an updater show **Setup required** until a heartbeat +confirms installation. The **Automatic agent updates** panel reports the running version, target version, last check, progress and any error. Offline hosts show their last @@ -185,27 +228,91 @@ reported status. Deferred updates report why they are waiting: for example an active agent operation, an unavailable controller or a newer updater bootstrap. Checks also defer while the agent service is stopped; an update does not start a service that was stopped outside the update workflow. -The independent updater holds the same execution lock, verifies the release +The independent updater holds the same execution lock, authenticates the exact +manifest bytes with the pinned keyring before parsing them, verifies the release checksum, version and supported protocol, replaces agent code, and restarts the agent service. It never restarts Docker or game containers. It confirms an authenticated heartbeat from the new agent process before accepting the update; failure restores the previous agent code. Its durable journal permits recovery after a host reboot or interrupted update. -The updater only installs newer compatible versions; rolling the controller back -does not downgrade agents. A rejected version/checksum pair stays blocked until +The updater only installs newer compatible versions; replaying an older signed +release or rolling the controller back does not downgrade agents. Signing cannot +force a controller to publish newer releases or prevent it withholding updates. +A rejected version/checksum pair stays blocked until the controller publishes a different pair that is still newer than the installed agent. Inspect the reported error when a host shows **Needs attention**. The updater itself is a stable bootstrap component: it changes only when the installer is rerun. If a future release requires an unsupported updater protocol, rerun that release's installer on the host. -### Publishing releases and inspecting updates +### Publishing signed releases When publishing changed agent code, bump its numeric `VERSION` in -`agent/ec_agent.py` and rebuild the controller. Packaging derives the manifest -version and checksums from the sources; an already installed version is never -replaced just because its bytes changed. +`agent/ec_agent.py`. Finalize and review all source changes before packaging and +signing. An already installed version is never replaced just because its bytes +changed. + +Use a separate offline publisher machine with GnuPG. Keep the private key and its +GnuPG home outside the checkout, Docker build context, controller, and CI. For +example, create a dedicated signing key and inspect its full fingerprint: + +```sh +ec_signing_home=/secure/electric-cherry-signing +install -d -m 0700 "$ec_signing_home" +gpg --homedir "$ec_signing_home" --quick-generate-key 'Electric Cherry Release Signing' ed25519 sign 2y +gpg --homedir "$ec_signing_home" --list-keys --with-fingerprint +``` + +Protect the key with a passphrase and keep its backup and revocation certificate +outside release artifacts. Set the full fingerprint of the signing key, then +export only its public key for independent host provisioning: + +```sh +ec_signing_fingerprint=REPLACE_WITH_FULL_SIGNING_KEY_FINGERPRINT +gpg --homedir "$ec_signing_home" --export "$ec_signing_fingerprint" > release-signing.gpg +``` + +From the reviewed checkout on the publisher machine, package and explicitly sign: + +```sh +npm run agent:package +python3 agent/sign_release.py --gnupghome "$ec_signing_home" --fingerprint "$ec_signing_fingerprint" +``` + +The signer defaults to `static/agent`; `--release-dir` selects another packaged +release directory. It checks that the manifest matches the agent, updater and +installer bytes, then produces the ASCII-armored detached `manifest.json.sig`. +The manifest records their SHA-256 hashes as `sha256`, `updaterSha256`, and +`installerSha256`, along with version, platform and protocol metadata. The +installer hash lets recipients check a publisher-distributed installer against +the authenticated manifest before running it. + +Commit or distribute the finalized `static/agent` public artifacts together, +including the signature, before building the controller image from the same +reviewed source revision. Never include private keys or the signing home. +Normal `npm run build`, development startup, +and Docker builds never sign: packaging preserves a signature only when the +manifest remains byte-for-byte identical and removes it after any covered source +change. Repackage, review and sign again after such a change. Unsigned builds can +run the controller, but hosts fail closed and leave their installed code intact. + +### Signing-key rotation and removal + +Verify new fingerprints independently and replace each host's root-owned keyring +through trusted administration before publishing releases signed by a new key. +For a planned transition, provision both approved public keys temporarily, then +remove the retired key once every host can verify the new signer. The controller +cannot authorize key rotation. + +Remove compromised or retired keys from every host's keyring. Publishing a +revocation certificate or allowing a key to expire is insufficient: [GnuPG's +`gpgv` documentation](https://www.gnupg.org/documentation/manuals/gnupg/gpgv.html) +states that it does not check key expiration or revocation. Host deletion removes +the pinned keyring with `/etc/electric-cherry`; provision trust again before +reenrolling that machine. + +### Inspecting updates The installer places the updater at `/usr/local/libexec/electric-cherry-update.py` and enables @@ -370,8 +477,10 @@ The agent uses HTTPS, operating-system certificate validation, outbound heartbea and polling; Docker has no exposed remote API. Agent endpoints reject HTTP unless `EC_AGENT_ALLOW_HTTP=true` is explicitly enabled for development. The agent's insecure development option is limited to loopback. Public release assets contain -no credentials. The release checksum is authenticated by the controller's HTTPS -origin; there is no separate offline release-signing infrastructure. +no credentials. HTTPS protects transport; an offline publisher signature over the +release manifest authenticates agent and updater code using the independently +provisioned host keyring. Controller credentials do not grant release-signing +authority. Browser sessions do not authorize agents. Similarly, host credentials never grant access to operator pages. Game telemetry uses a separate `EC_TELEMETRY_TOKEN`, @@ -397,9 +506,11 @@ explicit installer `--reenroll` when local credentials already exist. or `409` when the agent needs updating or unresolved ownership remains. - `POST /api/agent/enroll`: consume enrollment token and return host credentials. - `GET /api/agent/release/manifest.json`: public agent version, protocol and - checksum plus `updaterProtocolVersion` and `updaterSha256` for installer bootstrap. + checksum plus `updaterProtocolVersion`, `updaterSha256`, and `installerSha256`. +- `GET /api/agent/release/manifest.json.sig`: detached publisher signature over + the exact manifest bytes; absent for an unsigned build. - `GET /api/agent/release/ec-agent.py` and `ec-agent-update.py`: public release - source, verified against that HTTPS manifest before installation. + source, verified against the signed manifest before installation. - `POST /api/agent/heartbeat`: authenticated inventory, current operation and caches. - `GET /api/agent/operations`: deliver/reconcile durable operation IDs. - `POST /api/agent/operations/:id`: authenticated acceptance, progress or result. @@ -576,10 +687,11 @@ and invalidates unused installer links. A separate cleanup service removes the host's downloaded game libraries, instance files, agent journal and configuration, owned containers, and recorded runtime images unused by other containers. It stops and removes the agent service and executable, automatic update timer, -update service, updater executable and update state, then removes its own files. +update service, updater executable, update state and pinned signing keyring, then +removes its own files. The controller hides the host and revokes its ordinary credentials only after receiving the final uninstall receipt. Adding the machine again requires a new -host enrollment. +host enrollment and independently reprovisioned release trust. Game folders uploaded through Electric Cherry belong to the controller. Those original uploads remain available for other hosts and future deployments; only @@ -834,14 +946,18 @@ npm run build ``` The build packages the agent, standalone updater and installer into the static -release, generates the agent/updater checksum manifest and serves those same -assets in the Node-adapter image. Development serves the freshly packaged static -release even when an older production build exists. The Python programs use only -the standard library. +release, generates their checksum manifest and serves those same assets in the +Node-adapter image. Sign production artifacts using the +[offline workflow](#publishing-signed-releases) before building. Development +serves the freshly packaged static release even when an older production build +exists. The Python programs use only the standard library; signature verification +also requires the host's `gpgv` executable. Real signing tests require `gpg` and +`gpgv`, installed by CI; local runs skip those tests when either tool is absent. Automated tests cover enrollment/authentication, rotation/revocation, host readiness, scoped ports, immutable library publication, streaming/ranges, -checksums, interrupted downloads, instance generation ownership, local occupancy +signed manifests, forged checksums, unknown signing keys, interrupted downloads, +instance generation ownership, local occupancy holds, controller restart/reconnect, ambiguous results, occupied campaign staging, activation before images, collections and draining. Host deletion tests cover ownership guards, interrupted cleanup, uninstall receipts, credential revocation @@ -852,13 +968,15 @@ A real Debian/Docker/Plutonium acceptance run is still required before productio rollout. Automated simulations do not establish successful deployment on real game hosts. Run: -- Installer on Debian 12 and 13 amd64, then rerun without changing identity/data. -- Bootstrap an existing host with the current installer, then publish a newer - compatible agent release. Verify a timer-driven update, its authenticated +- Independently provision release trust and run the reviewed installer on Debian + 12 and 13 amd64, then rerun without changing identity/data. +- Migrate a protocol 1 host with the trusted installer, then publish a newer + compatible signed agent release. Verify a timer-driven update, its authenticated heartbeat and unchanged game containers. Exercise an active-operation hold, - bad checksum, failed startup/heartbeat rollback, controller downgrade, network - loss and host reboot during update. Verify updater protocol mismatch requests - an installer rerun. + missing/invalid signatures, an unknown key, a forged manifest matching changed + code, bad checksum, failed startup/heartbeat rollback, controller downgrade, + network loss and host reboot during update. Verify updater protocol mismatch + requests an installer rerun and test trusted keyring replacement for rotation. - T5 and T6 deployments and a mixed collection from a fresh destination host. - Warm the Plutonium runtime cache, then create and rebuild T5 and T6 lobbies; verify the running revision and compare first-start downloads and launch times diff --git a/src/lib/server/auth/guard.ts b/src/lib/server/auth/guard.ts index 1f7698e..31c0e7c 100644 --- a/src/lib/server/auth/guard.ts +++ b/src/lib/server/auth/guard.ts @@ -20,7 +20,8 @@ function publicAsset(routeId: string | null, path: string, method: string) { '/agent/ec-agent.py', '/agent/ec-agent-update.py', '/agent/install.sh', - '/agent/manifest.json' + '/agent/manifest.json', + '/agent/manifest.json.sig' ].includes(path)) ); } diff --git a/src/lib/server/hosts/http.ts b/src/lib/server/hosts/http.ts index 6094cda..da224d2 100644 --- a/src/lib/server/hosts/http.ts +++ b/src/lib/server/hosts/http.ts @@ -70,5 +70,5 @@ export function hostApiError(cause: unknown): Response { export function installerCommand(controller: string, token: string): string { const origin = new URL(controller).origin; const quote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'"; - return `curl --fail --silent --show-error --proto '=https' ${quote(origin + '/api/agent/install.sh')} | sudo bash -s -- --controller ${quote(origin)} --token ${quote(token)}`; + return `sudo bash ./electric-cherry-install.sh --controller ${quote(origin)} --token ${quote(token)}`; } diff --git a/src/lib/server/hosts/releases.ts b/src/lib/server/hosts/releases.ts index 9235e30..4e1a365 100644 --- a/src/lib/server/hosts/releases.ts +++ b/src/lib/server/hosts/releases.ts @@ -3,7 +3,8 @@ import { resolve } from 'node:path'; /** adapter-node copies public assets to build/client; Vite serves static in development. */ export async function agentReleaseAsset( - file: 'install.sh' | 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json' + file: + 'install.sh' | 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json' | 'manifest.json.sig' ): Promise { // A prior production build must never shadow a freshly packaged development release. if (process.env.NODE_ENV === 'development') diff --git a/src/lib/ui/components/AddHostDrawer.svelte b/src/lib/ui/components/AddHostDrawer.svelte index 2323826..b1a5aff 100644 --- a/src/lib/ui/components/AddHostDrawer.svelte +++ b/src/lib/ui/components/AddHostDrawer.svelte @@ -52,6 +52,12 @@

{name} created. Run this command with administrator access on that machine.

+

+ First copy the reviewed installer from your release publisher to + electric-cherry-install.sh and provision the publisher's public signing + key at /etc/electric-cherry/release-signing.gpg. Obtain both through a + trusted channel independent of this controller. +