Cache Plutonium runtime files on hosts
This commit is contained in:
+262
-4
@@ -33,7 +33,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.7.0"
|
||||
VERSION = "1.8.0"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
MANIFEST_MAX_ENTRIES = 10000
|
||||
@@ -42,6 +42,10 @@ MANIFEST_MAX_SECONDS = 15
|
||||
MANIFEST_MAX_DEPTH = 64
|
||||
LIBRARY_DOWNLOAD_WORKERS = 4
|
||||
HEALTH_LOCK_TIMEOUT_SECONDS = 5
|
||||
PLUTONIUM_MANIFEST_URL = "https://cdn.plutoniummod.com/updater/prod/info.json"
|
||||
PLUTONIUM_MANIFEST_LIMIT = 2 * 1024 * 1024
|
||||
PLUTONIUM_FILE_LIMIT = 512 * 1024 * 1024
|
||||
PLUTONIUM_CACHE_LIMIT = 4 * 1024 * 1024 * 1024
|
||||
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
|
||||
AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
|
||||
AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
|
||||
@@ -487,6 +491,34 @@ def managed_write(root, value, content, uid=1000, gid=1000):
|
||||
os.unlink(temp_name, dir_fd=parent)
|
||||
|
||||
|
||||
def managed_copy(root, value, source, uid=1000, gid=1000):
|
||||
"""Copy a private file, using Linux reflinks when supported, never hardlinks."""
|
||||
with managed_open(Path(source).parent, Path(source).name) as incoming:
|
||||
with managed_parent(root, value, create=True, uid=uid, gid=gid) as (parent, name):
|
||||
temp_name = ".ec-" + uuid.uuid4().hex
|
||||
fd = os.open(temp_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=parent)
|
||||
try:
|
||||
os.fchmod(fd, 0o644)
|
||||
if os.geteuid() == 0:
|
||||
os.fchown(fd, uid, gid)
|
||||
try:
|
||||
if not sys.platform.startswith("linux"):
|
||||
raise OSError("Reflinks unavailable")
|
||||
fcntl.ioctl(fd, 0x40049409, incoming.fileno()) # FICLONE
|
||||
except OSError:
|
||||
os.ftruncate(fd, 0)
|
||||
with os.fdopen(fd, "wb", closefd=False) as outgoing:
|
||||
shutil.copyfileobj(incoming, outgoing, 1024 * 1024)
|
||||
outgoing.flush()
|
||||
os.fsync(fd)
|
||||
os.replace(temp_name, name, src_dir_fd=parent, dst_dir_fd=parent)
|
||||
os.fsync(parent)
|
||||
finally:
|
||||
os.close(fd)
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(temp_name, dir_fd=parent)
|
||||
|
||||
|
||||
class ReadWriteGate:
|
||||
"""Let observations run together; global mutations wait for existing readers."""
|
||||
def __init__(self):
|
||||
@@ -921,11 +953,13 @@ class Agent:
|
||||
self.read_claims = set()
|
||||
self.health_probe_gate = threading.BoundedSemaphore(2)
|
||||
self.cancelled_creations = frozenset()
|
||||
self.defer_plutonium_preparation = False
|
||||
self.observation_keep_days = 0 # Older controllers do not authorize cleanup.
|
||||
self.library_progress = {}
|
||||
self.library_transfers = {}
|
||||
self.inventory_manifest_cache = {}
|
||||
self.inventory_manifest_lock = threading.Lock()
|
||||
self.plutonium_lock = threading.RLock()
|
||||
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
|
||||
self.executing_id = None
|
||||
self.lane_lock = threading.Lock()
|
||||
@@ -1360,8 +1394,9 @@ class Agent:
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
update = self.update_inventory()
|
||||
capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
plutonium_cache = self.plutonium_cache_inventory()
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation):
|
||||
health = self.inspect(instance_id, generation)
|
||||
@@ -1510,6 +1545,223 @@ class Agent:
|
||||
raise Rejected("Initial files exceed the 64 MiB operation limit")
|
||||
return game, library, image, port, bind, env
|
||||
|
||||
@staticmethod
|
||||
def plutonium_manifest(data):
|
||||
"""Validate the official updater format before using any paths or URLs."""
|
||||
if len(data) > PLUTONIUM_MANIFEST_LIMIT:
|
||||
raise AgentError("Plutonium manifest is too large")
|
||||
try:
|
||||
manifest = json.loads(data)
|
||||
except (ValueError, UnicodeError):
|
||||
raise AgentError("Invalid Plutonium manifest") from None
|
||||
if not isinstance(manifest, dict) or manifest.get("product") != "plutonium-core-prod":
|
||||
raise AgentError("Invalid Plutonium production manifest")
|
||||
revision = manifest.get("revision")
|
||||
if type(revision) is not int or not 1 <= revision <= 2147483647:
|
||||
raise AgentError("Invalid Plutonium revision")
|
||||
base = manifest.get("baseUrl")
|
||||
if not isinstance(base, str) or not re.fullmatch(r"https://cdn\.plutoniummod\.com/updater/prod/(?:[A-Za-z0-9_-]+/)*", base):
|
||||
raise AgentError("Plutonium downloads must use the official production CDN")
|
||||
entries = manifest.get("files")
|
||||
if not isinstance(entries, list) or not 1 <= len(entries) <= 4096:
|
||||
raise AgentError("Invalid Plutonium file list")
|
||||
names, total = set(), 0
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
raise AgentError("Invalid Plutonium file entry")
|
||||
name, size, checksum = entry.get("name"), entry.get("size"), entry.get("hash")
|
||||
if not isinstance(name, str) or len(name) > 512 or re.search(r'[\x00-\x1f\x7f\\:*?"<>|]', name):
|
||||
raise AgentError("Invalid Plutonium file path")
|
||||
path = PurePosixPath(name)
|
||||
if not path.parts or path.is_absolute() or str(path) != name or any(part in (".", "..") or part.endswith((".", " ")) for part in path.parts) or name.lower() == "cdn_info.json":
|
||||
raise AgentError("Invalid Plutonium file path")
|
||||
if name.lower() in names:
|
||||
raise AgentError("Duplicate Plutonium file path")
|
||||
names.add(name.lower())
|
||||
if type(size) is not int or not 0 <= size <= PLUTONIUM_FILE_LIMIT:
|
||||
raise AgentError("Invalid Plutonium file size")
|
||||
if not isinstance(checksum, str) or not re.fullmatch(r"[a-f0-9]{40}", checksum):
|
||||
raise AgentError("Invalid Plutonium file hash")
|
||||
total += size
|
||||
if total > PLUTONIUM_CACHE_LIMIT:
|
||||
raise AgentError("Plutonium cache exceeds the size limit")
|
||||
for name in names:
|
||||
if any(str(parent) in names for parent in PurePosixPath(name).parents if str(parent) != "."):
|
||||
raise AgentError("Conflicting Plutonium file paths")
|
||||
if "bin/plutonium-bootstrapper-win32.exe" not in {entry["name"] for entry in entries}:
|
||||
raise AgentError("Plutonium manifest is missing the server bootstrapper")
|
||||
# Match plutonium-updater's default: launcher assets are optional.
|
||||
return manifest, [entry for entry in entries if not entry["name"].startswith("launcher")]
|
||||
|
||||
@staticmethod
|
||||
def plutonium_file_matches(root, entry):
|
||||
try:
|
||||
with managed_open(root, entry["name"]) as stream:
|
||||
if os.fstat(stream.fileno()).st_size != entry["size"]:
|
||||
return False
|
||||
checksum = hashlib.sha1()
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
checksum.update(chunk)
|
||||
return checksum.hexdigest() == entry["hash"]
|
||||
except (OSError, AgentError):
|
||||
return False
|
||||
|
||||
def plutonium_open(self, url):
|
||||
if not isinstance(url, str) or not re.fullmatch(r"https://cdn\.plutoniummod\.com/updater/prod/[A-Za-z0-9_./-]+", url):
|
||||
raise AgentError("Plutonium downloads must use the official production CDN")
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
|
||||
raise AgentError("Plutonium download redirects are not allowed")
|
||||
# Never send the controller's host credentials to the public CDN.
|
||||
return urllib.request.build_opener(NoRedirect()).open(url, timeout=30)
|
||||
|
||||
def plutonium_snapshot(self):
|
||||
root = scoped(self.root / "agent", "plutonium-cache")
|
||||
pointer = json.loads(managed_read(root, "current.json") or b"null")
|
||||
if not isinstance(pointer, dict) or not isinstance(pointer.get("key"), str) or not re.fullmatch(r"[a-f0-9]{64}", pointer["key"]):
|
||||
raise AgentError("Plutonium cache is not prepared")
|
||||
content = scoped(root, pointer["key"] + "/content")
|
||||
raw = managed_read(content, "cdn_info.json")
|
||||
if raw is None or digest(raw) != pointer["key"]:
|
||||
raise AgentError("Plutonium cache manifest is invalid")
|
||||
manifest, files = self.plutonium_manifest(raw)
|
||||
if pointer.get("revision") != manifest["revision"] or pointer.get("bytes") != sum(entry["size"] for entry in files):
|
||||
raise AgentError("Plutonium cache metadata is invalid")
|
||||
# Heartbeat checks are cheap. Every file is hashed before it is seeded.
|
||||
for entry in files:
|
||||
info = scoped(content, entry["name"]).lstat()
|
||||
if not stat_module.S_ISREG(info.st_mode) or info.st_size != entry["size"]:
|
||||
raise AgentError("Plutonium cache files are incomplete")
|
||||
return pointer, content, raw, files
|
||||
|
||||
def plutonium_cache_inventory(self):
|
||||
with self.plutonium_lock:
|
||||
try:
|
||||
pointer, _, _, _ = self.plutonium_snapshot()
|
||||
return {"revision": pointer["revision"], "bytes": pointer["bytes"]}
|
||||
except (OSError, ValueError, AgentError):
|
||||
return None
|
||||
|
||||
def prepare_plutonium(self, payload):
|
||||
target, reserve = payload.get("revision", 0), payload.get("minimumFreeBytes", 0)
|
||||
if type(target) is not int or not 0 <= target <= 2147483647 or type(reserve) is not int or reserve < 0:
|
||||
raise Rejected("Invalid Plutonium preparation request")
|
||||
root = scoped(self.root / "agent", "plutonium-cache")
|
||||
root.mkdir(mode=0o700, exist_ok=True)
|
||||
os.chmod(root, 0o700)
|
||||
deadline = time.monotonic() + 1800
|
||||
def active():
|
||||
if self.stopping.is_set() or time.monotonic() >= deadline:
|
||||
raise AgentError("Plutonium cache preparation was interrupted or timed out")
|
||||
if self.defer_plutonium_preparation:
|
||||
raise Held("Plutonium cache preparation is waiting for other host operations")
|
||||
active()
|
||||
try:
|
||||
with self.plutonium_open(PLUTONIUM_MANIFEST_URL) as response:
|
||||
raw = response.read(PLUTONIUM_MANIFEST_LIMIT + 1)
|
||||
manifest, files = self.plutonium_manifest(raw)
|
||||
if manifest["revision"] < target:
|
||||
raise AgentError("Plutonium CDN has not reached the requested revision")
|
||||
key = digest(raw)
|
||||
previous = None
|
||||
with self.plutonium_lock:
|
||||
with contextlib.suppress(OSError, ValueError, AgentError):
|
||||
_, previous, _, _ = self.plutonium_snapshot()
|
||||
if previous is not None and previous.parent.name == key and all(self.plutonium_file_matches(previous, entry) for entry in files):
|
||||
return {"revision": manifest["revision"], "bytes": sum(entry["size"] for entry in files)}
|
||||
staging = scoped(root, key + ".partial/content")
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
expected = {entry["name"] for entry in files}
|
||||
# SIGKILL/power loss bypasses download/copy finally blocks. Remove
|
||||
# abandoned temporary files before counting space or publishing.
|
||||
for directory, subdirs, filenames in os.walk(staging, followlinks=False):
|
||||
for name in subdirs + filenames:
|
||||
path = Path(directory) / name
|
||||
if path.is_symlink() or (name in filenames and path.relative_to(staging).as_posix() not in expected):
|
||||
path.unlink()
|
||||
# Keep at most the current snapshot and this resumable preparation.
|
||||
current_key = previous.parent.name if previous is not None else None
|
||||
for directory in root.iterdir():
|
||||
if re.fullmatch(r"[a-f0-9]{64}(?:\.partial)?", directory.name) and directory.name not in (current_key, key + ".partial"):
|
||||
if directory.is_symlink():
|
||||
directory.unlink()
|
||||
else:
|
||||
shutil.rmtree(directory)
|
||||
missing = sum(entry["size"] for entry in files if not self.plutonium_file_matches(staging, entry))
|
||||
if shutil.disk_usage(root).free < reserve + missing + len(raw):
|
||||
raise AgentError("Insufficient free storage for the Plutonium cache and host reserve")
|
||||
for entry in files:
|
||||
active()
|
||||
if self.plutonium_file_matches(staging, entry):
|
||||
continue
|
||||
if previous is not None and self.plutonium_file_matches(previous, entry):
|
||||
managed_copy(staging, entry["name"], scoped(previous, entry["name"]), os.geteuid(), os.getegid())
|
||||
continue
|
||||
destination = scoped(staging, entry["name"])
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = destination.with_name(".download-" + uuid.uuid4().hex)
|
||||
try:
|
||||
with self.plutonium_open(manifest["baseUrl"] + entry["hash"]) as incoming, temporary.open("xb") as outgoing:
|
||||
size, checksum = 0, hashlib.sha1()
|
||||
while True:
|
||||
active()
|
||||
chunk = incoming.read(min(1024 * 1024, entry["size"] - size + 1))
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
if size > entry["size"]:
|
||||
raise AgentError("Plutonium download exceeds its expected size")
|
||||
checksum.update(chunk)
|
||||
outgoing.write(chunk)
|
||||
if size != entry["size"] or checksum.hexdigest() != entry["hash"]:
|
||||
raise AgentError("Plutonium download failed size or checksum verification")
|
||||
outgoing.flush()
|
||||
os.fsync(outgoing.fileno())
|
||||
os.replace(temporary, destination)
|
||||
sync_parent(destination)
|
||||
finally:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
temporary.unlink()
|
||||
# Publish metadata last. Interrupted transfers can never claim ready.
|
||||
active()
|
||||
atomic_bytes(staging / "cdn_info.json", raw)
|
||||
pointer = {"key": key, "revision": manifest["revision"], "bytes": sum(entry["size"] for entry in files)}
|
||||
with self.plutonium_lock:
|
||||
final = scoped(root, key)
|
||||
if final.exists():
|
||||
# Invalidate before replacing a damaged/same-revision cache.
|
||||
if current_key == key:
|
||||
(root / "current.json").unlink(missing_ok=True)
|
||||
shutil.rmtree(final)
|
||||
os.replace(staging.parent, final)
|
||||
sync_parent(final)
|
||||
atomic_json(root / "current.json", pointer)
|
||||
if previous is not None and current_key != key:
|
||||
shutil.rmtree(previous.parent)
|
||||
return {"revision": pointer["revision"], "bytes": pointer["bytes"]}
|
||||
except (urllib.error.URLError, TimeoutError, http.client.HTTPException):
|
||||
raise AgentError("Plutonium CDN download is unavailable; the previous cache is retained") from None
|
||||
|
||||
def seed_plutonium(self, app, image):
|
||||
if not re.fullmatch(r"ghcr\.io/ayymoss/plutainer(?::[A-Za-z0-9_.-]+)?(?:@sha256:[a-f0-9]{64})?", image):
|
||||
return False
|
||||
with self.plutonium_lock:
|
||||
try:
|
||||
_, content, raw, files = self.plutonium_snapshot()
|
||||
if not all(self.plutonium_file_matches(content, entry) for entry in files):
|
||||
scoped(self.root / "agent", "plutonium-cache/current.json").unlink(missing_ok=True)
|
||||
raise AgentError("Plutonium cache checksum verification failed")
|
||||
except (OSError, ValueError, AgentError):
|
||||
return False
|
||||
uid, gid = int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))
|
||||
marker = "runtime/plutonium/cdn_info.json"
|
||||
managed_write(app, marker, None, uid, gid)
|
||||
for entry in files:
|
||||
self.check_creation_cancellation()
|
||||
managed_copy(app, "runtime/plutonium/" + entry["name"], scoped(content, entry["name"]), uid, gid)
|
||||
managed_write(app, marker, raw, uid, gid)
|
||||
return True
|
||||
|
||||
def create(self, instance_id, generation, payload):
|
||||
self.check_creation_cancellation()
|
||||
directory = self.instance_dir(instance_id, generation)
|
||||
@@ -1527,6 +1779,7 @@ class Agent:
|
||||
app = directory / "app"
|
||||
app.mkdir(parents=True, exist_ok=True)
|
||||
self.make_writable(app, directory=True)
|
||||
self.seed_plutonium(app, image)
|
||||
self.write_initial_files(app, payload.get("files", []))
|
||||
atomic_json(directory / "spec.json", payload)
|
||||
env_path = directory / "environment"
|
||||
@@ -2184,6 +2437,8 @@ class Agent:
|
||||
return {"image": image, "ready": True}
|
||||
if kind == "host.inspect":
|
||||
return self.inventory()
|
||||
if kind == "plutonium.prepare":
|
||||
return self.prepare_plutonium(payload)
|
||||
if kind == "rcon.status-batch":
|
||||
return self.rcon_status_batch(operation)
|
||||
if kind == "library.download":
|
||||
@@ -2290,7 +2545,7 @@ class Agent:
|
||||
if not self.config.get("pendingCredential") or self.recover_credential():
|
||||
return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True})
|
||||
return row
|
||||
if kind == "runtime.pull":
|
||||
if kind in ("runtime.pull", "plutonium.prepare"):
|
||||
# Docker resumes already downloaded layers. Pull again under
|
||||
# the same operation ID: an existing tag can still refer to an
|
||||
# older image and cannot prove an interrupted pull completed.
|
||||
@@ -2298,6 +2553,8 @@ class Agent:
|
||||
try:
|
||||
result = self.dispatch(operation)
|
||||
return self.journal.update(operation["id"], "completed", {**result, "reconciled": True})
|
||||
except Held as err:
|
||||
return self.journal.update(operation["id"], "accepted", {"blocked": "host-busy", "message": str(err)})
|
||||
except Exception:
|
||||
return self.journal.update(operation["id"], "failed", error="Interrupted runtime download could not be resumed")
|
||||
if kind == "library.import":
|
||||
@@ -2407,6 +2664,7 @@ class Agent:
|
||||
# Only the heartbeat thread replaces this immutable snapshot; it never
|
||||
# mutates the operation journal or races Docker cleanup against create.
|
||||
self.cancelled_creations = frozenset(value for value in response.get("cancelledCreations", []) if isinstance(value, str))
|
||||
self.defer_plutonium_preparation = response.get("deferPlutoniumPreparation") is True
|
||||
keep_days = response.get("observationKeepDays")
|
||||
self.observation_keep_days = keep_days if valid_observation_keep_days(keep_days) else 0
|
||||
host = response.get("host", {})
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
import base64
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import urllib.error
|
||||
|
||||
import test_agent as fixtures
|
||||
|
||||
|
||||
module = fixtures.module
|
||||
AgentError = fixtures.AgentError
|
||||
IMAGE = "ghcr.io/ayymoss/plutainer:latest"
|
||||
BOOTSTRAPPER = "bin/plutonium-bootstrapper-win32.exe"
|
||||
BASE_URL = "https://cdn.plutoniummod.com/updater/prod/files/"
|
||||
|
||||
|
||||
class PlutoniumCacheTests(unittest.TestCase):
|
||||
tearDown = fixtures.AgentTests.tearDown
|
||||
operation = fixtures.AgentTests.operation
|
||||
create = fixtures.AgentTests.create
|
||||
|
||||
def setUp(self):
|
||||
fixtures.AgentTests.setUp(self)
|
||||
network = patch.object(self.agent, "plutonium_open", side_effect=AssertionError("Unexpected vendor request"), create=True)
|
||||
network.start()
|
||||
self.addCleanup(network.stop)
|
||||
|
||||
def release(self, revision=5354, contents=None):
|
||||
contents = contents or {
|
||||
BOOTSTRAPPER: b"bootstrapper",
|
||||
"bin/plutonium.dll": b"runtime",
|
||||
"storage/t5/zone/plutonium.ff": b"zone-data",
|
||||
"storage/iw5/images/~weapon_ak47_iw5_spc-rgb&weap~fa48e615.iwi": b"texture-data",
|
||||
"launcher/plutonium.exe": b"launcher-not-needed",
|
||||
}
|
||||
manifest = {
|
||||
"product": "plutonium-core-prod",
|
||||
"revision": revision,
|
||||
"baseUrl": BASE_URL,
|
||||
"files": [
|
||||
{"name": name, "size": len(content), "hash": hashlib.sha1(content).hexdigest()}
|
||||
for name, content in contents.items()
|
||||
],
|
||||
}
|
||||
return manifest, contents
|
||||
|
||||
def opener(self, manifest, contents, calls=None):
|
||||
calls = calls if calls is not None else []
|
||||
|
||||
def open_url(url):
|
||||
calls.append(url)
|
||||
if url == module.PLUTONIUM_MANIFEST_URL:
|
||||
return fixtures.Response(json.dumps(manifest).encode())
|
||||
self.assertTrue(url.startswith(BASE_URL), url)
|
||||
name = next(entry["name"] for entry in manifest["files"] if url == BASE_URL + entry["hash"])
|
||||
return fixtures.Response(contents[name])
|
||||
|
||||
return open_url
|
||||
|
||||
def file_url(self, content):
|
||||
return BASE_URL + hashlib.sha1(content).hexdigest()
|
||||
|
||||
def prepare(self, revision=5354, contents=None):
|
||||
manifest, contents = self.release(revision, contents)
|
||||
calls = []
|
||||
with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
|
||||
result = self.agent.prepare_plutonium({"revision": revision})
|
||||
return manifest, contents, calls, result
|
||||
|
||||
def cache_content(self):
|
||||
root = self.agent.root / "agent" / "plutonium-cache"
|
||||
pointer = json.loads((root / "current.json").read_text())
|
||||
return root / pointer["key"] / "content"
|
||||
|
||||
def app(self, name="app"):
|
||||
app = Path(self.temp.name) / name
|
||||
app.mkdir(exist_ok=True)
|
||||
return app
|
||||
|
||||
def assert_runtime(self, app, manifest, contents):
|
||||
runtime = app / "runtime" / "plutonium"
|
||||
self.assertEqual(json.loads((runtime / "cdn_info.json").read_text()), manifest)
|
||||
for name, content in contents.items():
|
||||
if name.startswith("launcher"):
|
||||
self.assertFalse((runtime / name).exists())
|
||||
else:
|
||||
self.assertEqual((runtime / name).read_bytes(), content)
|
||||
|
||||
def test_cold_preparation_verifies_downloads_and_reports_inventory(self):
|
||||
self.assertIsNone(self.agent.plutonium_cache_inventory())
|
||||
manifest, contents, calls, result = self.prepare()
|
||||
expected_bytes = sum(len(content) for name, content in contents.items() if not name.startswith("launcher"))
|
||||
self.assertEqual(result, {"revision": 5354, "bytes": expected_bytes})
|
||||
self.assertEqual(self.agent.plutonium_cache_inventory(), result)
|
||||
self.assertEqual(calls[0], module.PLUTONIUM_MANIFEST_URL)
|
||||
self.assertEqual(set(calls[1:]), {self.file_url(content) for name, content in contents.items() if not name.startswith("launcher")})
|
||||
self.assertEqual(json.loads((self.cache_content() / "cdn_info.json").read_text()), manifest)
|
||||
self.assertEqual((self.agent.root / "agent" / "plutonium-cache").stat().st_mode & 0o077, 0)
|
||||
|
||||
def test_warm_preparation_only_fetches_manifest_and_dispatches(self):
|
||||
manifest, contents, _, first = self.prepare()
|
||||
calls = []
|
||||
with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
|
||||
result = self.agent.dispatch({"kind": "plutonium.prepare", "payload": {"revision": 5354}})
|
||||
self.assertEqual(result, first)
|
||||
self.assertEqual(calls, [module.PLUTONIUM_MANIFEST_URL])
|
||||
|
||||
def test_refresh_downloads_changed_files_and_reuses_verified_unchanged_files(self):
|
||||
_, contents, _, _ = self.prepare()
|
||||
changed = {**contents, "bin/plutonium.dll": b"new-runtime", "storage/t6/new.ff": b"new-zone"}
|
||||
manifest, changed, calls, result = self.prepare(5355, changed)
|
||||
self.assertEqual(result["revision"], 5355)
|
||||
self.assertEqual(set(calls[1:]), {self.file_url(changed["bin/plutonium.dll"]), self.file_url(changed["storage/t6/new.ff"])})
|
||||
app = self.app()
|
||||
self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
|
||||
self.assert_runtime(app, manifest, changed)
|
||||
|
||||
def test_changed_manifest_is_refreshed_even_when_revision_number_is_unchanged(self):
|
||||
_, contents, _, _ = self.prepare()
|
||||
previous = self.cache_content()
|
||||
manifest, changed, calls, result = self.prepare(5354, {**contents, "bin/plutonium.dll": b"corrected-runtime"})
|
||||
self.assertEqual(result["revision"], 5354)
|
||||
self.assertNotEqual(self.cache_content(), previous)
|
||||
self.assertEqual(calls[1:], [self.file_url(changed["bin/plutonium.dll"])])
|
||||
app = self.app()
|
||||
self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
|
||||
self.assert_runtime(app, manifest, changed)
|
||||
|
||||
def test_failed_refresh_preserves_the_older_complete_cache(self):
|
||||
manifest, contents, _, result = self.prepare()
|
||||
previous = self.cache_content()
|
||||
next_manifest, next_contents = self.release(5355, {**contents, "bin/plutonium.dll": b"new-runtime"})
|
||||
next_contents["bin/plutonium.dll"] = b"bad-runtime"
|
||||
with patch.object(self.agent, "plutonium_open", side_effect=self.opener(next_manifest, next_contents)):
|
||||
with self.assertRaises(AgentError):
|
||||
self.agent.prepare_plutonium({"revision": 5355})
|
||||
self.assertEqual(self.cache_content(), previous)
|
||||
self.assertEqual(self.agent.plutonium_cache_inventory(), result)
|
||||
app = self.app()
|
||||
self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
|
||||
self.assert_runtime(app, manifest, contents)
|
||||
|
||||
def test_interrupted_preparation_resumes_without_advertising_a_partial_cache(self):
|
||||
manifest, contents = self.release()
|
||||
open_url = self.opener(manifest, contents)
|
||||
|
||||
def interrupted_open(url):
|
||||
if url == self.file_url(contents["bin/plutonium.dll"]):
|
||||
raise urllib.error.URLError("simulated network interruption")
|
||||
return open_url(url)
|
||||
|
||||
with patch.object(self.agent, "plutonium_open", side_effect=interrupted_open):
|
||||
with self.assertRaises(AgentError):
|
||||
self.agent.prepare_plutonium({})
|
||||
self.assertIsNone(self.agent.plutonium_cache_inventory())
|
||||
self.assertFalse((self.agent.root / "agent/plutonium-cache/current.json").exists())
|
||||
_, _, calls, result = self.prepare()
|
||||
self.assertNotIn(self.file_url(contents[BOOTSTRAPPER]), calls)
|
||||
self.assertEqual(result["revision"], manifest["revision"])
|
||||
|
||||
def test_agent_restart_reconciles_preparation_instead_of_accepting_an_older_cache(self):
|
||||
_, contents, _, _ = self.prepare()
|
||||
operation = {"id": "prepare-new-release", "kind": "plutonium.prepare", "payload": {"revision": 5355}}
|
||||
self.agent.journal.accept(operation)
|
||||
self.agent.journal.update(operation["id"], "running")
|
||||
self.agent.journal.close()
|
||||
self.agent = fixtures.Agent(self.config, self.docker)
|
||||
self.assertEqual(self.agent.journal.get(operation["id"])["state"], "unknown")
|
||||
manifest, contents = self.release(5355, {**contents, "bin/plutonium.dll": b"new-runtime"})
|
||||
calls = []
|
||||
with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
self.assertEqual(json.loads(result["result"])["revision"], 5355)
|
||||
self.assertTrue(json.loads(result["result"])["reconciled"])
|
||||
self.assertEqual(self.agent.plutonium_cache_inventory()["revision"], 5355)
|
||||
self.assertEqual(calls, [module.PLUTONIUM_MANIFEST_URL, self.file_url(contents["bin/plutonium.dll"])])
|
||||
with patch.object(self.agent, "plutonium_open", side_effect=AssertionError("Completed operation repeated")):
|
||||
self.assertEqual(self.agent.execute(operation)["result"], result["result"])
|
||||
|
||||
def test_heartbeat_defers_preparation_only_for_explicit_true(self):
|
||||
self.assertFalse(self.agent.defer_plutonium_preparation)
|
||||
for response, expected in (
|
||||
({"deferPlutoniumPreparation": True}, True),
|
||||
({}, False),
|
||||
({"deferPlutoniumPreparation": False}, False),
|
||||
({"deferPlutoniumPreparation": 1}, False),
|
||||
({"deferPlutoniumPreparation": "true"}, False),
|
||||
({"deferPlutoniumPreparation": None}, False),
|
||||
):
|
||||
self.agent.defer_plutonium_preparation = not expected
|
||||
with self.subTest(response=response), patch.object(self.agent, "request", return_value=response), patch.object(self.agent, "inventory", return_value={}), patch.object(self.agent, "record_health"):
|
||||
self.agent.heartbeat()
|
||||
self.assertIs(self.agent.defer_plutonium_preparation, expected)
|
||||
|
||||
def assert_preparation_yields_and_resumes(self, *, restarted):
|
||||
manifest, contents = self.release()
|
||||
operation = {"id": "cooperative-prepare", "kind": "plutonium.prepare", "payload": {"revision": 5354}}
|
||||
if restarted:
|
||||
self.agent.journal.accept(operation)
|
||||
self.agent.journal.update(operation["id"], "running")
|
||||
self.agent.journal.close()
|
||||
self.agent = fixtures.Agent(self.config, self.docker)
|
||||
self.assertEqual(self.agent.journal.get(operation["id"])["state"], "unknown")
|
||||
agent = self.agent
|
||||
open_url = self.opener(manifest, contents)
|
||||
|
||||
class BusyResponse(fixtures.Response):
|
||||
def read(self, size=-1):
|
||||
chunk = super().read(size)
|
||||
if chunk:
|
||||
agent.defer_plutonium_preparation = True
|
||||
return chunk
|
||||
|
||||
def become_busy_during_download(url):
|
||||
if url == self.file_url(contents["bin/plutonium.dll"]):
|
||||
return BusyResponse(contents["bin/plutonium.dll"])
|
||||
return open_url(url)
|
||||
|
||||
with patch.object(self.agent, "plutonium_open", side_effect=become_busy_during_download):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "accepted", result)
|
||||
self.assertIn("waiting for other host operations", json.loads(result["result"])["message"])
|
||||
self.assertIsNone(self.agent.executing_id)
|
||||
self.assertIsNone(self.agent.plutonium_cache_inventory())
|
||||
self.assertTrue(self.agent.defer_plutonium_preparation)
|
||||
# A yielded preparation must let other mutations finish before retrying.
|
||||
self.create()
|
||||
self.assertTrue(self.agent.inspect(fixtures.INSTANCE, 1)["running"])
|
||||
self.agent.defer_plutonium_preparation = False
|
||||
calls = []
|
||||
with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
self.assertEqual(result["id"], operation["id"])
|
||||
self.assertNotIn(self.file_url(contents[BOOTSTRAPPER]), calls)
|
||||
self.assertIn(self.file_url(contents["bin/plutonium.dll"]), calls)
|
||||
self.assertEqual(self.agent.plutonium_cache_inventory()["revision"], 5354)
|
||||
|
||||
def test_busy_host_yields_mid_download_and_resumes_completed_files(self):
|
||||
self.assert_preparation_yields_and_resumes(restarted=False)
|
||||
|
||||
def test_reconciled_preparation_yields_mid_download_and_resumes_same_operation(self):
|
||||
self.assert_preparation_yields_and_resumes(restarted=True)
|
||||
|
||||
def test_resumption_removes_orphans_before_storage_check_and_publication(self):
|
||||
manifest, contents = self.release()
|
||||
raw = json.dumps(manifest).encode()
|
||||
staging = self.agent.root / "agent/plutonium-cache" / (module.digest(raw) + ".partial/content")
|
||||
bootstrapper = staging / BOOTSTRAPPER
|
||||
bootstrapper.parent.mkdir(parents=True)
|
||||
bootstrapper.write_bytes(contents[BOOTSTRAPPER])
|
||||
orphans = [staging / ".download-abandoned", staging / "bin/.ec-abandoned", staging / "unlisted/extra.dat", staging / "cdn_info.json"]
|
||||
for orphan in orphans:
|
||||
orphan.parent.mkdir(parents=True, exist_ok=True)
|
||||
orphan.write_bytes(b"orphaned bytes")
|
||||
outside = Path(self.temp.name) / "outside-cache"
|
||||
outside.mkdir()
|
||||
(outside / "untouched").write_bytes(b"private outside bytes")
|
||||
link = staging / "outside-link"
|
||||
link.symlink_to(outside, target_is_directory=True)
|
||||
original_disk_usage = module.shutil.disk_usage
|
||||
|
||||
def checked_storage(path):
|
||||
for orphan in orphans:
|
||||
self.assertFalse(orphan.exists(), "orphan must be removed before storage accounting")
|
||||
self.assertFalse(link.is_symlink())
|
||||
return original_disk_usage(path)
|
||||
|
||||
calls = []
|
||||
with patch.object(module.shutil, "disk_usage", side_effect=checked_storage), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
|
||||
result = self.agent.prepare_plutonium({})
|
||||
content = self.cache_content()
|
||||
expected = {name for name in contents if not name.startswith("launcher")}
|
||||
actual = {path.relative_to(content).as_posix() for path in content.rglob("*") if path.is_file()}
|
||||
self.assertEqual(actual, expected | {"cdn_info.json"})
|
||||
self.assertEqual(result["bytes"], sum(len(contents[name]) for name in expected))
|
||||
self.assertEqual(sum(path.stat().st_size for path in content.rglob("*") if path.is_file()), result["bytes"] + len(raw))
|
||||
self.assertNotIn(self.file_url(contents[BOOTSTRAPPER]), calls)
|
||||
self.assertEqual((outside / "untouched").read_bytes(), b"private outside bytes")
|
||||
app = self.app()
|
||||
self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
|
||||
self.assert_runtime(app, manifest, contents)
|
||||
self.assertFalse((app / "runtime/plutonium/unlisted/extra.dat").exists())
|
||||
|
||||
def test_public_downloads_reject_other_origins_and_never_send_host_credentials(self):
|
||||
# Call the class method to bypass the fixture's default network blocker;
|
||||
# the underlying opener is mocked, so no request reaches the network.
|
||||
with patch.object(module.urllib.request, "build_opener") as build:
|
||||
for url in (
|
||||
"http://cdn.plutoniummod.com/updater/prod/info.json",
|
||||
"https://attacker.example/updater/prod/info.json",
|
||||
"https://cdn.plutoniummod.com.attacker.example/updater/prod/info.json",
|
||||
"https://secret@cdn.plutoniummod.com/updater/prod/info.json",
|
||||
"https://cdn.plutoniummod.com/updater/dev/info.json",
|
||||
):
|
||||
with self.subTest(url=url), self.assertRaises(AgentError):
|
||||
fixtures.Agent.plutonium_open(self.agent, url)
|
||||
build.assert_not_called()
|
||||
fixtures.Agent.plutonium_open(self.agent, module.PLUTONIUM_MANIFEST_URL)
|
||||
build.return_value.open.assert_called_once_with(module.PLUTONIUM_MANIFEST_URL, timeout=30)
|
||||
handler = build.call_args.args[0]
|
||||
for redirect in (module.PLUTONIUM_MANIFEST_URL, "https://attacker.example/collect"):
|
||||
with self.subTest(redirect=redirect), self.assertRaises(AgentError):
|
||||
handler.redirect_request(None, None, 302, "Found", {}, redirect)
|
||||
|
||||
def test_rejects_revision_downgrade_and_invalid_preparation_arguments(self):
|
||||
self.prepare()
|
||||
manifest, contents = self.release(5353)
|
||||
with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents)):
|
||||
with self.assertRaises(AgentError):
|
||||
self.agent.prepare_plutonium({"revision": 5354})
|
||||
for payload in ({"revision": -1}, {"revision": True}, {"revision": "5354"},
|
||||
{"minimumFreeBytes": -1}, {"minimumFreeBytes": True}, {"minimumFreeBytes": "10"}):
|
||||
with self.subTest(payload=payload), self.assertRaises(AgentError):
|
||||
self.agent.prepare_plutonium(payload)
|
||||
|
||||
def test_disk_reserve_failure_leaves_existing_cache_usable(self):
|
||||
_, contents, _, result = self.prepare()
|
||||
manifest, contents = self.release(5355, {**contents, "bin/plutonium.dll": b"new-runtime"})
|
||||
calls = []
|
||||
usage = type(module.shutil.disk_usage(self.agent.root))(10000, 9900, 100)
|
||||
with patch.object(module.shutil, "disk_usage", return_value=usage), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
|
||||
with self.assertRaises(AgentError):
|
||||
self.agent.prepare_plutonium({"minimumFreeBytes": 1000})
|
||||
self.assertEqual(calls, [module.PLUTONIUM_MANIFEST_URL])
|
||||
self.assertEqual(self.agent.plutonium_cache_inventory(), result)
|
||||
|
||||
def test_malformed_manifest_never_publishes_a_cache(self):
|
||||
good, contents = self.release()
|
||||
malformed = [
|
||||
None, [], {},
|
||||
{**good, "product": "other-product"},
|
||||
{**good, "revision": 0},
|
||||
{**good, "revision": True},
|
||||
{**good, "revision": "5354"},
|
||||
{**good, "baseUrl": "http://cdn.plutoniummod.com/updater/prod/"},
|
||||
{**good, "baseUrl": "https://attacker.example/"},
|
||||
{**good, "files": []},
|
||||
{**good, "files": {}},
|
||||
{**good, "files": good["files"][1:]},
|
||||
{**good, "files": good["files"] + [good["files"][0]]},
|
||||
]
|
||||
for manifest in malformed:
|
||||
with self.subTest(manifest=manifest), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents)):
|
||||
with self.assertRaises(AgentError):
|
||||
self.agent.prepare_plutonium({})
|
||||
self.assertIsNone(self.agent.plutonium_cache_inventory())
|
||||
|
||||
def test_manifest_rejects_unsafe_paths_sizes_and_hashes_before_downloading(self):
|
||||
good, contents = self.release()
|
||||
bad_entries = [
|
||||
{"name": name} for name in (
|
||||
".", "../escape", "/absolute", "bin/../../escape", "bin//file", "bin/./file",
|
||||
"bin\\file", "C:/escape", "bin/file\x00", "cdn_info.json",
|
||||
)
|
||||
] + [
|
||||
{"size": size} for size in (-1, True, "5", None)
|
||||
] + [
|
||||
{"hash": value} for value in ("", "0" * 39, "z" * 40, None)
|
||||
]
|
||||
for override in bad_entries:
|
||||
manifest = copy.deepcopy(good)
|
||||
manifest["files"][1].update(override)
|
||||
calls = []
|
||||
with self.subTest(override=override), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
|
||||
with self.assertRaises(AgentError):
|
||||
self.agent.prepare_plutonium({})
|
||||
self.assertEqual(calls, [module.PLUTONIUM_MANIFEST_URL])
|
||||
self.assertIsNone(self.agent.plutonium_cache_inventory())
|
||||
|
||||
def test_short_oversize_and_wrong_hash_downloads_are_not_published(self):
|
||||
manifest, contents = self.release()
|
||||
for body in (b"short", contents[BOOTSTRAPPER] + b"extra", b"X" * len(contents[BOOTSTRAPPER])):
|
||||
with self.subTest(body=body), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, {**contents, BOOTSTRAPPER: body})):
|
||||
with self.assertRaises(AgentError):
|
||||
self.agent.prepare_plutonium({})
|
||||
self.assertIsNone(self.agent.plutonium_cache_inventory())
|
||||
|
||||
def test_seeds_are_private_and_preserve_existing_lobby_state(self):
|
||||
manifest, contents, _, _ = self.prepare()
|
||||
first, second = self.app("first"), self.app("second")
|
||||
private = {
|
||||
"configs/server.cfg": b"secret config",
|
||||
"runtime/plutonium/plugins/private.dll": b"private plugin",
|
||||
"runtime/plutonium/storage/t5/logs/games_mp.log": b"lobby log",
|
||||
}
|
||||
for name, body in private.items():
|
||||
target = first / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(body)
|
||||
self.assertTrue(self.agent.seed_plutonium(first, IMAGE))
|
||||
self.assertTrue(self.agent.seed_plutonium(second, IMAGE))
|
||||
self.assert_runtime(first, manifest, contents)
|
||||
self.assert_runtime(second, manifest, contents)
|
||||
for name, body in private.items():
|
||||
self.assertEqual((first / name).read_bytes(), body)
|
||||
self.assertFalse((second / name).exists())
|
||||
first_file = first / "runtime/plutonium" / BOOTSTRAPPER
|
||||
second_file = second / "runtime/plutonium" / BOOTSTRAPPER
|
||||
cached_file = self.cache_content() / BOOTSTRAPPER
|
||||
self.assertNotEqual(first_file.stat().st_ino, second_file.stat().st_ino)
|
||||
self.assertNotEqual(first_file.stat().st_ino, cached_file.stat().st_ino)
|
||||
first_file.write_bytes(b"container-updated")
|
||||
self.assertEqual(second_file.read_bytes(), contents[BOOTSTRAPPER])
|
||||
self.assertEqual(cached_file.read_bytes(), contents[BOOTSTRAPPER])
|
||||
|
||||
def test_absent_or_corrupt_cache_and_unsupported_image_fall_back(self):
|
||||
app = self.app()
|
||||
self.assertFalse(self.agent.seed_plutonium(app, IMAGE))
|
||||
self.assertFalse((app / "runtime/plutonium/cdn_info.json").exists())
|
||||
self.prepare()
|
||||
for image in ("example.invalid/plutainer:latest", "ghcr.io/ayymoss/plutainer-other:latest", "ghcr.io/attacker/plutainer:latest"):
|
||||
with self.subTest(image=image):
|
||||
self.assertFalse(self.agent.seed_plutonium(app, image))
|
||||
(self.cache_content() / BOOTSTRAPPER).write_bytes(b"broken-cache")
|
||||
self.assertFalse(self.agent.seed_plutonium(app, IMAGE))
|
||||
self.assertFalse((app / "runtime/plutonium/cdn_info.json").exists())
|
||||
self.assertIsNone(self.agent.plutonium_cache_inventory())
|
||||
|
||||
def test_interrupted_seed_does_not_install_revision_marker_and_can_retry(self):
|
||||
manifest, contents, _, _ = self.prepare()
|
||||
app = self.app()
|
||||
marker = app / "runtime/plutonium/cdn_info.json"
|
||||
marker.parent.mkdir(parents=True)
|
||||
marker.write_text(json.dumps({"revision": 5353}))
|
||||
copy_file = module.managed_copy
|
||||
copied = []
|
||||
|
||||
def interrupted_copy(root, name, source, *args, **kwargs):
|
||||
self.assertFalse(marker.exists(), "revision marker must be installed only after all files")
|
||||
if copied:
|
||||
raise OSError("simulated full disk")
|
||||
copied.append(name)
|
||||
return copy_file(root, name, source, *args, **kwargs)
|
||||
|
||||
with patch.object(module, "managed_copy", side_effect=interrupted_copy):
|
||||
with self.assertRaises((AgentError, OSError)):
|
||||
self.agent.seed_plutonium(app, IMAGE)
|
||||
self.assertFalse(marker.exists())
|
||||
self.assertTrue(copied)
|
||||
self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
|
||||
self.assert_runtime(app, manifest, contents)
|
||||
|
||||
def test_failed_seed_prevents_container_start(self):
|
||||
self.prepare()
|
||||
with patch.object(module, "managed_copy", side_effect=OSError("simulated full disk")):
|
||||
payload = {"game": "t5", "libraryId": self.library_id, "image": IMAGE, "port": 28960, "env": {}}
|
||||
result = self.agent.execute(self.operation("instance.create", payload, "failed-seed"))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertFalse(any(call[:2] == ("container", "start") for call in self.docker.calls))
|
||||
self.assertFalse((self.agent.instance_dir(fixtures.INSTANCE, 1) / "app/runtime/plutonium/cdn_info.json").exists())
|
||||
|
||||
def test_symlink_in_cache_cannot_seed_outside_data(self):
|
||||
_, contents, _, _ = self.prepare()
|
||||
cached = self.cache_content() / BOOTSTRAPPER
|
||||
outside = Path(self.temp.name) / "outside"
|
||||
outside.write_bytes(contents[BOOTSTRAPPER])
|
||||
cached.unlink()
|
||||
cached.symlink_to(outside)
|
||||
app = self.app()
|
||||
self.assertFalse(self.agent.seed_plutonium(app, IMAGE))
|
||||
self.assertFalse((app / "runtime/plutonium/cdn_info.json").exists())
|
||||
|
||||
def test_create_applies_initial_overlay_after_seed_and_starts_without_cache(self):
|
||||
self.create()
|
||||
self.assertTrue(self.agent.inspect(fixtures.INSTANCE, 1)["running"])
|
||||
self.assertFalse((self.agent.instance_dir(fixtures.INSTANCE, 1) / "app/runtime/plutonium/cdn_info.json").exists())
|
||||
manifest, contents, _, _ = self.prepare()
|
||||
payload = self.agent.spec(fixtures.INSTANCE, 1)
|
||||
payload["port"] += 1
|
||||
payload["files"].append({"path": "runtime/plutonium/" + BOOTSTRAPPER, "contentBase64": base64.b64encode(b"custom-overlay").decode()})
|
||||
other_id = "00000000-0000-4000-8000-000000000002"
|
||||
result = self.agent.execute({**self.operation("instance.create", payload, "second-create"), "instanceId": other_id})
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
app = self.agent.instance_dir(other_id, 1) / "app"
|
||||
self.assertEqual((app / "runtime/plutonium" / BOOTSTRAPPER).read_bytes(), b"custom-overlay")
|
||||
self.assertEqual(json.loads((app / "runtime/plutonium/cdn_info.json").read_text()), manifest)
|
||||
self.assertEqual((app / "configs/server.cfg").read_bytes(), b"original")
|
||||
|
||||
def test_rebuild_seeds_the_new_generation_from_current_cache(self):
|
||||
payload = self.create()
|
||||
manifest, contents, _, _ = self.prepare()
|
||||
self.agent.rcon = lambda *_: {"ok": True, "players": 0}
|
||||
result = self.agent.execute(self.operation("instance.rebuild", {**payload, "nextGeneration": 2}, "rebuild"))
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
self.assertTrue(self.agent.inspect(fixtures.INSTANCE, 2)["running"])
|
||||
self.assert_runtime(self.agent.instance_dir(fixtures.INSTANCE, 2) / "app", manifest, contents)
|
||||
|
||||
def test_restart_preserves_the_instances_downloaded_runtime(self):
|
||||
self.prepare()
|
||||
self.create()
|
||||
runtime_file = self.agent.instance_dir(fixtures.INSTANCE, 1) / "app/runtime/plutonium" / BOOTSTRAPPER
|
||||
runtime_file.write_bytes(b"container-updated-runtime")
|
||||
self.agent.rcon = lambda *_: {"ok": True, "players": 0}
|
||||
with patch.object(self.agent, "seed_plutonium", side_effect=AssertionError("Restart must retain its runtime")):
|
||||
result = self.agent.execute(self.operation("instance.restart", op_id="restart"))
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
self.assertEqual(runtime_file.read_bytes(), b"container-updated-runtime")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user