Reuse verified artifacts during rebuilds (Refs #154)
This commit is contained in:
+159
-1
@@ -1666,7 +1666,7 @@ class AgentTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(AgentError, "Private bank"):
|
||||
self.agent.activate(INSTANCE, 1, "private", override_safety=True)
|
||||
with self.assertRaisesRegex(AgentError, "Private bank"):
|
||||
self.agent.rebuild_before(INSTANCE, 1, "private", {"files": []})
|
||||
self.agent.rebuild_before(INSTANCE, 1, "private", {})
|
||||
with self.assertRaisesRegex(AgentError, "Private bank"):
|
||||
self.agent.restore_files(app, [{"path": value, "contentBase64": None}])
|
||||
with self.assertRaisesRegex(AgentError, "Private bank"):
|
||||
@@ -1934,6 +1934,164 @@ class AgentTests(unittest.TestCase):
|
||||
self.assertEqual(json.loads(result["result"])["before"][0]["contentBase64"], base64.b64encode(b"original").decode())
|
||||
self.assertEqual(self.docker.calls, calls)
|
||||
|
||||
def artifact_rebuild(self, staged=False):
|
||||
if staged:
|
||||
operation = self.staged_rebuild()
|
||||
else:
|
||||
spec = self.create()
|
||||
self.agent.rcon = lambda *_: {"ok": True, "players": 0}
|
||||
operation = self.operation("instance.rebuild", {"spec": spec, "nextGeneration": 2}, "rebuild")
|
||||
spec = operation["payload"]["spec"]
|
||||
contents = {"configs/server.cfg": b"replacement", "scripts/replacement.gsc": b"verified artifact bytes"}
|
||||
spec["files"] = [{"path": path, "contentSha256": module.digest(content), "sha256": module.digest(content)} for path, content in contents.items()]
|
||||
spec["files"].append({"path": "configs/empty.cfg", "contentBase64": ""})
|
||||
spec["bank"] = {"configPath": "configs/electric_cherry_bank.cfg", "token": "ec_bank_" + "a" * 32}
|
||||
return operation, contents
|
||||
|
||||
def assert_rebuild_uses_preflight_artifacts(self, staged):
|
||||
operation, contents = self.artifact_rebuild(staged)
|
||||
original_request = json.dumps(operation, sort_keys=True)
|
||||
available = {module.digest(content): content for content in contents.values()}
|
||||
|
||||
def fetch(method, url):
|
||||
# Connectivity disappears as soon as the complete preflight set has
|
||||
# been read, before staged comparison or destructive replacement.
|
||||
if not available:
|
||||
raise module.urllib.error.URLError(TimeoutError())
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
|
||||
return Response(available.pop(url.rsplit("/", 1)[-1]))
|
||||
|
||||
with patch.object(self.agent, "artifact_open", side_effect=fetch) as downloads:
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "completed", result)
|
||||
self.assertEqual(downloads.call_count, len(contents))
|
||||
self.assertFalse(available)
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
|
||||
app = self.agent.instance_dir(INSTANCE, 2) / "app"
|
||||
for path, content in contents.items():
|
||||
self.assertEqual((app / path).read_bytes(), content)
|
||||
self.assertEqual((app / "configs/empty.cfg").read_bytes(), b"")
|
||||
self.assertEqual((app / "configs/electric_cherry_bank.cfg").read_text(), 'set tws_bank_token "ec_bank_' + "a" * 32 + '"\n')
|
||||
self.assertEqual(json.dumps(operation, sort_keys=True), original_request)
|
||||
self.assertEqual(json.loads(result["request"]), operation)
|
||||
self.assertEqual(self.agent.spec(INSTANCE, 2), operation["payload"]["spec"])
|
||||
labels = self.docker.inspect(self.agent.container_name(INSTANCE, 2))["Config"]["Labels"]
|
||||
self.assertEqual(labels[module.LABEL + ".operation"], operation["id"])
|
||||
self.assertEqual(labels[module.LABEL + ".spec-sha256"], module.digest(json.dumps(operation["payload"]["spec"], sort_keys=True, separators=(",", ":")).encode()))
|
||||
receipt = json.loads(result["result"])
|
||||
self.assertEqual(receipt["before"], [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"original").decode()}] if staged else [])
|
||||
for content in contents.values():
|
||||
self.assertNotIn(base64.b64encode(content).decode(), json.dumps(result))
|
||||
|
||||
# A later operation with the same artifacts must download its own set.
|
||||
later = self.operation("instance.rebuild", {"spec": operation["payload"]["spec"], "nextGeneration": 3}, "later-rebuild", generation=2)
|
||||
with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
|
||||
self.assertEqual(self.agent.execute(later)["state"], "failed")
|
||||
downloads.assert_called_once()
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, 3)["exists"])
|
||||
|
||||
def test_unstaged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self):
|
||||
self.assert_rebuild_uses_preflight_artifacts(staged=False)
|
||||
|
||||
def test_staged_rebuild_succeeds_without_artifact_connectivity_after_preflight(self):
|
||||
self.assert_rebuild_uses_preflight_artifacts(staged=True)
|
||||
|
||||
def test_rebuild_preflight_download_failure_preserves_original_and_discards_partial_set(self):
|
||||
operation, contents = self.artifact_rebuild()
|
||||
with patch.object(self.agent, "artifact_open", side_effect=[Response(contents["configs/server.cfg"]), module.urllib.error.URLError(TimeoutError())]) as downloads:
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertEqual(downloads.call_count, 2)
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
|
||||
self.assertFalse(self.agent.instance_dir(INSTANCE, 2).exists())
|
||||
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
|
||||
operation["id"] = "retry-rebuild"
|
||||
with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
|
||||
self.assertEqual(self.agent.execute(operation)["state"], "failed")
|
||||
self.assertTrue(downloads.call_args.args[1].endswith(module.digest(contents["configs/server.cfg"])))
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
|
||||
|
||||
def test_held_rebuild_discards_prepared_artifacts_before_retry(self):
|
||||
operation, contents = self.artifact_rebuild(staged=True)
|
||||
with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}):
|
||||
self.assertEqual(self.agent.execute(operation)["state"], "accepted")
|
||||
with patch.object(self.agent, "artifact_open", side_effect=module.urllib.error.URLError(TimeoutError())) as downloads:
|
||||
self.assertEqual(self.agent.execute(operation)["state"], "failed")
|
||||
downloads.assert_called_once()
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
|
||||
self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
|
||||
|
||||
def test_rebuild_preflight_rejects_corrupt_and_oversized_artifacts(self):
|
||||
operation, contents = self.artifact_rebuild()
|
||||
for index, content in enumerate((b"corrupt", b"x" * (module.MAX_FILE + 1))):
|
||||
with self.subTest(index=index):
|
||||
operation["id"] = f"invalid-artifact-{index}"
|
||||
with patch.object(self.agent, "artifact_open", return_value=Response(content)):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("checksum mismatch" if index == 0 else "size limit", result["error"])
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
|
||||
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
|
||||
|
||||
def test_rebuild_preflight_bounds_complete_file_set_at_64_mib(self):
|
||||
operation, _ = self.artifact_rebuild()
|
||||
spec = operation["payload"]["spec"]
|
||||
content = b"x" * module.MAX_FILE
|
||||
spec["files"] = [{"path": f"scripts/{index}.gsc", "contentSha256": module.digest(content)} for index in range(2)]
|
||||
# The generated private bank file also counts toward the operation limit.
|
||||
with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("64 MiB operation limit", result["error"])
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
|
||||
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
|
||||
del spec["bank"]
|
||||
operation["id"] = "at-limit"
|
||||
with patch.object(self.agent, "artifact_open", side_effect=lambda *args: Response(content)), patch.object(self.agent, "rcon", return_value={"ok": True, "players": 1}):
|
||||
self.assertEqual(self.agent.execute(operation)["state"], "accepted")
|
||||
|
||||
def test_replacement_validation_rejects_incomplete_or_mismatched_prepared_bytes_without_downloads(self):
|
||||
operation, contents = self.artifact_rebuild()
|
||||
spec = operation["payload"]["spec"]
|
||||
del spec["bank"]
|
||||
prepared = {**contents, "configs/empty.cfg": b""}
|
||||
invalid_sets = [
|
||||
{path: content for path, content in prepared.items() if path != "configs/server.cfg"},
|
||||
{**prepared, "configs/extra.cfg": b"unexpected"},
|
||||
{**prepared, "configs/server.cfg": b"wrong artifact"},
|
||||
{**prepared, "configs/empty.cfg": b"wrong inline bytes"},
|
||||
{**prepared, "configs/server.cfg": None},
|
||||
]
|
||||
with patch.object(self.agent, "artifact_open", side_effect=AssertionError("Prepared validation must not download")) as downloads:
|
||||
for index, files in enumerate(invalid_sets):
|
||||
with self.subTest(index=index), self.assertRaises(AgentError):
|
||||
self.agent.validate_spec(spec, prepared_files=files)
|
||||
downloads.assert_not_called()
|
||||
|
||||
def test_rebuild_preflight_preserves_file_path_and_duplicate_checks(self):
|
||||
operation, _ = self.artifact_rebuild()
|
||||
spec = operation["payload"]["spec"]
|
||||
for index, paths in enumerate((["../escape.cfg"], ["configs/electric_cherry_bank.cfg"], ["configs/same.cfg"] * 2)):
|
||||
with self.subTest(paths=paths):
|
||||
operation["id"] = f"invalid-paths-{index}"
|
||||
spec["files"] = [{"path": path, "contentBase64": ""} for path in paths]
|
||||
self.assertEqual(self.agent.execute(operation)["state"], "failed")
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
|
||||
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
|
||||
|
||||
def test_staged_rebuild_rejects_verified_artifacts_that_differ_from_the_stage(self):
|
||||
operation, contents = self.artifact_rebuild(staged=True)
|
||||
contents["configs/server.cfg"] = b"different verified replacement"
|
||||
operation["payload"]["spec"]["files"][0].update(contentSha256=module.digest(contents["configs/server.cfg"]), sha256=module.digest(contents["configs/server.cfg"]))
|
||||
with patch.object(self.agent, "artifact_open", side_effect=[Response(content) for content in contents.values()]):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIn("does not match staged file content", result["error"])
|
||||
self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
|
||||
self.assertFalse(any(args[:2] in (("container", "stop"), ("container", "rm")) for args in self.docker.calls))
|
||||
|
||||
def test_rebuild_moves_generation_and_rejects_old_commands(self):
|
||||
payload = self.create()
|
||||
self.agent.rcon = lambda *_: {"ok": True, "players": 0}
|
||||
|
||||
Reference in New Issue
Block a user