From d9fa15cdf6df15a12fb0c8e70773283afe63a487 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 12 Oct 2026 23:32:20 -0400 Subject: [PATCH] Protect private bank configuration from drift reconciliation (Refs #152) --- agent/CHANGELOG.md | 8 + agent/ec_agent.py | 25 +- agent/tests/test_agent.py | 53 +++- docs/guides/bank.md | 9 + src/lib/server/backends/files.ts | 8 + src/lib/server/db/file-blobs.ts | 18 ++ src/lib/server/fleet/drift.ts | 22 +- src/lib/server/fleet/private-config.ts | 54 ++++ src/lib/server/fleet/scripts.ts | 7 + src/lib/server/fleet/staged-files.ts | 2 + src/lib/server/hosts/index.ts | 34 ++- src/lib/server/maintenance/recovery.ts | 10 +- .../server/provisioning/activation-files.ts | 2 + src/lib/server/provisioning/native.ts | 9 +- .../api/agent/blobs/[sha256]/+server.ts | 2 + .../campaigns/[id]/file/[fileId]/+server.ts | 3 +- .../deployments/[id]/file/[fileId]/+server.ts | 3 +- static/agent/ec-agent.py | 25 +- static/agent/manifest.json | 4 +- static/agent/manifest.json.sig | 8 - tests/unit/native-drift-manifest.test.ts | 1 + tests/unit/private-bank-drift.test.ts | 269 ++++++++++++++++++ tests/unit/watchdog.test.ts | 49 ++++ 23 files changed, 597 insertions(+), 28 deletions(-) create mode 100644 src/lib/server/fleet/private-config.ts delete mode 100644 static/agent/manifest.json.sig create mode 100644 tests/unit/private-bank-drift.test.ts diff --git a/agent/CHANGELOG.md b/agent/CHANGELOG.md index dfd1a9a..fb23cab 100644 --- a/agent/CHANGELOG.md +++ b/agent/CHANGELOG.md @@ -2,6 +2,14 @@ ## 2026-10-12 +### 1.9.3 — Protect private bank configuration (EC #152) + +- Exclude the generated bank config from file manifests and reject file reads, + staged changes, stale activations, rollback snapshots and log reads targeting + it. Creation and rebuilding still inject the private instance credential. +- Package and offline-sign this release, then update agents before resuming + drift recovery. Old private-file stages fail safely and require a fresh plan. + ### 1.9.1 — Public game-bank endpoint - Companion to [TWS scripts #13](https://gitea.thewrightserver.net/josh/tws-zombies-scripts/issues/13). diff --git a/agent/ec_agent.py b/agent/ec_agent.py index a0d6031..e8515d6 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.9.2" +VERSION = "1.9.3" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -400,7 +400,17 @@ def managed_open(root, value): os.close(fd) +def private_bank_config(value): + return isinstance(value, str) and PurePosixPath(value).as_posix() == "configs/electric_cherry_bank.cfg" + + +def require_public_file(value): + if private_bank_config(value): + raise Rejected("Private bank configuration cannot be read or changed by file operations") + + def managed_read(root, value): + require_public_file(value) try: with managed_open(root, value) as stream: if os.fstat(stream.fileno()).st_size > MAX_FILE: @@ -446,6 +456,8 @@ def managed_manifest(root, payload, stopping): def validate_paths(): for value in paths: + if private_bank_config(value): + continue check_budget() try: with managed_open(root, value): @@ -480,7 +492,7 @@ def managed_manifest(root, payload, stopping): entries += 1 if entries > MANIFEST_MAX_ENTRIES: raise AgentError("Manifest scan exceeded its entry limit") - if child.name in ignored: + if child.name in ignored or private_bank_config(relative + "/" + child.name): continue info = os.stat(child.name, dir_fd=directory, follow_symlinks=False) if stat_module.S_ISLNK(info.st_mode): @@ -1949,6 +1961,8 @@ class Agent: files = payload.get("files", []) if not isinstance(files, list): raise Rejected("Invalid initial file list") + for item in files: + require_public_file(item["path"]) bank = payload.get("bank") if bank is None: return files @@ -2219,6 +2233,7 @@ class Agent: paths = set() total_size = 0 for item in files: + require_public_file(item["path"]) scoped(directory / "app", item["path"]) if item["path"] in paths: raise AgentError("Duplicate staged file path") @@ -2267,6 +2282,8 @@ class Agent: stage = directory / "stages" / identifier(stage_id) marker = stage / "manifest.json" saved = json.loads(marker.read_text()) + for item in saved["files"]: + require_public_file(item["path"]) if saved["state"] == "activated": return saved["before"] if saved["state"] != "staged": @@ -2297,6 +2314,8 @@ class Agent: return before def restore_files(self, app, before): + for item in before: + require_public_file(item["path"]) for item in before: managed_write(app, item["path"], None if item["contentBase64"] is None else self.decode_file(item), int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) @@ -2351,6 +2370,7 @@ class Agent: def read_log(self, instance_id, generation, payload): app = self.instance_dir(instance_id, generation) / "app" value = payload["path"] + require_public_file(value) count = min(2000, max(1, int(payload.get("lines", 200)))) # Validate the original relative spelling independently of symlinks. scoped(self.root / "agent" / "log-validation", value) @@ -2367,6 +2387,7 @@ class Agent: raise AgentError("Log symlink resolves outside the managed instance") from None # Reopen the resolved relative target through no-follow directory FDs; # swapping either the original link or a target component cannot escape. + require_public_file(relative) with managed_open(app, relative) as stream: stat = os.fstat(stream.fileno()) if payload.get("mode") == "incremental": diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 0942458..0547107 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -1644,9 +1644,51 @@ class AgentTests(unittest.TestCase): self.assertEqual(self.agent.execute(self.operation("instance.remove"))["state"], "failed") self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls)) - def test_create_materializes_private_bank_config_from_instance_credential(self): + def test_private_bank_file_survives_stale_file_plans_without_snapshots(self): + self.create() + app = self.agent.instance_dir(INSTANCE, 1) / "app" + value = "configs/electric_cherry_bank.cfg" + target = app / value + secret = b'set tws_bank_token "ec_bank_private_regression"\n' + target.write_bytes(secret) + files = [{"id": 9, "path": value, "delete": True, "contentBase64": None}] + with self.assertRaisesRegex(AgentError, "Private bank"): + self.agent.stage(INSTANCE, 1, {"stageId": "private", "files": files}, "stage") + marker = self.agent.instance_dir(INSTANCE, 1) / "stages/private/manifest.json" + # A stage written by a previous agent must be rejected on activation too. + module.atomic_json(marker, {"state": "staged", "files": files}) + with self.assertRaisesRegex(AgentError, "Private bank"): + self.agent.activate(INSTANCE, 1, "private", override_safety=True) + with self.assertRaisesRegex(AgentError, "Private bank"): + self.agent.rebuild_before(INSTANCE, 1, "private", {"files": []}) + with self.assertRaisesRegex(AgentError, "Private bank"): + self.agent.restore_files(app, [{"path": value, "contentBase64": None}]) + with self.assertRaisesRegex(AgentError, "Private bank"): + self.agent.file_operation("files.read", INSTANCE, 1, {"path": value}, "read") + with self.assertRaisesRegex(AgentError, "Private bank"): + self.agent.read_log(INSTANCE, 1, {"path": value}) + manifest = self.agent.file_operation("files.manifest", INSTANCE, 1, + {"roots": ["configs"], "paths": [value], "ignoreNames": []}, "manifest") + self.assertNotIn(value, [item["path"] for item in manifest["files"]]) + self.assertEqual(target.read_bytes(), secret) + self.assertNotIn("before", json.loads(marker.read_text())) + with self.assertRaisesRegex(AgentError, "Private bank"): + self.agent.initial_files({"files": [{"path": value, "contentBase64": base64.b64encode(secret).decode()}]}) + + def prepare_t6_library(self): + target = self.agent.library_dir("t6", self.library_id) + module.shutil.copytree(self.library, target) + manifest = json.loads((target / "manifest.json").read_text()) + manifest["game"] = "t6" + module.atomic_json(target / "manifest.json", manifest) + + def test_t6_create_materializes_private_bank_config(self): + self.prepare_t6_library() + self.test_create_materializes_private_bank_config_from_instance_credential(game="t6") + + def test_create_materializes_private_bank_config_from_instance_credential(self, game="t5"): secret = "ec_bank_" + "a" * 43 - payload = {"name": "server", "game": "t5", "libraryId": self.library_id, + payload = {"name": "server", "game": game, "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960, "bank": {"token": secret, "configPath": "configs/electric_cherry_bank.cfg"}} operation = self.operation("instance.create", payload, "create") @@ -1699,9 +1741,14 @@ class AgentTests(unittest.TestCase): self.assertFalse(path.exists()) self.assertEqual(self.docker.calls, calls) - def test_endpoint_upgrade_rebuild_retries_original_decisions_before_starting_public_bank(self): + def test_t6_rebuild_preserves_private_bank_configuration(self): + self.prepare_t6_library() + self.test_endpoint_upgrade_rebuild_retries_original_decisions_before_starting_public_bank(game="t6") + + def test_endpoint_upgrade_rebuild_retries_original_decisions_before_starting_public_bank(self, game="t5"): secret = "ec_bank_" + "a" * 43 payload = self.create() + payload["game"] = game payload["bank"] = {"token": secret, "configPath": "configs/electric_cherry_bank.cfg"} paths = [self.bank_journal(game=game, endpoint=endpoint) for game in ("t5", "t6") diff --git a/docs/guides/bank.md b/docs/guides/bank.md index 98fd596..bd4e969 100644 --- a/docs/guides/bank.md +++ b/docs/guides/bank.md @@ -44,6 +44,15 @@ Do not upload `electric_cherry_bank.cfg`: its filename is reserved. Protect the controller database, host operation journal and per-instance runtime directory, which contain private credentials. +Drift scans exclude the exact generated configuration path for both games. +A maintenance rescan resolves old bank-only incidents and retains other file +changes in mixed incidents. Private-file campaign plans saved before this fix +are rejected at activation; cancel them and rescan rather than retrying the old +plan. Upgrade host agents to **1.9.3 or later** before resuming drift recovery +so stages already saved on a host also reject private-file reads and changes. +Legacy private bank-file campaign diffs and their blob downloads are blocked; +this does not erase previously captured bytes from databases or backups. + Retries, rebuilds and host moves of the same instance reuse its credential. Retiring the instance immediately revokes authentication. Creating another server with the same display name produces a new instance ID and secret; it diff --git a/src/lib/server/backends/files.ts b/src/lib/server/backends/files.ts index 73ad9f8..43875a0 100644 --- a/src/lib/server/backends/files.ts +++ b/src/lib/server/backends/files.ts @@ -1,3 +1,4 @@ +import { isPrivateConfigPath } from '../fleet/private-config.js'; import * as fs from 'node:fs/promises'; import { join, posix, relative, resolve, sep } from 'node:path'; import { config } from '../config.js'; @@ -85,6 +86,13 @@ export async function nativeFileManifest( } export async function readFile(path: string): Promise { + for (const root of [config.serversRoot, config.t6ServersRoot]) { + const [, ...parts] = relative(resolve(root), resolve(path)).split(sep); + if (isPrivateConfigPath(parts.join('/'))) + throw new Error( + 'Private bank configuration is not available through shared file reads.' + ); + } const target = remote(path); if (!target) return fs.readFile(path); const result = await nativeRequest<{ diff --git a/src/lib/server/db/file-blobs.ts b/src/lib/server/db/file-blobs.ts index 03171db..5f81838 100644 --- a/src/lib/server/db/file-blobs.ts +++ b/src/lib/server/db/file-blobs.ts @@ -1,3 +1,4 @@ +import { isPrivateConfigEvidencePath } from '../fleet/private-config.js'; import { createHash } from 'node:crypto'; import type Database from 'better-sqlite3'; import { db } from './index.js'; @@ -22,7 +23,18 @@ export function putFileBlob( return sha256; } +export function isPrivateFileBlob(sha256: string, handle: Database.Database = db()): boolean { + const rows = handle + .prepare( + `SELECT rel_path FROM deployment_files + WHERE content_before_hash=? OR content_after_hash=?` + ) + .all(sha256, sha256) as { rel_path: string }[]; + return rows.some((row) => isPrivateConfigEvidencePath(row.rel_path)); +} + export function getFileBlob(sha256: string, handle: Database.Database = db()): Buffer { + if (isPrivateFileBlob(sha256, handle)) throw new Error('Private file content is unavailable.'); const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as { content: Buffer } | undefined; if (!row) throw new Error('File content is unavailable or has expired.'); @@ -58,6 +70,12 @@ export function packFileJson( if (Array.isArray(value)) return { value: value.map((item) => packFileJson(item, handle, hashes).value), hashes }; if (!value || typeof value !== 'object') return { value, hashes }; + if ( + 'path' in value && + typeof value.path === 'string' && + isPrivateConfigEvidencePath(value.path) + ) + return { value: { path: value.path, redacted: true }, hashes }; const result: Record = {}; for (const [key, item] of Object.entries(value)) { if ( diff --git a/src/lib/server/fleet/drift.ts b/src/lib/server/fleet/drift.ts index 58f0895..872f957 100644 --- a/src/lib/server/fleet/drift.ts +++ b/src/lib/server/fleet/drift.ts @@ -1,3 +1,4 @@ +import { isPrivateConfigTarget } from './private-config.js'; import { putFileBlob } from '../db/file-blobs.js'; import { getVersionContent } from './scripts.js'; import { @@ -213,6 +214,7 @@ async function scanServerFiles(server: string): Promise { if (onDisk.length === 0) continue; const untracked = await scanBatch(onDisk, async (relPath): Promise => { + if (isPrivateConfigTarget(server, kind, relPath)) return null; if (managed.has(`${kind}:${relPath}`)) return null; if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) return null; @@ -438,6 +440,7 @@ function driftTargets(servers?: string[]): string[] { async function describePlan(servers: string[], entries: DriftEntry[]): Promise { const files: DriftPlanFile[] = []; for (const entry of entries) { + if (isPrivateConfigTarget(entry.server, entry.kind, entry.relPath)) continue; const before = await readFile( join(destOfKind(entry.server, entry.kind), entry.relPath) ).catch((error: NodeJS.ErrnoException) => { @@ -506,7 +509,9 @@ export interface PreparedDriftCampaign { /** Capture a rebuild's file evidence without changing the running instance. */ export async function prepareDriftCampaign(server: string): Promise { const servers = driftTargets([server]); - const entries = await scanServerDrift(server); + const entries = (await scanServerDrift(server)).filter( + (entry) => !isPrivateConfigTarget(server, entry.kind, entry.relPath) + ); const files = await scanBatch(entries, async (entry) => { const before = await readExistingFile(join(destOfKind(server, entry.kind), entry.relPath)); const after = @@ -547,7 +552,11 @@ export function createDriftRebuildCampaign( incidentId: string | number, prepared: PreparedDriftCampaign ): string | null { - if (!prepared.files.length) return null; + // Prepared evidence may have been created before the private-file exemption. + const files = prepared.files.filter( + ({ file }) => !isPrivateConfigTarget(file.server, file.kind, file.relPath) + ); + if (!files.length) return null; if ( prepared.plan.servers.length !== 1 || prepared.plan.servers[0] !== server || @@ -575,7 +584,7 @@ export function createDriftRebuildCampaign( const insertFile = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`); - for (const { file, before, after } of prepared.files) { + for (const { file, before, after } of files) { insertFile.run( id, server, @@ -592,7 +601,7 @@ export function createDriftRebuildCampaign( `INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id) VALUES (?, ?, ?, 'pending', ?)` ) - .run(id, server, prepared.files.length, operationId); + .run(id, server, files.length, operationId); db() .prepare( "INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)" @@ -627,7 +636,9 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise = []; for (const name of scanTargets) { - const entries = await scanServerDrift(name); + const entries = (await scanServerDrift(name)).filter( + (entry) => !isPrivateConfigTarget(name, entry.kind, entry.relPath) + ); perServer.push({ server: name, removed: [], restored: [], skipped: [], entries }); } @@ -692,6 +703,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise(); const admitted = rows.filter((operation) => { + // Withhold unsafe legacy commands without breaking the entire delivery stream. + if (operation.kind.startsWith('files.') || operation.kind === 'logs.read') { + try { + getHostOperationPayload(hostId, operation.id); + } catch (error) { + if (error instanceof PrivateFileOperationError) return false; + throw error; + } + } if ( !libraryKinds.includes(operation.kind) || preparing.has(hostId) || @@ -1431,6 +1448,9 @@ export function getHostOperationPayload(hostId: string, id: string): Buffer { ) .get(id, hostId) as { payload: Buffer } | undefined; if (!row) throw new Error('Host operation was not found for this host.'); + // Block old private-file commands before delivery to agents predating the guard. + if (row.payload.includes(BANK_CONFIG_FILE)) + assertPublicFilePayload(JSON.parse(row.payload.toString())); if ( !row.payload.includes('"contentSha256"') || getHost(hostId)?.inventory.capabilities?.includes('files.blobs') @@ -1468,8 +1488,18 @@ export function reportHostOperationStatus( if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.'); if (result !== null) { + const target = + operation.kind === 'files.read' + ? (database() + .prepare('SELECT target_path FROM ec_host_operations WHERE id=?') + .get(id) as { target_path: string | null }) + : null; const packed = packFileJson( - operation.kind === 'logs.read' ? logResultMetadata(input.result) : input.result + target?.target_path && isPrivateConfigPath(target.target_path) + ? { redacted: true } + : operation.kind === 'logs.read' + ? logResultMetadata(input.result) + : input.result ); result = JSON.stringify(packed.value); referenceBlobs('operation-result', id, packed.hashes); diff --git a/src/lib/server/maintenance/recovery.ts b/src/lib/server/maintenance/recovery.ts index d6101e7..3829594 100644 --- a/src/lib/server/maintenance/recovery.ts +++ b/src/lib/server/maintenance/recovery.ts @@ -710,7 +710,15 @@ async function scanServer(server: string, forceScan: boolean): Promise { setSetting(driftKey, JSON.stringify({ identity, at: iso() })); if (!prepared.plan.files.length) { const current = active(server, 'drift'); - if (current) transition(current.id, 'resolved', 'Server files match the promoted setup'); + if (current) { + db() + .prepare('UPDATE watchdog_incidents SET details=? WHERE id=?') + .run( + JSON.stringify({ files: [], fingerprint: prepared.plan.fingerprint }), + current.id + ); + transition(current.id, 'resolved', 'Server files match the promoted setup'); + } return; } const problem = detect( diff --git a/src/lib/server/provisioning/activation-files.ts b/src/lib/server/provisioning/activation-files.ts index bf19bdf..09312e5 100644 --- a/src/lib/server/provisioning/activation-files.ts +++ b/src/lib/server/provisioning/activation-files.ts @@ -1,3 +1,4 @@ +import { assertPublicFileTarget } from '../fleet/private-config.js'; import { db } from '../db/index.js'; import { blobSize, getFileBlob } from '../db/file-blobs.js'; import type { LaunchRecipePlan } from '../fleet/launch-recipes.js'; @@ -22,6 +23,7 @@ export function applyActivationFiles( version: number | null; })[]; for (const change of changes) { + assertPublicFileTarget(change.server, change.kind, change.rel_path); if (change.kind !== 'script' && change.kind !== 'config' && change.kind !== 'plugin') throw new Error('File activation requires a runtime file kind.'); recipe.files = recipe.files.filter( diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 65de993..2669107 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -1,3 +1,4 @@ +import { assertPublicFileTarget, isPrivateConfigPath } from '../fleet/private-config.js'; import { assertScalingOwnership } from '../autoscaling/ownership.js'; import { removalAuthorization } from '../autoscaling/authorization.js'; import { scalingIntent } from '../autoscaling/admission.js'; @@ -509,6 +510,7 @@ function filesFor(job: Job): FilePlan[] { 'id' | 'kind' | 'rel_path' | 'check_before' | 'expected_before_sha' > & { content_hash: string | null } >; + for (const row of rows) assertPublicFileTarget(job.server, row.kind, row.rel_path); return rows.map((row) => ({ id: row.id, path: posix.join( @@ -532,7 +534,12 @@ function storedFiles(job: Job): FilePlan[] { const row = db() .prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?') .get(job.id) as { files_json: string | null } | undefined; - return (JSON.parse(row?.files_json ?? '[]') as FilePlan[]).map(fileMetadata); + const files = (JSON.parse(row?.files_json ?? '[]') as FilePlan[]).map(fileMetadata); + if (files.some((file) => isPrivateConfigPath(file.path))) + throw new Error( + 'Private bank configuration cannot be captured or changed by a file campaign.' + ); + return files; } /** Agent receipts are the authoritative before images; do not read a controller mount. */ diff --git a/src/routes/api/agent/blobs/[sha256]/+server.ts b/src/routes/api/agent/blobs/[sha256]/+server.ts index 1e0874a..1afef0e 100644 --- a/src/routes/api/agent/blobs/[sha256]/+server.ts +++ b/src/routes/api/agent/blobs/[sha256]/+server.ts @@ -1,3 +1,4 @@ +import { isPrivateFileBlob } from '$lib/server/db/file-blobs.js'; import { error } from '@sveltejs/kit'; import { authenticateAgent } from '$lib/server/hosts/http.js'; import { db } from '$lib/server/db/index.js'; @@ -5,6 +6,7 @@ import { db } from '$lib/server/db/index.js'; export function GET({ request, params }: { request: Request; params: { sha256: string } }) { const host = authenticateAgent(request); if (!/^[a-f0-9]{64}$/.test(params.sha256)) error(404, 'File not found.'); + if (isPrivateFileBlob(params.sha256)) error(404, 'File not found.'); const row = db() .prepare( `SELECT b.content,b.size FROM file_blobs b WHERE b.sha256=? AND EXISTS ( diff --git a/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts b/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts index cbefc86..29de4d0 100644 --- a/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts +++ b/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts @@ -1,3 +1,4 @@ +import { isPrivateConfigEvidencePath } from '$lib/server/fleet/private-config.js'; import { blobContent } from '$lib/server/db/file-blobs.js'; import { json, error } from '@sveltejs/kit'; import { db } from '$lib/server/db/index.js'; @@ -35,7 +36,7 @@ export async function GET({ params }: { params: { id: string; fileId: string } } snapshot_pending: number; } | undefined; - if (!row) throw error(404, 'file not found'); + if (!row || isPrivateConfigEvidencePath(row.rel_path)) throw error(404, 'file not found'); if (row.snapshot_pending) return json( { diff --git a/src/routes/api/deployments/[id]/file/[fileId]/+server.ts b/src/routes/api/deployments/[id]/file/[fileId]/+server.ts index 56d9aa9..f5fdde9 100644 --- a/src/routes/api/deployments/[id]/file/[fileId]/+server.ts +++ b/src/routes/api/deployments/[id]/file/[fileId]/+server.ts @@ -1,3 +1,4 @@ +import { isPrivateConfigEvidencePath } from '$lib/server/fleet/private-config.js'; import { blobContent } from '$lib/server/db/file-blobs.js'; import { json, error } from '@sveltejs/kit'; import { db } from '$lib/server/db/index.js'; @@ -22,7 +23,7 @@ export async function GET({ params }: { params: { id: string; fileId: string } } content_after: Buffer | null; } | undefined; - if (!row) throw error(404, 'file not found'); + if (!row || isPrivateConfigEvidencePath(row.rel_path)) throw error(404, 'file not found'); return json({ verb: row.verb, diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index a0d6031..e8515d6 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.9.2" +VERSION = "1.9.3" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -400,7 +400,17 @@ def managed_open(root, value): os.close(fd) +def private_bank_config(value): + return isinstance(value, str) and PurePosixPath(value).as_posix() == "configs/electric_cherry_bank.cfg" + + +def require_public_file(value): + if private_bank_config(value): + raise Rejected("Private bank configuration cannot be read or changed by file operations") + + def managed_read(root, value): + require_public_file(value) try: with managed_open(root, value) as stream: if os.fstat(stream.fileno()).st_size > MAX_FILE: @@ -446,6 +456,8 @@ def managed_manifest(root, payload, stopping): def validate_paths(): for value in paths: + if private_bank_config(value): + continue check_budget() try: with managed_open(root, value): @@ -480,7 +492,7 @@ def managed_manifest(root, payload, stopping): entries += 1 if entries > MANIFEST_MAX_ENTRIES: raise AgentError("Manifest scan exceeded its entry limit") - if child.name in ignored: + if child.name in ignored or private_bank_config(relative + "/" + child.name): continue info = os.stat(child.name, dir_fd=directory, follow_symlinks=False) if stat_module.S_ISLNK(info.st_mode): @@ -1949,6 +1961,8 @@ class Agent: files = payload.get("files", []) if not isinstance(files, list): raise Rejected("Invalid initial file list") + for item in files: + require_public_file(item["path"]) bank = payload.get("bank") if bank is None: return files @@ -2219,6 +2233,7 @@ class Agent: paths = set() total_size = 0 for item in files: + require_public_file(item["path"]) scoped(directory / "app", item["path"]) if item["path"] in paths: raise AgentError("Duplicate staged file path") @@ -2267,6 +2282,8 @@ class Agent: stage = directory / "stages" / identifier(stage_id) marker = stage / "manifest.json" saved = json.loads(marker.read_text()) + for item in saved["files"]: + require_public_file(item["path"]) if saved["state"] == "activated": return saved["before"] if saved["state"] != "staged": @@ -2297,6 +2314,8 @@ class Agent: return before def restore_files(self, app, before): + for item in before: + require_public_file(item["path"]) for item in before: managed_write(app, item["path"], None if item["contentBase64"] is None else self.decode_file(item), int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) @@ -2351,6 +2370,7 @@ class Agent: def read_log(self, instance_id, generation, payload): app = self.instance_dir(instance_id, generation) / "app" value = payload["path"] + require_public_file(value) count = min(2000, max(1, int(payload.get("lines", 200)))) # Validate the original relative spelling independently of symlinks. scoped(self.root / "agent" / "log-validation", value) @@ -2367,6 +2387,7 @@ class Agent: raise AgentError("Log symlink resolves outside the managed instance") from None # Reopen the resolved relative target through no-follow directory FDs; # swapping either the original link or a target component cannot escape. + require_public_file(relative) with managed_open(app, relative) as stream: stat = os.fstat(stream.fileno()) if payload.get("mode") == "incremental": diff --git a/static/agent/manifest.json b/static/agent/manifest.json index c8a9848..6e8d4ee 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.9.2", + "version": "1.9.3", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "acff8fdbf75652821638a9c90ee9cc4ef5254bc2f19fabc436b9135f93307a5a", + "sha256": "a0baaab0f0a60e848300e63f31eeefff6293d8b749b5ca9397c36e8e75ff4a93", "updaterProtocolVersion": 2, "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb", "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231" diff --git a/static/agent/manifest.json.sig b/static/agent/manifest.json.sig deleted file mode 100644 index 88dce3b..0000000 --- a/static/agent/manifest.json.sig +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCas2drBsUgAAAAAAEAA5t -YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfqXUwD/UtXrU4iXCqcAGzYY+ETl -JyvOrDoi5txZ3qz4pkwsZZIBAO3nz5BzaX4k3GfvKOYFsBJsfHDa76WeWwuC1h7m -zdoN -=mSzg ------END PGP SIGNATURE----- diff --git a/tests/unit/native-drift-manifest.test.ts b/tests/unit/native-drift-manifest.test.ts index 93f9937..59516fa 100644 --- a/tests/unit/native-drift-manifest.test.ts +++ b/tests/unit/native-drift-manifest.test.ts @@ -110,6 +110,7 @@ async function setup(capable = true) { ); // The runtime owns this generated file; neither scanner should classify it as drift. contents.set('configs/dedicated.cfg', Buffer.from('runtime generated')); + contents.set('configs/electric_cherry_bank.cfg', Buffer.from('set tws_bank_token private')); const backend = await import('../../src/lib/server/backends/native.js'); const request = vi diff --git a/tests/unit/private-bank-drift.test.ts b/tests/unit/private-bank-drift.test.ts new file mode 100644 index 0000000..4378710 --- /dev/null +++ b/tests/unit/private-bank-drift.test.ts @@ -0,0 +1,269 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +let fixture: Fixture; +const servers = ['t5_zombie_theater_1', 't6_zombie_town_1']; +const filename = 'electric_cherry_bank.cfg'; +const secret = Buffer.from('set tws_bank_token "ec_bank_private_regression_secret"\n'); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture(servers); + const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); + await seedServerRegistry(); + const { destOfKind } = await import('../../src/lib/server/config.js'); + for (const server of servers) { + await mkdir(destOfKind(server, 'config'), { recursive: true }); + await writeFile(join(destOfKind(server, 'config'), filename), secret); + } +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); + +async function evidenceIsPrivate() { + const { db } = await import('../../src/lib/server/db/index.js'); + expect(db().prepare('SELECT 1 FROM file_blobs WHERE content=?').all(secret)).toEqual([]); + expect(db().prepare('SELECT 1 FROM deployment_files WHERE rel_path=?').all(filename)).toEqual( + [] + ); +} + +describe.each(servers)('private bank config on %s', (server) => { + it('does not read, preview, stage, or capture the generated file while repairing real drift', async () => { + const drift = await import('../../src/lib/server/fleet/drift.js'); + const { destOfKind } = await import('../../src/lib/server/config.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); + const { applyStagedFilesForOperation } = + await import('../../src/lib/server/fleet/staged-files.js'); + const backend = await import('../../src/lib/server/backends/files.js'); + const reads = vi.spyOn(backend, 'readFile'); + expect(await drift.scanServerDrift(server)).toEqual([]); + expect((await drift.previewDrift([server])).files).toEqual([]); + expect((await drift.prepareDriftCampaign(server)).files).toEqual([]); + expect((await drift.runDriftDeployment({ servers: [server] })).deploymentId).toBeNull(); + const dest = destOfKind(server, 'config'); + await writeFile(join(dest, 'stray.cfg'), 'unwanted'); + // The exemption is for the generated path, not every matching basename. + await mkdir(join(dest, 'nested')); + await writeFile(join(dest, 'nested', filename), 'unwanted nested file'); + for (const name of ['modified.gsc', 'missing.gsc']) { + const artifact = createScript({ + name, + releaseVersion: '1.0.0', + targetRelPath: name, + kind: 'script', + scopeType: 'game', + scopeValue: server.slice(0, 2), + content: Buffer.from('promoted') + }); + db() + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(artifact.version.id, artifact.script.id); + } + await writeFile(join(destOfKind(server, 'script'), 'modified.gsc'), 'modified'); + const plan = await drift.previewDrift([server]); + expect(plan.files.map((file) => file.relPath).sort()).toEqual([ + 'missing.gsc', + 'modified.gsc', + `nested/${filename}`, + 'stray.cfg' + ]); + const result = await drift.runDriftDeployment({ + servers: [server], + expectedFingerprint: plan.fingerprint + }); + const operation = requestRestart(server, { kind: 'campaign', id: result.deploymentId! }); + applyStagedFilesForOperation(operation.id, server); + expect(await drift.scanServerDrift(server)).toEqual([]); + expect(await readFile(join(dest, filename))).toEqual(secret); + expect(reads.mock.calls.some(([path]) => path === join(dest, filename))).toBe(false); + await evidenceIsPrivate(); + }); + + it('filters old prepared bank evidence while retaining legitimate campaign changes', async () => { + const { prepareDriftCampaign, createDriftRebuildCampaign } = + await import('../../src/lib/server/fleet/drift.js'); + const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { destOfKind } = await import('../../src/lib/server/config.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + await writeFile(join(destOfKind(server, 'config'), 'stray.cfg'), 'unwanted'); + const prepared = await prepareDriftCampaign(server); + const privateFile = { + ...prepared.files[0], + before: secret, + file: { ...prepared.files[0].file, relPath: filename } + }; + prepared.files.push(privateFile); + prepared.plan.files.push(privateFile.file); + const operation = beginDeployment(server, 'rebuild', 'watchdog'); + const id = createDriftRebuildCampaign(server, operation, 'old-incident', prepared); + expect( + db().prepare('SELECT rel_path FROM deployment_files WHERE deployment_id=?').all(id) + ).toEqual([{ rel_path: 'stray.cfg' }]); + expect( + createDriftRebuildCampaign(server, operation, 'bank-only', { + ...prepared, + files: [privateFile] + }) + ).toBeNull(); + await evidenceIsPrivate(); + }); + + it('rejects new and persisted private deletions before activation or evidence capture', async () => { + const staged = await import('../../src/lib/server/fleet/staged-files.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); + const { applyActivationFiles } = + await import('../../src/lib/server/provisioning/activation-files.js'); + const { captureRecipePlan } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { destOfKind } = await import('../../src/lib/server/config.js'); + db() + .prepare( + "INSERT INTO deployments(id,action,started_at,triggered_by) VALUES('old','drift',datetime('now'),'ui')" + ) + .run(); + db() + .prepare( + "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('old',?,1,'pending')" + ) + .run(server); + const change = { + sourceKind: 'campaign' as const, + sourceId: 'old', + server, + kind: 'config' as const, + relPath: filename, + content: null, + inventorySource: 'drift:old' + }; + expect(() => staged.queueFileChange(change)).toThrow('Private bank configuration'); + db() + .prepare( + "INSERT INTO staged_file_changes(source_kind,source_id,server,kind,rel_path,inventory_source) VALUES('campaign','old',?,'config',?,'drift:old')" + ) + .run(server, filename); + const operation = requestRestart(server, { kind: 'campaign', id: 'old' }); + expect(() => staged.applyStagedFilesForOperation(operation.id, server)).toThrow( + 'Private bank configuration' + ); + const recipe = captureRecipePlan( + server.startsWith('t6') ? 't6' : 't5', + server.startsWith('t6') ? 'town' : 'theater' + ); + expect(() => applyActivationFiles(recipe, operation.id, true)).toThrow( + 'Private bank configuration' + ); + expect(await readFile(join(destOfKind(server, 'config'), filename))).toEqual(secret); + await evidenceIsPrivate(); + }); +}); + +it('rejects the reserved upload name and redacts legacy receipts before packing their bytes', async () => { + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { packFileJson } = await import('../../src/lib/server/db/file-blobs.js'); + expect(() => + createScript({ + name: 'private', + kind: 'config', + targetRelPath: filename, + scopeType: 'game', + scopeValue: 't5', + content: secret + }) + ).toThrow('reserved'); + const packed = packFileJson({ + before: [{ path: `configs/${filename}`, contentBase64: secret.toString('base64') }] + }); + expect(packed.hashes.size).toBe(0); + expect(JSON.stringify(packed.value)).not.toContain(secret.toString('base64')); + await evidenceIsPrivate(); +}); + +it('blocks legacy private campaign diffs and blob downloads', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { putFileBlob, getFileBlob } = await import('../../src/lib/server/db/file-blobs.js'); + db() + .prepare( + "INSERT INTO deployments(id,action,started_at,triggered_by) VALUES('legacy','drift',datetime('now'),'ui')" + ) + .run(); + const hash = putFileBlob(secret)!; + const file = db() + .prepare( + "INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before_hash) VALUES('legacy',?,'removed',?,?)" + ) + .run(servers[0], filename, hash); + for (const route of [ + await import('../../src/routes/api/campaigns/[id]/file/[fileId]/+server.js'), + await import('../../src/routes/api/deployments/[id]/file/[fileId]/+server.js') + ]) { + await expect( + route.GET({ params: { id: 'legacy', fileId: String(file.lastInsertRowid) } }) + ).rejects.toMatchObject({ status: 404 }); + } + expect(() => getFileBlob(hash)).toThrow('Private file content'); +}); + +it('blocks old host commands and redacts late private read receipts from older agents', async () => { + const hosts = await import('../../src/lib/server/hosts/index.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const host = hosts.createHost({ name: 'Legacy private command host' }); + const credential = hosts.enrollHost(hosts.createEnrollment(host.id).token, {}).credential; + const payload = { + files: [{ path: `configs/${filename}`, contentBase64: secret.toString('base64') }] + }; + expect(() => hosts.queueHostOperation(host.id, 'files.stage', payload)).toThrow('Private bank'); + const stage = hosts.queueHostOperation(host.id, 'files.stage', { files: [] }); + db() + .prepare('UPDATE ec_host_operations SET payload=? WHERE id=?') + .run(JSON.stringify(payload), stage.id); + expect(() => hosts.getHostOperationPayload(host.id, stage.id)).toThrow('Private bank'); + const healthyRead = hosts.queueHostOperation(host.id, 'files.read', { + path: 'configs/stray.cfg' + }); + expect(hosts.pendingHostOperationRows(host.id).map((operation) => operation.id)).toContain( + healthyRead.id + ); + expect(hosts.pendingHostOperationRows(host.id).map((operation) => operation.id)).not.toContain( + stage.id + ); + const read = hosts.queueHostOperation(host.id, 'files.read', { path: 'configs/stray.cfg' }); + db() + .prepare('UPDATE ec_host_operations SET payload=?,target_path=? WHERE id=?') + .run(JSON.stringify({ path: `configs/${filename}` }), `configs/${filename}`, read.id); + hosts.reportHostOperationStatus(host.id, read.id, { + state: 'completed', + result: { contentBase64: secret.toString('base64') } + }); + expect(hosts.getHostOperation(read.id)?.result).toEqual({ redacted: true }); + await evidenceIsPrivate(); + + // Even an old operation reference cannot authorize downloading captured credentials. + const { putFileBlob, referenceBlobs } = await import('../../src/lib/server/db/file-blobs.js'); + const hash = putFileBlob(secret)!; + referenceBlobs('operation', stage.id, [hash]); + db() + .prepare( + "INSERT INTO deployments(id,action,started_at,triggered_by) VALUES('legacy','drift',datetime('now'),'ui')" + ) + .run(); + db() + .prepare( + "INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before_hash) VALUES('legacy',?,'removed',?,?)" + ) + .run(servers[0], filename, hash); + const { GET } = await import('../../src/routes/api/agent/blobs/[sha256]/+server.js'); + expect(() => + GET({ + request: new Request('https://test', { + headers: { Authorization: `Bearer ${credential}` } + }), + params: { sha256: hash } + }) + ).toThrow(); +}); diff --git a/tests/unit/watchdog.test.ts b/tests/unit/watchdog.test.ts index 82aebdc..8c7d4fe 100644 --- a/tests/unit/watchdog.test.ts +++ b/tests/unit/watchdog.test.ts @@ -122,6 +122,55 @@ async function sustainedFailure() { } describe('Watchdog detection and ownership', () => { + it.each([false, true])( + 'rescans legacy bank drift and preserves real differences (mixed=%s)', + async (mixed) => { + const { mkdir } = await import('node:fs/promises'); + const { destOfKind } = await import('../../src/lib/server/config.js'); + const { setMaintenancePolicy, pauseMaintenance } = + await import('../../src/lib/server/maintenance/policies.js'); + setMaintenancePolicy('drift', false); + pauseMaintenance(true); + const dest = destOfKind(server, 'config'); + await mkdir(dest, { recursive: true }); + const secret = 'set tws_bank_token "ec_bank_watchdog_private"'; + await writeFile(join(dest, 'electric_cherry_bank.cfg'), secret); + if (mixed) await writeFile(join(dest, 'stray.cfg'), 'real drift'); + database + .db() + .prepare( + `INSERT INTO watchdog_incidents + (id,server,kind,state,summary,details,first_seen_at,last_seen_at,failures) + VALUES('legacy-bank',?,'drift','open','Old drift',?,?,?,1)` + ) + .run( + server, + JSON.stringify({ + files: [{ kind: 'config', relPath: 'electric_cherry_bank.cfg' }] + }), + new Date().toISOString(), + new Date().toISOString() + ); + await watchdog.watchdogTick(true); + const row = database + .db() + .prepare('SELECT state,details FROM watchdog_incidents WHERE id=?') + .get('legacy-bank') as { state: string; details: string }; + expect(row.state).toBe(mixed ? 'open' : 'resolved'); + expect( + JSON.parse(row.details).files.map((file: { relPath: string }) => file.relPath) + ).toEqual(mixed ? ['stray.cfg'] : []); + expect(operations()).toEqual([]); + expect( + database + .db() + .prepare('SELECT 1 FROM file_blobs WHERE instr(content,?)>0') + .all(Buffer.from(secret)) + ).toEqual([]); + expect(await readFile(join(dest, 'electric_cherry_bank.cfg'), 'utf8')).toBe(secret); + } + ); + it('waits for status-monitor evidence without issuing its own routine probes', async () => { down(); for (let minute = 0; minute < 4; minute++) {