diff --git a/README.md b/README.md index 3dd1b87..7eabc58 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,13 @@ stopping games; activation uses the same occupancy and concurrency gates as lifecycle operations. Agent before images preserve exact rollback bytes and operation history. +Artifact bodies are stored once by SHA-256 in SQLite. Versions, staging plans, +rollback snapshots and launch commands reference those shared bytes. The first +startup after this upgrade backfills existing rows transactionally; preserve the +data-directory backup when upgrading. Current agents fetch assigned artifacts +from an authenticated endpoint and verify the hash. Older agents continue to +receive inline content until their automatic update completes. + T5 Zombies uses `t5sp` and `dedicated_sp.cfg`; T6 uses `t6zm` and `dedicated_zm.cfg`. Map selection generates `electric_cherry_map.cfg`, loaded after the main config. T6 variants retain their distinct catalog selections. diff --git a/agent/ec_agent.py b/agent/ec_agent.py index 22f2a18..9c98576 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -33,7 +33,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.5.3" +VERSION = "1.6.0" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1356,7 +1356,7 @@ class Agent: if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() - capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) + capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation): @@ -1559,6 +1559,18 @@ class Agent: managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) def decode_file(self, item): + if "contentSha256" in item: + sha = item["contentSha256"] + if not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha): + raise AgentError("Invalid file content hash") + url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha + with self.artifact_open("GET", url) as response: + content = response.read(MAX_FILE + 1) + if len(content) > MAX_FILE: + raise AgentError("File exceeds staging size limit") + if digest(content) != sha or item.get("sha256", sha) != sha: + raise AgentError("Staged file checksum mismatch") + return content try: content = base64.b64decode(item["contentBase64"], validate=True) except (ValueError, KeyError, TypeError): @@ -1677,7 +1689,7 @@ class Agent: if item["path"] in paths: raise AgentError("Duplicate staged file path") paths.add(item["path"]) - deleting = item.get("delete") is True or item.get("contentBase64") is None + deleting = item.get("delete") is True or (item.get("contentBase64") is None and "contentSha256" not in item) content = None if deleting else self.decode_file(item) total_size += len(content) if content is not None else 0 if total_size > 64 * 1024 * 1024: diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 06a483f..4f4124b 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -832,6 +832,26 @@ class AgentTests(unittest.TestCase): self.assertEqual(result["state"], "completed", result) return payload + def test_hash_file_downloads_are_verified_and_staged_as_writes(self): + self.create() + content = b"downloaded plugin" + sha = module.digest(content) + file = {"id": 1, "path": "configs/server.cfg", "contentSha256": sha, "sha256": sha} + with patch.object(self.agent, "artifact_open", return_value=Response(content)) as fetch: + result = self.agent.stage(INSTANCE, 1, {"stageId": "hash-stage", "files": [file]}, "hash-operation") + self.assertEqual(result["state"], "staged") + self.assertTrue(fetch.call_args.args[1].endswith("/api/agent/blobs/" + sha)) + manifest = json.loads((self.agent.instance_dir(INSTANCE, 1) / "stages/hash-stage/manifest.json").read_text()) + self.assertFalse(manifest["files"][0]["delete"]) + self.assertEqual(base64.b64decode(manifest["files"][0]["contentBase64"]), content) + with patch.object(self.agent, "artifact_open", return_value=Response(b"wrong bytes")): + with self.assertRaisesRegex(AgentError, "checksum mismatch"): + self.agent.decode_file(file) + with patch.object(self.agent, "artifact_open") as fetch: + with self.assertRaisesRegex(AgentError, "Invalid file content hash"): + self.agent.decode_file({"contentSha256": "../../secret"}) + fetch.assert_not_called() + def test_same_id_reports_saved_result_without_executing_again(self): self.create() operation = self.operation("instance.restart") diff --git a/src/lib/server/db/file-blobs.ts b/src/lib/server/db/file-blobs.ts new file mode 100644 index 0000000..2116474 --- /dev/null +++ b/src/lib/server/db/file-blobs.ts @@ -0,0 +1,123 @@ +import { createHash } from 'node:crypto'; +import type Database from 'better-sqlite3'; +import { db } from './index.js'; + +export function ensureFileBlobSchema(handle: Database.Database): void { + handle.exec(`CREATE TABLE IF NOT EXISTS file_blobs(sha256 TEXT PRIMARY KEY,size INTEGER NOT NULL,content BLOB NOT NULL); + CREATE TABLE IF NOT EXISTS file_blob_references(owner_type TEXT NOT NULL,owner_id TEXT NOT NULL, + sha256 TEXT NOT NULL REFERENCES file_blobs(sha256),PRIMARY KEY(owner_type,owner_id,sha256)); + CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256);`); +} + +export function putFileBlob(content: Uint8Array | null, handle: Database.Database = db()): string | null { + if (content === null) return null; + const sha256 = createHash('sha256').update(content).digest('hex'); + handle.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)').run(sha256, content.byteLength, content); + return sha256; +} + +export function getFileBlob(sha256: string, handle: Database.Database = db()): Buffer { + const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as { content: Buffer } | undefined; + if (!row) throw new Error('File content is unavailable or has expired.'); + return row.content; +} + +/** SQL fragments are for fixed schema identifiers, never user input. */ +export const blobContent = (column: string) => `(SELECT content FROM file_blobs WHERE sha256=${column})`; +export const blobSize = (column: string) => `(SELECT size FROM file_blobs WHERE sha256=${column})`; + +export function referenceBlobs(ownerType: string, ownerId: string, hashes: Iterable, handle: Database.Database = db()): void { + handle.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?').run(ownerType, ownerId); + const insert = handle.prepare('INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)'); + for (const hash of hashes) insert.run(ownerType, ownerId, hash); +} + +/** Immutable file bytes in JSON are replaced with hash references, including before images. */ +export function packFileJson(value: unknown, handle: Database.Database = db(), hashes = new Set()): { value: unknown; hashes: Set } { + if (Array.isArray(value)) return { value: value.map(item => packFileJson(item, handle, hashes).value), hashes }; + if (!value || typeof value !== 'object') return { value, hashes }; + const result: Record = {}; + for (const [key, item] of Object.entries(value)) { + if ((key === 'contentBase64' || key === 'content' && 'path' in value && 'sha256' in value) && typeof item === 'string') { + const hash = putFileBlob(Buffer.from(item, 'base64'), handle)!; + result[key === 'content' ? 'contentHash' : 'contentSha256'] = hash; + hashes.add(hash); + } else { + result[key] = packFileJson(item, handle, hashes).value; + if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') hashes.add(item); + } + } + return { value: result, hashes }; +} + +export function unpackFileJson(value: unknown, handle: Database.Database = db()): any { + if (Array.isArray(value)) return value.map(item => unpackFileJson(item, handle)); + if (!value || typeof value !== 'object') return value; + const result: Record = {}; + for (const [key, item] of Object.entries(value)) { + if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') { + result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(item, handle).toString('base64'); + } else result[key] = unpackFileJson(item, handle); + } + return result; +} + +export const blobColumns = [ + ['script_versions', 'content'], ['staged_file_changes', 'content'], + ['deployment_files', 'content_before'], ['deployment_files', 'content_after'], + ['beta_file_snapshots', 'prior_content'], ['beta_revision_files', 'content'] +] as const; + +/** Finish each SELECT before writing; better-sqlite3 iterators hold an active cursor. */ +export function* migrationRows(handle: Database.Database, sql: string): Generator { + const select = handle.prepare(sql); + let cursor = -Number.MAX_SAFE_INTEGER; + for (;;) { + const row = select.get(cursor) as (T & { rid: number }) | undefined; + if (!row) return; + cursor = row.rid; + yield row; + } +} + +/** Backfill one row at a time so large libraries do not exhaust startup memory. */ +export function migrateFileBlobs(handle: Database.Database): void { + for (const [table, column] of blobColumns) { + if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue; + handle.transaction(() => { + handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`); + const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`); + for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) { + update.run(putFileBlob(row.content, handle), row.rid); + } + handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`); + })(); + } + for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`); + if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) { + handle.transaction(() => { + for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle, + 'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) { + const packed = packFileJson(JSON.parse(row.recipe), handle); + handle.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(packed.value), row.deployment_id); + referenceBlobs('recipe', row.deployment_id, packed.hashes, handle); + } + handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')"); + })(); + } + handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN + DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`); + for (const action of ['INSERT', 'UPDATE OF recipe']) handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'} + AFTER ${action} ON server_launch_recipes BEGIN + DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=NEW.deployment_id; + INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) + SELECT 'recipe',NEW.deployment_id,json_extract(value,'$.contentHash') FROM json_each(NEW.recipe,'$.files') + WHERE json_extract(value,'$.contentHash') IS NOT NULL; END;`); +} + +export function pruneFileBlobs(limit = 100): number { + return db().prepare(`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b + WHERE NOT EXISTS (SELECT 1 FROM file_blob_references r WHERE r.sha256=b.sha256) + ${blobColumns.map(([table, column]) => `AND NOT EXISTS (SELECT 1 FROM ${table} WHERE ${column}_hash=b.sha256)`).join('\n')} + LIMIT ?)`).run(limit).changes; +} diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index fcb10b6..ddfd252 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -12,6 +12,7 @@ import { migrateServerHistory } from './server-history.js'; import { migrateNativeInstances } from '../backends/schema.js'; import { installPerformanceIndexes } from './performance-indexes.js'; import { migrateVersionLayout } from './version-layout.js'; +import { migrateFileBlobs } from './file-blobs.js'; import { tuneConnection } from './connection.js'; import { timeStatement } from './slow-statements.js'; @@ -393,6 +394,7 @@ export function db(): Database.Database { initializeFleetSettings(handle); migrateNativeInstances(handle); migrateVersionLayout(handle); + migrateFileBlobs(handle); // Lifecycle history is retained indefinitely. Admission and status reads // must narrow by server or active state instead of scanning that history. // Install after migrations that rebuild this table or add server_deleted_at. diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 46c81b4..63f0084 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -1,3 +1,13 @@ +-- Immutable file bodies are shared across versions, campaigns, betas and host commands. +CREATE TABLE IF NOT EXISTS file_blobs ( + sha256 TEXT PRIMARY KEY, size INTEGER NOT NULL, content BLOB NOT NULL +); +CREATE TABLE IF NOT EXISTS file_blob_references ( + owner_type TEXT NOT NULL, owner_id TEXT NOT NULL, sha256 TEXT NOT NULL REFERENCES file_blobs(sha256), + PRIMARY KEY(owner_type,owner_id,sha256) +); +CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256); + -- Electric Cherry schema. Idempotent — safe to run at every boot via migrate.ts. -- The old betas/beta_scripts/beta_deploys tables are dropped in-code before this -- runs (see dropLegacyBetaTables in db/index.ts); the new script-centric model @@ -134,7 +144,7 @@ CREATE TABLE IF NOT EXISTS script_versions ( state TEXT NOT NULL DEFAULT 'untested', -- 'untested' | 'tested' | 'test-failed' kind TEXT NOT NULL DEFAULT 'script', -- denormalized from scripts.kind for join-free filtering uploaded_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - content BLOB NOT NULL, + content_hash TEXT REFERENCES file_blobs(sha256), UNIQUE(script_id, version) ); CREATE INDEX IF NOT EXISTS idx_script_versions_script ON script_versions(script_id); @@ -201,7 +211,7 @@ CREATE TABLE IF NOT EXISTS beta_revision_files ( version INTEGER NOT NULL, release_version TEXT, sha256 TEXT NOT NULL, - content BLOB NOT NULL, + content_hash TEXT REFERENCES file_blobs(sha256), PRIMARY KEY (beta_id, revision, script_id), FOREIGN KEY (beta_id, revision) REFERENCES beta_revisions(beta_id, revision) ON DELETE CASCADE ); @@ -306,7 +316,7 @@ CREATE TABLE IF NOT EXISTS beta_file_snapshots ( beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, server TEXT NOT NULL REFERENCES servers(name) ON DELETE CASCADE, rel_path TEXT NOT NULL, - prior_content BLOB NOT NULL, + prior_content_hash TEXT REFERENCES file_blobs(sha256), prior_sha256 TEXT NOT NULL, prior_size INTEGER NOT NULL, PRIMARY KEY (beta_id, server, rel_path) @@ -364,8 +374,8 @@ CREATE TABLE IF NOT EXISTS deployment_files ( rel_path TEXT NOT NULL, sha256_before TEXT, sha256_after TEXT, - content_before BLOB, - content_after BLOB, + content_before_hash TEXT REFERENCES file_blobs(sha256), + content_after_hash TEXT REFERENCES file_blobs(sha256), -- An unknown before image is distinct from a confirmed absent file. snapshot_pending INTEGER NOT NULL DEFAULT 0 ); @@ -382,7 +392,7 @@ CREATE TABLE IF NOT EXISTS staged_file_changes ( server TEXT NOT NULL, kind TEXT NOT NULL, rel_path TEXT NOT NULL, - content BLOB, + content_hash TEXT REFERENCES file_blobs(sha256), script_version_id INTEGER, inventory_source TEXT NOT NULL, deployment_file_id INTEGER REFERENCES deployment_files(id) ON DELETE CASCADE, diff --git a/src/lib/server/db/version-layout.ts b/src/lib/server/db/version-layout.ts index cbfb5aa..2a05161 100644 --- a/src/lib/server/db/version-layout.ts +++ b/src/lib/server/db/version-layout.ts @@ -3,7 +3,7 @@ import type Database from 'better-sqlite3'; /** Keep metadata ahead of overflow pages, preserving IDs and all references. */ export function migrateVersionLayout(handle: Database.Database): void { const columns = handle.pragma('table_info(script_versions)') as { name: string }[]; - if (columns.at(-1)?.name === 'content') return; + if (!columns.some(column => column.name === 'content') || columns.at(-1)?.name === 'content') return; const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'").get() as { sql: string }; const content = /\bcontent\s+BLOB\s+NOT\s+NULL\s*,/i; if (!content.test(sql)) throw new Error('Unrecognized script version content layout'); diff --git a/src/lib/server/fleet/beta-revisions.ts b/src/lib/server/fleet/beta-revisions.ts index a18244c..349eb95 100644 --- a/src/lib/server/fleet/beta-revisions.ts +++ b/src/lib/server/fleet/beta-revisions.ts @@ -12,8 +12,8 @@ export function archiveBetaRevision(id: number, revision: number, startedAt: str db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)') .run(id, revision, startedAt, notes); db().prepare(`INSERT OR IGNORE INTO beta_revision_files - (beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content) - SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content + (beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content_hash) + SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content_hash FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id WHERE bs.beta_id = ?`).run(revision, id); } diff --git a/src/lib/server/fleet/beta.ts b/src/lib/server/fleet/beta.ts index afd56b4..592eb96 100644 --- a/src/lib/server/fleet/beta.ts +++ b/src/lib/server/fleet/beta.ts @@ -1,3 +1,4 @@ +import { putFileBlob, blobContent } from '../db/file-blobs.js'; import { getVersionContent } from './scripts.js'; import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; import { storedPlan, testView, activity, setOutcome } from './beta-tests.js'; @@ -734,7 +735,7 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual const hasCollapsing = bundle.some((b) => isCollapsingKind(b.script.kind)); const getSnapshot = db().prepare( - `SELECT prior_content, prior_sha256, prior_size FROM beta_file_snapshots + `SELECT ${blobContent('prior_content_hash')} AS prior_content, prior_sha256, prior_size FROM beta_file_snapshots WHERE beta_id = ? AND server = ? AND rel_path = ?` ); @@ -797,10 +798,10 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual } for (const change of restorations) { const file = db().prepare(`INSERT INTO deployment_files - (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) + (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash) VALUES (?, ?, ?, ?, NULL, ?, NULL, ?)`).run(deploymentId, change.server, change.content === null ? 'removed' : 'updated', change.relPath, - change.content === null ? null : sha256Buffer(change.content), change.content); + change.content === null ? null : sha256Buffer(change.content), putFileBlob(change.content)); queueFileChange({ sourceKind: 'campaign', sourceId: deploymentId, server: change.server, kind: change.kind, relPath: change.relPath, content: change.content, scriptVersionId: change.versionId, inventorySource: `rollback:${betaId}`, @@ -969,7 +970,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom const plans: Plan[] = []; const getSnapshotForPromote = db().prepare( - `SELECT prior_content, prior_sha256 FROM beta_file_snapshots + `SELECT ${blobContent('prior_content_hash')} AS prior_content, prior_sha256 FROM beta_file_snapshots WHERE beta_id = ? AND server = ? AND rel_path = ?` ); @@ -1096,7 +1097,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom ); const insertDepFile = db().prepare( `INSERT INTO deployment_files - (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending) + (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash, snapshot_pending) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` ); @@ -1121,8 +1122,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom p.script.target_rel_path, p.priorSha, p.version.sha256, - p.priorContent, - getVersionContent(p.version.id), + putFileBlob(p.priorContent), + putFileBlob(getVersionContent(p.version.id)), betaServerSet.has(p.server) ? 0 : 1 ); if (plansToActivate.has(p)) { @@ -1141,8 +1142,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom configFileOf(cp.server), cp.priorSha, cp.composedSha, - cp.priorContent, - cp.composedContent, + putFileBlob(cp.priorContent), + putFileBlob(cp.composedContent), betaServerSet.has(cp.server) ? 0 : 1 ); if (cp.requiresActivation) { diff --git a/src/lib/server/fleet/drift.ts b/src/lib/server/fleet/drift.ts index a4b57c7..6f18774 100644 --- a/src/lib/server/fleet/drift.ts +++ b/src/lib/server/fleet/drift.ts @@ -1,3 +1,4 @@ +import { putFileBlob } from '../db/file-blobs.js'; import { getVersionContent } from './scripts.js'; import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js'; import { randomUUID } from 'node:crypto'; @@ -401,11 +402,11 @@ export function createDriftRebuildCampaign( db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')") .run(id, new Date().toISOString()); const insertFile = db().prepare(`INSERT INTO deployment_files - (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) + (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`); for (const { file, before, after } of prepared.files) { insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated', - file.relPath, file.beforeSha256, file.afterSha256, before, after); + file.relPath, file.beforeSha256, file.afterSha256, putFileBlob(before), putFileBlob(after)); } db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id) VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId); @@ -480,7 +481,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise { - const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, content, null); + const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, putFileBlob(content), null); queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server, kind: entry.kind, relPath: entry.relPath, content: null, inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid), @@ -538,7 +539,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise { const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath, - before ? sha256Buffer(before) : null, sha256Buffer(written), before, written); + before ? sha256Buffer(before) : null, sha256Buffer(written), putFileBlob(before), putFileBlob(written)); queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server, kind: entry.kind, relPath: entry.relPath, content: written, scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null, diff --git a/src/lib/server/fleet/launch-recipes.ts b/src/lib/server/fleet/launch-recipes.ts index 34a797c..2e713d5 100644 --- a/src/lib/server/fleet/launch-recipes.ts +++ b/src/lib/server/fleet/launch-recipes.ts @@ -1,3 +1,4 @@ +import { packFileJson, unpackFileJson } from '../db/file-blobs.js'; import { randomUUID } from 'node:crypto'; import type { BlueprintSummary, LaunchRecipeSummary, LaunchFileSummary } from '$lib/deployment-plan.js'; import { config, configFileForGame } from '../config.js'; @@ -118,7 +119,7 @@ export function summarizeRecipe(recipe: LaunchRecipePlan, fingerprint = sha256Bu /** Keep the small detail response beside the immutable attempted file bytes. */ export function recipeStorage(recipe: LaunchRecipePlan): { recipe: string; recipeSummary: string } { const encoded = JSON.stringify(recipe); - return { recipe: encoded, recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) }; + return { recipe: JSON.stringify(packFileJson(recipe).value), recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) }; } export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null { @@ -137,7 +138,7 @@ export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSu export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null { const row = db().prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?') .get(id) as { recipe: string; blueprint_name: string | null } | undefined; - return row ? { recipe: JSON.parse(row.recipe), blueprintName: row.blueprint_name } : null; + return row ? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name } : null; } export function blueprintRecipe(id: string): { recipe: LaunchRecipePlan; blueprintName: string } { diff --git a/src/lib/server/fleet/rollback.ts b/src/lib/server/fleet/rollback.ts index b6cbcfd..cf7cb0f 100644 --- a/src/lib/server/fleet/rollback.ts +++ b/src/lib/server/fleet/rollback.ts @@ -1,3 +1,4 @@ +import { putFileBlob, blobContent } from '../db/file-blobs.js'; import type { ArtifactKind } from '../config.js'; import { db } from '../db/index.js'; import { emit } from '../pipeline/bus.js'; @@ -101,7 +102,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise { const s = insertScript.run(input.name, targetRelPath, input.scopeType ?? null, input.scopeValue ?? null, kind); scriptId = Number(s.lastInsertRowid); - const v = insertVersion.run(scriptId, sha, size, input.content, input.notes ?? null, kind, releaseVersion); + const v = insertVersion.run(scriptId, sha, size, putFileBlob(input.content), input.notes ?? null, kind, releaseVersion); versionId = Number(v.lastInsertRowid); })(); @@ -281,10 +282,10 @@ export function uploadNewVersion(scriptId: number, content: Buffer, notes?: stri const res = db() .prepare( - `INSERT INTO script_versions (script_id, version, sha256, size, content, notes, kind, release_version) + `INSERT INTO script_versions (script_id, version, sha256, size, content_hash, notes, kind, release_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` ) - .run(scriptId, nextVersion, sha, content.byteLength, content, notes ?? null, script.kind, release); + .run(scriptId, nextVersion, sha, content.byteLength, putFileBlob(content), notes ?? null, script.kind, release); return getVersion(Number(res.lastInsertRowid))!; } diff --git a/src/lib/server/fleet/staged-files.ts b/src/lib/server/fleet/staged-files.ts index e3ce1c1..e1086e7 100644 --- a/src/lib/server/fleet/staged-files.ts +++ b/src/lib/server/fleet/staged-files.ts @@ -1,3 +1,4 @@ +import { putFileBlob, blobContent } from '../db/file-blobs.js'; import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; import { dirname, relative, resolve, isAbsolute } from 'node:path'; import { destOfKind, serverDir, type ArtifactKind } from '../config.js'; @@ -39,22 +40,22 @@ function targetPath(server: string, kind: ArtifactKind, relPath: string): string /** Capture immutable desired bytes without changing the server or its inventory. */ export function queueFileChange(change: FileChange): void { targetPath(change.server, change.kind, change.relPath); - db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content, + db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content_hash, script_version_id, inventory_source, deployment_file_id, preserve_before_image, check_before, expected_before_sha) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(source_kind, source_id, server, kind, rel_path) DO UPDATE SET - content = excluded.content, script_version_id = excluded.script_version_id, + content_hash = excluded.content_hash, script_version_id = excluded.script_version_id, inventory_source = excluded.inventory_source, deployment_file_id = excluded.deployment_file_id, preserve_before_image = excluded.preserve_before_image, check_before = excluded.check_before, expected_before_sha = excluded.expected_before_sha WHERE staged_file_changes.state = 'pending'`) - .run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, change.content, + .run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, putFileBlob(change.content), change.scriptVersionId ?? null, change.inventorySource, change.deploymentFileId ?? null, change.preserveBeforeImage ? 1 : 0, change.expectedBeforeSha !== undefined ? 1 : 0, change.expectedBeforeSha ?? null); } export function stagedFileChanges(sourceKind: SourceKind, sourceId: string, server?: string): StagedFileChange[] { - return db().prepare(`SELECT * FROM staged_file_changes WHERE source_kind = ? AND source_id = ? + return db().prepare(`SELECT *, ${blobContent('content_hash')} AS content FROM staged_file_changes WHERE source_kind = ? AND source_id = ? ${server === undefined ? '' : 'AND server = ?'} ORDER BY id`) .all(sourceKind, sourceId, ...(server === undefined ? [] : [server])) as StagedFileChange[]; } @@ -81,13 +82,13 @@ function readExisting(path: string): Buffer | null { } /** The first activation owns the rollback image, including an absent file. */ -export function captureBetaBeforeImage(row: StagedFileChange, before: Buffer | null): void { +export function captureBetaBeforeImage(row: Pick, before: Buffer | null): void { if (db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?') .get(row.source_id, row.server, row.rel_path)) return; const prior = getServerScript(row.server, row.rel_path); if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots - (beta_id, server, rel_path, prior_content, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`) - .run(row.source_id, row.server, row.rel_path, before, sha256Buffer(before), before.length); + (beta_id, server, rel_path, prior_content_hash, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`) + .run(row.source_id, row.server, row.rel_path, putFileBlob(before), sha256Buffer(before), before.length); db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ? AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`) .run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, row.server, row.rel_path, row.kind); @@ -100,7 +101,7 @@ export function captureBetaBeforeImage(row: StagedFileChange, before: Buffer | n * from interleaving between the final gates and dispatching the restart. */ export function applyStagedFilesForOperation(operationId: string, server: string): void { - const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s + const rows = db().prepare(`SELECT f.*, ${blobContent('f.content_hash')} AS content FROM staged_file_changes f JOIN provisioning_sources s ON s.kind = f.source_kind AND s.source_id = f.source_id AND s.server = f.server WHERE s.operation_id = ? AND f.server = ? AND f.state IN ('pending','applying') ORDER BY f.id`) .all(operationId, server) as StagedFileChange[]; @@ -132,8 +133,8 @@ export function applyStagedFilesForOperation(operationId: string, server: string } if (row.deployment_file_id && !row.preserve_before_image) { db().prepare(`UPDATE deployment_files SET - verb = ?, content_before = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`) - .run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before, + verb = ?, content_before_hash = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`) + .run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', putFileBlob(before), before === null ? null : sha256Buffer(before), row.deployment_file_id); if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') .run(row.source_id, `${server}: captured rollback snapshot for ${row.rel_path}`); diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index ee12437..880fe98 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -1,3 +1,4 @@ +import { packFileJson, unpackFileJson, referenceBlobs } from '../db/file-blobs.js'; import { pruneHistory } from '../retention.js'; import { recordAgentReceipt } from '../provisioning/agent-events.js'; import { EventEmitter } from 'node:events'; @@ -396,7 +397,7 @@ export type HostOperationStatus = Omit; const operationStatusColumns = 'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at'; const operationPayloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'; function operationStatusFromRow(row: Omit): HostOperationStatus { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; } -function operationFromRow(row: OperationRow): HostOperation { return { ...operationStatusFromRow(row), payload: JSON.parse(row.payload) }; } +function operationFromRow(row: OperationRow): HostOperation { const status = operationStatusFromRow(row); return { ...status, result: unpackFileJson(status.result), payload: unpackFileJson(JSON.parse(row.payload)) }; } export function getHostOperation(id: string): HostOperation | null { const row = database().prepare(`SELECT ${operationStatusColumns},${operationPayloadSql} AS payload FROM ec_host_operations o WHERE o.id=?`) .get(id) as OperationRow | undefined; @@ -474,20 +475,24 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu FROM ec_host_operations o WHERE o.host_id=? AND o.idempotency_key=?`).get(payload, hostId, options.idempotencyKey) as { id: string; kind: string; instance_id: string | null; generation: number | null; history_expired_at: string | null; request_hash: string | null; same_payload: number } | undefined; if (existing) { - const samePayload = existing.history_expired_at ? existing.request_hash === createHash('sha256').update(payload).digest('hex') : !!existing.same_payload; + const samePayload = existing.request_hash ? existing.request_hash === createHash('sha256').update(payload).digest('hex') : !!existing.same_payload; if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || !samePayload) throw new Error('Operation idempotency key is already associated with a different request.'); return getHostOperationStatus(existing.id)!; } } if (['instance.create','create'].includes(kind)) requireReadyHost(hostId); if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.'); + const requestHash = createHash('sha256').update(payload).digest('hex'); + const packed = packFileJson(JSON.parse(payload.toString())); + payload = JSON.stringify(packed.value); const id = options.id ?? randomUUID(); const timestamp = now(); const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?') .get(options.instanceId) as { server: string } | undefined)?.server ?? null : null; - database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name) - VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?)`) + database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name,request_hash) + VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?,?)`) .run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, - routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName); + routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName, requestHash); + referenceBlobs('operation', id, packed.hashes); if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload); database().prepare(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END, target_path=${operationTargetSql} WHERE o.id=?`).run(id); @@ -562,7 +567,8 @@ export function getHostOperationPayload(hostId: string, id: string): Buffer { const row = database().prepare(`SELECT CAST(${operationPayloadSql} AS BLOB) AS payload FROM ec_host_operations o WHERE o.id=? AND o.host_id=?`) .get(id, hostId) as { payload: Buffer } | undefined; if (!row) throw new Error('Host operation was not found for this host.'); - return row.payload; + if (!row.payload.includes('"contentSha256"') || getHost(hostId)?.inventory.capabilities?.includes('files.blobs')) return row.payload; + return Buffer.from(JSON.stringify(unpackFileJson(JSON.parse(row.payload.toString())))); } export function reportHostOperationStatus(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperationStatus { const operation = database().transaction(() => { @@ -575,8 +581,13 @@ export function reportHostOperationStatus(hostId: string, id: string, input: { s const state = input.state as HostOperationState; if (operation.state === 'running' && state === 'accepted') return operation; const error = input.error === undefined || input.error === null ? null : String(input.error).slice(0, 4000); - const result = input.result === undefined ? null : JSON.stringify(input.result); + let result = input.result === undefined ? null : JSON.stringify(input.result); if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.'); + if (result !== null) { + const packed = packFileJson(input.result); + result = JSON.stringify(packed.value); + referenceBlobs('operation-result', id, packed.hashes); + } const observation = state === 'completed' && !error && successfulObservation(operation.kind, operation.kind === 'rcon.execute' ? JSON.parse(getHostOperationPayload(hostId, id).toString('utf8')) : {}, input.result); database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=?,observation_success=? WHERE id=?') diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts index 6bfcca3..985267b 100644 --- a/src/lib/server/hosts/schema.ts +++ b/src/lib/server/hosts/schema.ts @@ -1,3 +1,5 @@ +import { createHash } from 'node:crypto'; +import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js'; import type Database from 'better-sqlite3'; import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js'; @@ -6,6 +8,7 @@ export function ensureHostSchema(handle: Database.Database): void { if (initialized.has(handle)) return; // Background readers open only after controller migrations have completed. if (handle.readonly) { initialized.add(handle); return; } + ensureFileBlobSchema(handle); handle.exec(` CREATE TABLE IF NOT EXISTS ec_hosts ( id TEXT PRIMARY KEY, name TEXT NOT NULL, hostname TEXT NOT NULL DEFAULT '', @@ -150,5 +153,37 @@ export function ensureHostSchema(handle: Database.Database): void { handle.exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)'); })(); } + if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('operation-file-blobs')) { + handle.transaction(() => { + for (const row of migrationRows<{ id: string; payload: string; result: string | null }>(handle, + `SELECT o.rowid AS rid,o.id,COALESCE(p.payload,o.payload) payload,o.result + FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id + WHERE o.rowid>? AND o.kind IN ('files.stage','instance.create','instance.rebuild','instance.restart','instance.start','files.read') + ORDER BY o.rowid LIMIT 1`)) { + const packed = packFileJson(JSON.parse(row.payload), handle); + handle.prepare('UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?) WHERE id=?') + .run(createHash('sha256').update(row.payload).digest('hex'), row.id); + if (packed.hashes.size) { + handle.prepare(`INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?) + ON CONFLICT(operation_id) DO UPDATE SET payload=excluded.payload`).run(row.id, JSON.stringify(packed.value)); + handle.prepare("UPDATE ec_host_operations SET payload=json_remove(payload,'$.files') WHERE id=?").run(row.id); + referenceBlobs('operation', row.id, packed.hashes, handle); + } + if (row.result) { + const result = packFileJson(JSON.parse(row.result), handle); + if (result.hashes.size) { + handle.prepare('UPDATE ec_host_operations SET result=? WHERE id=?').run(JSON.stringify(result.value), row.id); + referenceBlobs('operation-result', row.id, result.hashes, handle); + } + } + } + handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('operation-file-blobs'); + })(); + } + handle.exec(`CREATE TRIGGER IF NOT EXISTS release_operation_blobs AFTER DELETE ON ec_host_operations BEGIN + DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=OLD.id; END; + CREATE TRIGGER IF NOT EXISTS expire_operation_blobs AFTER UPDATE OF history_expired_at ON ec_host_operations + WHEN NEW.history_expired_at IS NOT NULL BEGIN + DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=NEW.id; END;`); initialized.add(handle); } diff --git a/src/lib/server/provisioning/launch-preparation.ts b/src/lib/server/provisioning/launch-preparation.ts index 6e999dc..7f73ff9 100644 --- a/src/lib/server/provisioning/launch-preparation.ts +++ b/src/lib/server/provisioning/launch-preparation.ts @@ -20,6 +20,7 @@ export interface NativeLaunchInput { export interface PreparedNativeLaunch { recipe: LaunchRecipePlan; recipeSummary: string; + blobs?: Array<{ sha256: string; content: Uint8Array }>; recipeJson: Uint8Array; payloadJson: Uint8Array; } diff --git a/src/lib/server/provisioning/launch-worker.ts b/src/lib/server/provisioning/launch-worker.ts index 5216073..0eeaef5 100644 --- a/src/lib/server/provisioning/launch-worker.ts +++ b/src/lib/server/provisioning/launch-worker.ts @@ -29,17 +29,20 @@ parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeL PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: input.serverKey }, files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), - contentBase64: file.content, sha256: file.sha256 })), + contentSha256: file.sha256, sha256: file.sha256 })), ...(input.rebuild ? { nextGeneration: instance.generation + 1, ...(input.safetyOverride ? { safetyOverride: true } : {}) } : {}) }; const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)); - const recipeJson = encode(recipe); + const blobs = [...new Map(recipe.files.map(file => [file.sha256, + { sha256: file.sha256, content: new Uint8Array(Buffer.from(file.content, 'base64')) }])).values()]; + const storedRecipe = { ...recipe, files: recipe.files.map(({ content, ...file }) => ({ ...file, contentHash: file.sha256 })) }; + const recipeJson = encode(storedRecipe); const payloadJson = encode(payload); const plan = { ...recipe, files: recipe.files.map(({ content: _content, ...file }) => file) }; - const recipeSummary = JSON.stringify(summarizeRecipe(plan, sha256Buffer(Buffer.from(recipeJson.buffer)))); + const recipeSummary = JSON.stringify(summarizeRecipe(plan, sha256Buffer(Buffer.from(JSON.stringify(recipe))))); // Transfer ownership of the encoded bytes; do not clone giant strings or objects. - parentPort!.postMessage({ id, value: { recipe: plan, recipeSummary, recipeJson, payloadJson } }, - [recipeJson.buffer, payloadJson.buffer]); + parentPort!.postMessage({ id, value: { recipe: plan, recipeSummary, recipeJson, payloadJson, blobs } }, + [recipeJson.buffer, payloadJson.buffer, ...blobs.map(blob => blob.content.buffer)]); } catch (error) { parentPort!.postMessage({ id, error: error instanceof Error ? error.message : 'Launch preparation failed.' }); } diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index c854030..768e2eb 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -1,3 +1,4 @@ +import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js'; import { concurrencyBlocker, preparingJobs } from './concurrency.js'; import { isIP } from 'node:net'; import { setImmediate as yieldToController } from 'node:timers/promises'; @@ -215,13 +216,14 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; } } -function filesFor(job: Job): Array { - const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s +function filesFor(job: Job): FilePlan[] { + const rows = db().prepare(`SELECT f.id,f.kind,f.rel_path,f.content_hash,f.check_before,f.expected_before_sha + FROM staged_file_changes f JOIN provisioning_sources s ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server - WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as StagedFileChange[]; + WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as Array< + Pick & { content_hash: string | null }>; return rows.map(row => ({ id: row.id, path: posix.join(relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), row.rel_path), - contentBase64: row.content?.toString('base64') ?? null, sha256: row.content ? sha256Buffer(row.content) : null, - checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha })); + sha256: row.content_hash, checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha })); } function fileMetadata({ contentBase64, ...file }: FilePlan & { contentBase64?: string | null }): FilePlan { return file; } @@ -236,27 +238,29 @@ function applyReceipt(job: Job, result: Record) { if (job.applied_at) return; const plans = storedFiles(job); if (!plans.length) return; - const before = result.before as { id?: number; path: string; contentBase64: string | null }[] | undefined; + const before = result.before as { id?: number; path: string; contentBase64: string | null; contentSha256?: string }[] | undefined; db().transaction(() => { for (const plan of plans) { const snapshot = before?.find(item => item.id === plan.id || item.path === plan.path); if (!snapshot) throw new Error('Agent activation receipt is missing a before image.'); - const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange; + const row = db().prepare(`SELECT *, ${blobSize('content_hash')} AS size FROM staged_file_changes WHERE id=?`).get(plan.id) as + Omit & { content_hash: string | null; size: number }; if (!row || row.state === 'applied') continue; - const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64'); + const bytes = snapshot.contentSha256 ? getFileBlob(snapshot.contentSha256) + : snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64'); if (row.source_kind === 'beta') { captureBetaBeforeImage(row, bytes); db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server); } if (row.deployment_file_id && !row.preserve_before_image) { - db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=?,snapshot_pending=0 WHERE id=?') - .run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id); + db().prepare('UPDATE deployment_files SET verb=?,content_before_hash=?,sha256_before=?,snapshot_pending=0 WHERE id=?') + .run(row.content_hash === null ? 'removed' : bytes === null ? 'added' : 'updated', putFileBlob(bytes), bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id); if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') .run(row.source_id, `${job.server}: captured rollback snapshot for ${row.rel_path}`); } - if (row.content === null) removeScript(job.server, row.rel_path); + if (row.content_hash === null) removeScript(job.server, row.rel_path); else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null, - sha256Buffer(row.content), row.content.length, row.inventory_source); + row.content_hash, row.size, row.inventory_source); db().prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?").run(new Date().toISOString(), row.id); } db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id); @@ -596,7 +600,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { const files = job.stage_operation_id ? storedFiles(job) : filesFor(job); if (files.length && !job.stage_operation_id) { db().transaction(() => { - const stage = queueHostOperationSerialized(instance.host_id, 'files.stage', Buffer.from(JSON.stringify({ stageId: job.id, files })), + const stage = queueHostOperationSerialized(instance.host_id, 'files.stage', Buffer.from(JSON.stringify({ stageId: job.id, files: files.map(file => ({ ...file, ...(file.sha256 ? { contentSha256: file.sha256 } : { contentBase64: null }) })) })), { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:stage` }, { stageId: job.id }); db().prepare('UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?') .run(stage.id, JSON.stringify(files.map(fileMetadata)), job.id); @@ -713,6 +717,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { if (prepared) { deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted', checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } }); + for (const blob of prepared.blobs ?? []) putFileBlob(blob.content); const recipe = prepared.recipe; db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?') .run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id); diff --git a/src/lib/server/retention.ts b/src/lib/server/retention.ts index 1809c24..748ff51 100644 --- a/src/lib/server/retention.ts +++ b/src/lib/server/retention.ts @@ -1,3 +1,4 @@ +import { blobSize, pruneFileBlobs } from './db/file-blobs.js'; import { createHash } from 'node:crypto'; import { db } from './db/index.js'; import { fleetSettings } from './settings.js'; @@ -42,7 +43,7 @@ function policies(): Policy[] { return [ { category: 'campaign', label: 'Campaigns', table: 'deployments', id: 'id', clock: 'started_at', days: 'deployKeepDays', eligible: `${campaignSafe} AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='file-content' AND c.record_id=o.id AND c.protected=1)`, failed: failedCampaign, - bytes: `(SELECT COALESCE(SUM(COALESCE(length(content_before),0)+COALESCE(length(content_after),0)),0) FROM deployment_files WHERE deployment_id=o.id)` }, + bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` }, { category: 'provisioning', label: 'Provisioning workflows', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'provisioningKeepDays', failed: "o.state='failed'", eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')}) AND NOT EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=o.id AND r.healthy_at IS NULL) AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id) WHERE n.deployment_id=o.id AND a.state NOT IN ('completed','failed')) @@ -70,22 +71,22 @@ function policies(): Policy[] { { category: 'beta', label: 'Completed betas', table: 'betas', id: 'id', clock: 'closed_at', days: 'betaKeepDays', eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta-file-content' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id) AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`, - bytes: '(SELECT COALESCE(SUM(length(content)),0) FROM beta_revision_files WHERE beta_id=o.id)' }, + bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` }, { category: 'file-content', label: 'Campaign file bodies', table: 'deployments', id: 'id', clock: 'finished_at', days: 'fileContentKeepDays', eligible: `${campaignSafe} AND NOT (${failedCampaign}) AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=o.id AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`, - bytes: '(SELECT COALESCE(SUM(COALESCE(length(content_before),0)+COALESCE(length(content_after),0)),0) FROM deployment_files WHERE deployment_id=o.id)' }, + bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` }, { category: 'beta-file-content', label: 'Closed beta file snapshots', table: 'betas', id: 'id', clock: 'closed_at', days: 'fileContentKeepDays', eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id) AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT)) AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`, - bytes: '(SELECT COALESCE(SUM(length(content)),0) FROM beta_revision_files WHERE beta_id=o.id)' }, + bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` }, { category: 'version', label: 'Unused artifact versions', table: 'script_versions', id: 'id', clock: 'uploaded_at', days: 'versionKeepDays', minimum: false, eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id) AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id) - AND ${versionReferences()}`, bytes: 'length(o.content)' }, + AND ${versionReferences()}`, bytes: `${blobSize('o.content_hash')}` }, { category: 'audit', label: 'Authentication audit events', table: 'auth_audit', id: 'id', clock: 'created_at', days: 'auditKeepDays', eligible: '1', bytes: 'length(o.details)' } ]; } @@ -181,18 +182,18 @@ export function pruneHistory(only?: string): number { db().prepare('DELETE FROM ec_host_operation_events WHERE operation_id=?').run(row.id); if (policy.category === 'operation') { const payload = db().prepare(`SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(row.id) as { payload: string }; - db().prepare(`UPDATE ec_host_operations SET request_hash=?,payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`) + db().prepare(`UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?),payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`) .run(createHash('sha256').update(payload.payload).digest('hex'),new Date().toISOString(),row.id); } db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(row.id); } if (policy.category === 'beta-file-content') { - db().prepare("UPDATE beta_revision_files SET content=X'' WHERE beta_id=?").run(row.id); + db().prepare("UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?").run(row.id); db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id=?').run(row.id); db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString()); } else if (policy.category === 'file-content') { - db().prepare('UPDATE deployment_files SET content_before=NULL,content_after=NULL WHERE deployment_id=?').run(row.id); - db().prepare("UPDATE staged_file_changes SET content=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')").run(row.id); + db().prepare('UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?').run(row.id); + db().prepare("UPDATE staged_file_changes SET content_hash=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')").run(row.id); db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString()); } else if (policy.category !== 'operation') { db().prepare(`DELETE FROM ${policy.table} WHERE rowid=?`).run(row.rid); @@ -202,6 +203,7 @@ export function pruneHistory(only?: string): number { return rows.length; })(); } + pruneFileBlobs(); return removed; } diff --git a/src/routes/api/agent/blobs/[sha256]/+server.ts b/src/routes/api/agent/blobs/[sha256]/+server.ts new file mode 100644 index 0000000..c4030e8 --- /dev/null +++ b/src/routes/api/agent/blobs/[sha256]/+server.ts @@ -0,0 +1,17 @@ +import { error } from '@sveltejs/kit'; +import { authenticateAgent } from '$lib/server/hosts/http.js'; +import { db } from '$lib/server/db/index.js'; + +export function GET({ request, params }: { request: Request; params: { sha256: string } }) { + const host = authenticateAgent(request); + if (!/^[a-f0-9]{64}$/.test(params.sha256)) error(404, 'File not found.'); + const row = db().prepare(`SELECT b.content,b.size FROM file_blobs b WHERE b.sha256=? AND EXISTS ( + SELECT 1 FROM file_blob_references r JOIN ec_host_operations o ON o.id=r.owner_id + WHERE r.sha256=b.sha256 AND r.owner_type='operation' AND o.host_id=?)`).get(params.sha256, host.id) as + { content: Buffer; size: number } | undefined; + if (!row) error(404, 'File not found.'); + return new Response(new Uint8Array(row.content), { headers: { + 'Content-Type': 'application/octet-stream', 'Content-Length': String(row.size), + 'Cache-Control': 'private, no-store', ETag: `"${params.sha256}"` + } }); +} diff --git a/src/routes/api/betas/[id]/revisions/[revision]/files/[scriptId]/+server.ts b/src/routes/api/betas/[id]/revisions/[revision]/files/[scriptId]/+server.ts index cd9fb84..4341489 100644 --- a/src/routes/api/betas/[id]/revisions/[revision]/files/[scriptId]/+server.ts +++ b/src/routes/api/betas/[id]/revisions/[revision]/files/[scriptId]/+server.ts @@ -1,3 +1,4 @@ +import { blobContent } from '$lib/server/db/file-blobs.js'; import { db } from '$lib/server/db/index.js'; /** Current files can be reviewed before the first revision is archived. */ @@ -5,12 +6,12 @@ export async function GET({ params }: { params: { id: string; revision: string; const values = [params.id, params.revision, params.scriptId].map(Number); if (values.some(value => !Number.isSafeInteger(value) || value < 1)) return new Response('Invalid file', { status: 400 }); const [id, revision, scriptId] = values; - const file = db().prepare('SELECT content FROM beta_revision_files WHERE beta_id = ? AND revision = ? AND script_id = ?') + const file = db().prepare(`SELECT ${blobContent('content_hash')} AS content FROM beta_revision_files WHERE beta_id = ? AND revision = ? AND script_id = ?`) .get(id, revision, scriptId) as { content: Buffer } | undefined - ?? (revision === 1 ? db().prepare(`SELECT v.content FROM beta_scripts bs JOIN script_versions v ON v.id = bs.script_version_id + ?? (revision === 1 ? db().prepare(`SELECT ${blobContent('v.content_hash')} AS content FROM beta_scripts bs JOIN script_versions v ON v.id = bs.script_version_id WHERE bs.beta_id = ? AND bs.script_id = ? AND NOT EXISTS (SELECT 1 FROM beta_revisions WHERE beta_id = ?)`) .get(id, scriptId, id) as { content: Buffer } | undefined : undefined); - if (!file) return new Response('File not found', { status: 404 }); + if (!file?.content) return new Response('File not found', { status: 404 }); return new Response(new Uint8Array(file.content), { headers: { 'Content-Type': 'application/octet-stream', 'Content-Disposition': 'attachment', 'Cache-Control': 'no-store' } }); } diff --git a/src/routes/api/campaigns/[id]/+server.ts b/src/routes/api/campaigns/[id]/+server.ts index 1647488..159474c 100644 --- a/src/routes/api/campaigns/[id]/+server.ts +++ b/src/routes/api/campaigns/[id]/+server.ts @@ -1,3 +1,4 @@ +import { blobSize } from '$lib/server/db/file-blobs.js'; import { json, error } from '@sveltejs/kit'; import { db } from '$lib/server/db/index.js'; @@ -16,8 +17,8 @@ export async function GET({ params }: { params: { id: string } }) { const files = db() .prepare( `SELECT id, server, verb, rel_path, sha256_before, sha256_after, snapshot_pending, - LENGTH(content_before) AS size_before, - LENGTH(content_after) AS size_after, + ${blobSize('content_before_hash')} AS size_before, + ${blobSize('content_after_hash')} AS size_after, COALESCE((SELECT state FROM staged_file_changes WHERE deployment_file_id = deployment_files.id), 'applied') AS apply_state FROM deployment_files WHERE deployment_id = ? diff --git a/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts b/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts index a858f3a..606210d 100644 --- a/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts +++ b/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts @@ -1,3 +1,4 @@ +import { blobContent } from '$lib/server/db/file-blobs.js'; import { json, error } from '@sveltejs/kit'; import { db } from '$lib/server/db/index.js'; @@ -10,7 +11,7 @@ export async function GET({ params }: { params: { id: string; fileId: string } } } const row = db() .prepare( - `SELECT verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending + `SELECT verb, rel_path, sha256_before, sha256_after, ${blobContent('content_before_hash')} AS content_before, ${blobContent('content_after_hash')} AS content_after, snapshot_pending FROM deployment_files WHERE deployment_id = ? AND id = ?` ) diff --git a/src/routes/api/deployments/[id]/+server.ts b/src/routes/api/deployments/[id]/+server.ts index 772f996..a6880e6 100644 --- a/src/routes/api/deployments/[id]/+server.ts +++ b/src/routes/api/deployments/[id]/+server.ts @@ -1,3 +1,4 @@ +import { blobSize } from '$lib/server/db/file-blobs.js'; import { json, error } from '@sveltejs/kit'; import { db } from '$lib/server/db/index.js'; import { getDeployment } from '$lib/server/provisioning/index.js'; @@ -19,8 +20,8 @@ export async function GET({ params, url }: { params: { id: string }; url?: URL } const files = db() .prepare( `SELECT id, server, verb, rel_path, sha256_before, sha256_after, - LENGTH(content_before) AS size_before, - LENGTH(content_after) AS size_after + ${blobSize('content_before_hash')} AS size_before, + ${blobSize('content_after_hash')} AS size_after FROM deployment_files WHERE deployment_id = ? ORDER BY server, rel_path` diff --git a/src/routes/api/deployments/[id]/file/[fileId]/+server.ts b/src/routes/api/deployments/[id]/file/[fileId]/+server.ts index b2e86cf..56d9aa9 100644 --- a/src/routes/api/deployments/[id]/file/[fileId]/+server.ts +++ b/src/routes/api/deployments/[id]/file/[fileId]/+server.ts @@ -1,3 +1,4 @@ +import { blobContent } from '$lib/server/db/file-blobs.js'; import { json, error } from '@sveltejs/kit'; import { db } from '$lib/server/db/index.js'; @@ -7,7 +8,7 @@ import { db } from '$lib/server/db/index.js'; export async function GET({ params }: { params: { id: string; fileId: string } }) { const row = db() .prepare( - `SELECT verb, rel_path, sha256_before, sha256_after, content_before, content_after + `SELECT verb, rel_path, sha256_before, sha256_after, ${blobContent('content_before_hash')} AS content_before, ${blobContent('content_after_hash')} AS content_after FROM deployment_files WHERE deployment_id = ? AND id = ?` ) diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index 22f2a18..9c98576 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -33,7 +33,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.5.3" +VERSION = "1.6.0" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1356,7 +1356,7 @@ class Agent: if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() - capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) + capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation): @@ -1559,6 +1559,18 @@ class Agent: managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) def decode_file(self, item): + if "contentSha256" in item: + sha = item["contentSha256"] + if not isinstance(sha, str) or not re.fullmatch(r"[a-f0-9]{64}", sha): + raise AgentError("Invalid file content hash") + url = self.config["controller"].rstrip("/") + "/api/agent/blobs/" + sha + with self.artifact_open("GET", url) as response: + content = response.read(MAX_FILE + 1) + if len(content) > MAX_FILE: + raise AgentError("File exceeds staging size limit") + if digest(content) != sha or item.get("sha256", sha) != sha: + raise AgentError("Staged file checksum mismatch") + return content try: content = base64.b64decode(item["contentBase64"], validate=True) except (ValueError, KeyError, TypeError): @@ -1677,7 +1689,7 @@ class Agent: if item["path"] in paths: raise AgentError("Duplicate staged file path") paths.add(item["path"]) - deleting = item.get("delete") is True or item.get("contentBase64") is None + deleting = item.get("delete") is True or (item.get("contentBase64") is None and "contentSha256" not in item) content = None if deleting else self.decode_file(item) total_size += len(content) if content is not None else 0 if total_size > 64 * 1024 * 1024: diff --git a/static/agent/manifest.json b/static/agent/manifest.json index d9a21b3..6889268 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.5.3", + "version": "1.6.0", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "ebd6f67a44fdb4bd0683597d4abd54999311473561204de368864b04a5436256", + "sha256": "bcc0a207125f2a1ad754c294decf2238064995ba93183b845a589a41a41925f6", "updaterProtocolVersion": 1, "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8" } diff --git a/tests/unit/beta-promotion.test.ts b/tests/unit/beta-promotion.test.ts index 9035da7..3217986 100644 --- a/tests/unit/beta-promotion.test.ts +++ b/tests/unit/beta-promotion.test.ts @@ -1,3 +1,4 @@ +import { putFileBlob } from '../../src/lib/server/db/file-blobs.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import Database from 'better-sqlite3'; import { existsSync } from 'node:fs'; @@ -35,13 +36,13 @@ describe('reviewed beta promotion', () => { it('upgrades existing campaign history without marking known snapshots pending', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('legacy', 'promote', ?)").run(new Date().toISOString()); - db().prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_before) VALUES ('legacy', ?, 'updated', 'old.gsc', ?)") - .run(servers[0], Buffer.from('original')); + db().prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_before_hash) VALUES ('legacy', ?, 'updated', 'old.gsc', ?)") + .run(servers[0], putFileBlob(Buffer.from('original'), db())); closeDb(); const legacy = new Database(fixture.dbPath); try { legacy.exec('ALTER TABLE deployment_files DROP COLUMN snapshot_pending'); } finally { legacy.close(); } - expect(db().prepare('SELECT snapshot_pending, content_before FROM deployment_files').get()) + expect(db().prepare('SELECT snapshot_pending, (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before FROM deployment_files').get()) .toEqual({ snapshot_pending: 0, content_before: Buffer.from('original') }); }); it.each(['script', 'baseline'] as const)('queues %s promotion without contacting additional host agents', async kind => { @@ -82,7 +83,7 @@ describe('reviewed beta promotion', () => { await writeFile(path, before); } await activateFiles('campaign', result.deploymentId); - expect(db().prepare('SELECT snapshot_pending, verb, content_before FROM deployment_files WHERE id = ?').get(file.id)) + expect(db().prepare('SELECT snapshot_pending, verb, (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before FROM deployment_files WHERE id = ?').get(file.id)) .toEqual({ snapshot_pending: 0, verb: before === null ? 'added' : 'updated', content_before: before === null ? null : Buffer.from(before) }); expect(db().prepare('SELECT message FROM campaign_events WHERE deployment_id = ?').all(result.deploymentId)) .toContainEqual({ message: `${servers[1]}: captured rollback snapshot for ${kind === 'script' ? 'review.gsc' : configFileOf(servers[1])}` }); diff --git a/tests/unit/beta-revisions.test.ts b/tests/unit/beta-revisions.test.ts index edf5abe..4169512 100644 --- a/tests/unit/beta-revisions.test.ts +++ b/tests/unit/beta-revisions.test.ts @@ -61,7 +61,7 @@ describe('beta revisions', () => { expect(history[0]).toMatchObject({ revision: 2, notes: 'Small fix', results: null }); expect(history[1].results?.test?.servers[0].passed).toBe(true); expect(history[1].results?.test?.evidence).toHaveLength(1); - expect(x.db.prepare('SELECT content FROM beta_revision_files WHERE beta_id = ? AND revision = 1').get(x.betaId)) + expect(x.db.prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM beta_revision_files WHERE beta_id = ? AND revision = 1').get(x.betaId)) .toEqual({ content: Buffer.from('revision one') }); expect(await readFile(x.path, 'utf8')).toBe('revision one'); await expect(x.promoteBeta(x.betaId)).rejects.toThrow('has not passed'); diff --git a/tests/unit/beta-staging.test.ts b/tests/unit/beta-staging.test.ts index a458b3d..dba3825 100644 --- a/tests/unit/beta-staging.test.ts +++ b/tests/unit/beta-staging.test.ts @@ -193,7 +193,7 @@ describe('staged beta files', () => { const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js'); const relPath = kind === 'script' ? script.target_rel_path : configFileOf(servers[1]); expect(await readFile(join(destOfKind(servers[1], kind), relPath), 'utf8')).toContain(newestBytes.trim()); - expect(db().prepare('SELECT content_before FROM deployment_files WHERE deployment_id = ? AND server = ?') + expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before FROM deployment_files WHERE deployment_id = ? AND server = ?') .get(latest.deploymentId, servers[1])).toEqual({ content_before: null }); }); @@ -219,7 +219,7 @@ describe('staged beta files', () => { await activateFiles('campaign', result.deploymentId); expect(await readFile(path, 'utf8')).toContain('set sv_voice "0"'); } - const history = db().prepare('SELECT content_before FROM deployment_files WHERE deployment_id = ? AND server = ?') + const history = db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before FROM deployment_files WHERE deployment_id = ? AND server = ?') .get(result.deploymentId, servers[0]) as { content_before: Buffer }; expect(history.content_before.toString()).toBe('original configuration'); const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); diff --git a/tests/unit/campaign-resume.test.ts b/tests/unit/campaign-resume.test.ts index 33cdd83..89f62bd 100644 --- a/tests/unit/campaign-resume.test.ts +++ b/tests/unit/campaign-resume.test.ts @@ -50,7 +50,7 @@ describe('campaign resume API', () => { expect(db().prepare('SELECT restart_state, error, provisioning_id FROM deployment_servers WHERE deployment_id = ?').get(campaign)) .toEqual({ restart_state: 'pending', error: null, provisioning_id: null }); expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign)).toEqual({ finished_at: null }); - expect(db().prepare('SELECT state, content FROM staged_file_changes WHERE source_id = ?').get(campaign)) + expect(db().prepare('SELECT state, (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_id = ?').get(campaign)) .toEqual({ state: 'pending', content: restored }); expect(await readFile(destination())).toEqual(before); expect(kick).toHaveBeenCalledTimes(1); diff --git a/tests/unit/drift-kinds.test.ts b/tests/unit/drift-kinds.test.ts index 5f7b598..d7d3ca4 100644 --- a/tests/unit/drift-kinds.test.ts +++ b/tests/unit/drift-kinds.test.ts @@ -1,3 +1,4 @@ +import { putFileBlob } from '../../src/lib/server/db/file-blobs.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; @@ -143,7 +144,7 @@ describe('reviewed drift plans', () => { const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'dedicated_sp.cfg'); await writeFile(path, 'existing settings'); const plan = await previewDrift(['t5_zombie_theater_1']); - db().prepare('UPDATE script_versions SET content = ? WHERE id = ?').run(Buffer.from('set sv_voice "0"\n'), version.id); + db().prepare('UPDATE script_versions SET content_hash = ? WHERE id = ?').run(putFileBlob(Buffer.from('set sv_voice "0"\n'), db()), version.id); await expect(applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged); expect(await readFile(path, 'utf8')).toBe('existing settings'); }); @@ -316,7 +317,7 @@ describe('drift scanning', () => { expect(result).toMatchObject({ totalRemoved: 0, totalRestored: 1 }); expect(await readFile(target)).toEqual(content); expect(getServerScript(server, 'zm_spawn_fix.gsc')?.script_version_id).toBe(version.id); - expect(db().prepare('SELECT verb, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(result.deploymentId)) + expect(db().prepare('SELECT verb, (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before, (SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files WHERE deployment_id = ?').all(result.deploymentId)) .toEqual([{ verb: 'updated', content_before: before, content_after: content }]); expect(await scanServerDrift(server)).toEqual([]); expect((await applyDriftDeployment({ servers: [server] })).deploymentId).toBeNull(); @@ -376,7 +377,7 @@ describe('drift rebuild campaigns', () => { expect(campaignId).toBeTruthy(); expect(await readFile(target, 'utf8')).toBe('old content'); expect(await readFile(stray, 'utf8')).toBe('untracked'); - expect(db().prepare('SELECT rel_path, verb, content_before, content_after FROM deployment_files WHERE deployment_id = ? ORDER BY rel_path').all(campaignId)) + expect(db().prepare('SELECT rel_path, verb, (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before, (SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files WHERE deployment_id = ? ORDER BY rel_path').all(campaignId)) .toEqual([ { rel_path: 'managed.gsc', verb: 'updated', content_before: Buffer.from('old content'), content_after: (await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id) }, { rel_path: 'stray.cfg', verb: 'removed', content_before: Buffer.from('untracked'), content_after: null } diff --git a/tests/unit/file-blobs.test.ts b/tests/unit/file-blobs.test.ts new file mode 100644 index 0000000..e980135 --- /dev/null +++ b/tests/unit/file-blobs.test.ts @@ -0,0 +1,85 @@ +import Database from 'better-sqlite3'; +import { createHash } from 'node:crypto'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import schema from '../../src/lib/server/db/schema.sql?raw'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +let fixture: Fixture; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); + +it('shares bytes across versions, staged copies and commands, and scopes downloads to the assigned host', async () => { + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const { pruneFileBlobs } = await import('../../src/lib/server/db/file-blobs.js'); + const content = Buffer.alloc(1024 * 1024, 71); + const hash = createHash('sha256').update(content).digest('hex'); + for (let i = 0; i < 20; i++) { + createScript({ name: `plugin-${i}`, kind: 'plugin', targetRelPath: 'plugin.dll', content }); + queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: `t5_server_${i}`, kind: 'plugin', relPath: 'plugin.dll', content, inventorySource: 'test' }); + } + const host = hosts.createHost({ name: 'Owner' }); + const credential = hosts.enrollHost(hosts.createEnrollment(host.id).token, { capabilities: ['files.blobs'] }).credential; + const other = hosts.createHost({ name: 'Other' }); + const otherCredential = hosts.enrollHost(hosts.createEnrollment(other.id).token, {}).credential; + const payload = { stageId: 'one', files: [{ path: 'plugins/plugin.dll', contentBase64: content.toString('base64'), sha256: hash }] }; + const op = hosts.queueHostOperation(host.id, 'files.stage', payload, { idempotencyKey: 'one' }); + expect(hosts.queueHostOperation(host.id, 'files.stage', payload, { idempotencyKey: 'one' }).id).toBe(op.id); + expect(() => hosts.queueHostOperation(host.id, 'files.stage', { ...payload, stageId: 'two' }, { idempotencyKey: 'one' })).toThrow('different request'); + expect(db().prepare('SELECT COUNT(*) n,SUM(size) bytes FROM file_blobs').get()).toEqual({ n: 1, bytes: content.length }); + const packed = JSON.parse(hosts.getHostOperationPayload(host.id, op.id).toString()); + expect(packed.files[0]).toEqual({ path: 'plugins/plugin.dll', contentSha256: hash, sha256: hash }); + expect(pruneFileBlobs()).toBe(0); + const { GET } = await import('../../src/routes/api/agent/blobs/[sha256]/+server.js'); + const request = (token: string) => new Request(`https://ec.test/api/agent/blobs/${hash}`, { headers: { authorization: `Bearer ${token}` } }); + expect(Buffer.from(await GET({ request: request(credential), params: { sha256: hash } }).arrayBuffer())).toEqual(content); + expect(() => GET({ request: request(otherCredential), params: { sha256: hash } })).toThrow(); + closeDb(); + expect(hosts.getHostOperation(op.id)?.payload).toEqual(payload); + // Previous releases receive their original inline protocol during a rolling upgrade. + hosts.heartbeat(host.id, {}); + expect(JSON.parse(hosts.getHostOperationPayload(host.id, op.id).toString())).toEqual(payload); +}); + +it('backfills legacy bytes once while preserving foreign keys, IDs and pending work', async () => { + const legacy = new Database(fixture.dbPath); + const content = Buffer.from([0, 255, 1, 88]); + const hash = createHash('sha256').update(content).digest('hex'); + try { + legacy.exec(schema.replace(/\b(content|content_before|content_after|prior_content)_hash(\s+)TEXT REFERENCES file_blobs\(sha256\)/g, '$1$2BLOB')); + legacy.exec("INSERT INTO scripts(id,name,target_rel_path) VALUES(7,'legacy','plugin.dll')"); + legacy.prepare('INSERT INTO script_versions(id,script_id,version,sha256,size,content) VALUES(9,7,1,?,?,?)').run(hash, content.length, content); + legacy.prepare("INSERT INTO staged_file_changes(source_kind,source_id,server,kind,rel_path,content,inventory_source) VALUES('campaign','old','t5_one','plugin','plugin.dll',?,'test')").run(content); + legacy.exec("INSERT INTO deployments(id,action,started_at) VALUES('old','promote','2026-10-01T00:00:00.000Z')"); + legacy.prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before,content_after) VALUES('old','t5_one','updated','plugin.dll',?,?)").run(content, content); + } finally { legacy.close(); } + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const { getVersionContent } = await import('../../src/lib/server/fleet/scripts.js'); + for (let pass = 0; pass < 2; pass++) { + expect(getVersionContent(9)).toEqual(content); + expect(db().prepare('SELECT content_hash,state FROM staged_file_changes').get()).toEqual({ content_hash: hash, state: 'pending' }); + expect(db().prepare('SELECT COUNT(*) n FROM file_blobs').get()).toEqual({ n: 1 }); + expect(db().pragma('foreign_key_check')).toEqual([]); + expect((db().pragma('table_info(script_versions)') as { name: string }[]).some(column => column.name === 'content')).toBe(false); + closeDb(); + } +}); + +it('migrates legacy command and receipt bytes without changing replay identity', async () => { + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const host = hosts.createHost({ name: 'Legacy' }); + hosts.enrollHost(hosts.createEnrollment(host.id).token, {}); + const payload = { stageId: 'legacy', files: [{ path: 'plugin.dll', contentBase64: 'AAEC' }] }; + const result = { before: [{ path: 'plugin.dll', contentBase64: 'AAEC' }] }; + db().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,idempotency_key,payload,state,result,created_at,updated_at) + VALUES('legacy',?,'files.stage','legacy',?,'running',?, ?,?)`).run(host.id, JSON.stringify(payload), JSON.stringify(result), new Date().toISOString(), new Date().toISOString()); + db().prepare("DELETE FROM ec_host_schema_migrations WHERE name IN ('staged-operation-payloads','operation-file-blobs')").run(); + closeDb(); + expect(hosts.getHostOperation('legacy')).toMatchObject({ payload, result, state: 'running' }); + expect(hosts.queueHostOperation(host.id, 'files.stage', payload, { idempotencyKey: 'legacy' }).id).toBe('legacy'); + expect(db().prepare('SELECT COUNT(*) n FROM file_blobs').get()).toEqual({ n: 1 }); + expect(hosts.getHostOperationStatus('legacy')?.result).toEqual({ before: [{ path: 'plugin.dll', contentSha256: createHash('sha256').update(Buffer.from('AAEC', 'base64')).digest('hex') }] }); +}); diff --git a/tests/unit/fleet-overview.test.ts b/tests/unit/fleet-overview.test.ts index 9892241..39efd98 100644 --- a/tests/unit/fleet-overview.test.ts +++ b/tests/unit/fleet-overview.test.ts @@ -113,7 +113,7 @@ describe('persisted Fleet operations', () => { const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js'); expect(fleetOperations(now)[0]).toMatchObject({ kind: 'recovery', state: 'attention', detail: 'RCON not answering' }); handle.prepare("INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')").run(); - handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 0, X'')").run(); + handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content_hash) VALUES (1, 1, 1, 'hash', 0, NULL)").run(); handle.prepare("INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'running')").run(); handle.prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage) VALUES (1, 'server', 0, 'awaiting-match')").run(); expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ kind: 'beta', label: 'Observing a match', href: '/betas/1' })); diff --git a/tests/unit/github-plugin-import.test.ts b/tests/unit/github-plugin-import.test.ts index 575d8a5..cb40d09 100644 --- a/tests/unit/github-plugin-import.test.ts +++ b/tests/unit/github-plugin-import.test.ts @@ -72,7 +72,8 @@ describe('GitHub DLL imports', () => { expect(imported.sha256).toBe(sha256); expect(imported.betaUrl).toBe(`/betas?pluginId=${imported.pluginId}&versionId=${imported.versionId}`); expect(getScript(imported.pluginId)).toMatchObject({ kind: 'plugin', target_rel_path: 't6-gsc-utils.dll', scope_value: 't6', current_version_id: null }); - expect(getVersion(imported.versionId)).toMatchObject({ release_version: 'v3.1.0', content, notes: metadata().body, state: 'untested' }); + expect(getVersion(imported.versionId)).toMatchObject({ release_version: 'v3.1.0', notes: metadata().body, state: 'untested' }); + expect((await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(imported.versionId)).toEqual(content); expect(pluginSource(imported.pluginId)).toMatchObject({ repository: 'alicealys/t6-gsc-utils', release_id: 100, asset_id: 200, release_url: url }); const duplicates = await Promise.all([1, 2].map(() => importPluginRelease({ url, assetId: 200, pluginId: imported.pluginId }))); expect(duplicates.every(r => r.alreadyImported && r.versionId === imported.versionId)).toBe(true); diff --git a/tests/unit/launch-recipe-summary-migration.test.ts b/tests/unit/launch-recipe-summary-migration.test.ts index b91974e..e9d44d9 100644 --- a/tests/unit/launch-recipe-summary-migration.test.ts +++ b/tests/unit/launch-recipe-summary-migration.test.ts @@ -41,6 +41,8 @@ describe('persisted launch recipe summaries', () => { expect(recipeSummaryForDeployment('missing')).toBeNull(); } finally { parsing.mockRestore(); } expect(recipeForDeployment('legacy')).toEqual({ recipe, blueprintName: 'Old collection' }); - expect(db().prepare('SELECT recipe FROM server_launch_recipes').get()).toEqual({ recipe: encoded }); + const stored = db().prepare('SELECT recipe FROM server_launch_recipes').get() as { recipe: string }; + expect(stored.recipe.length).toBeLessThan(1024); + expect(JSON.parse(stored.recipe).files[0]).not.toHaveProperty('content'); }); }); diff --git a/tests/unit/native-hosts.test.ts b/tests/unit/native-hosts.test.ts index 0ed3aa7..e2507b2 100644 --- a/tests/unit/native-hosts.test.ts +++ b/tests/unit/native-hosts.test.ts @@ -64,7 +64,10 @@ describe('native host enrollment and scoped authentication', () => { const api = await enrolled(); const { db } = await import('../../src/lib/server/db/index.js'); const payload = { files: [{ path: 'plugins/fixture.dll', contentBase64: Buffer.alloc(8 * 1024 ** 2, 42).toString('base64') }] }; - const bytes = Buffer.from(JSON.stringify(payload)); + const { putFileBlob } = await import('../../src/lib/server/db/file-blobs.js'); + const hash = putFileBlob(Buffer.from(payload.files[0].contentBase64, 'base64')); + // Launch preparation transfers bytes separately and serializes only references. + const bytes = Buffer.from(JSON.stringify({ files: [{ path: payload.files[0].path, contentSha256: hash }] })); const options = { instanceId: 'immutable-instance', generation: 1, idempotencyKey: 'large-launch' }; const parse = JSON.parse; const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => { @@ -115,8 +118,8 @@ describe('native host enrollment and scoped authentication', () => { { payload: string; last_delivered_at: string | null }; expect(stored.payload).toBe(JSON.stringify(routing)); expect(stored.last_delivered_at).not.toBeNull(); - expect(db().prepare('SELECT length(CAST(payload AS BLOB)) AS bytes FROM ec_host_operation_payloads WHERE operation_id=?').get(launch.id)) - .toEqual({ bytes: bytes.length }); + const saved = db().prepare('SELECT length(CAST(payload AS BLOB)) AS bytes FROM ec_host_operation_payloads WHERE operation_id=?').get(launch.id) as { bytes: number }; + expect(saved.bytes).toBeLessThan(1024); expect(api.getHostOperationPayload(api.host.id, launch.id)).toEqual(bytes); expect(api.getHostOperationPayload(api.host.id, inline.id).toString('utf8')).toBe(JSON.stringify(inline.payload)); expect(db().prepare('SELECT 1 FROM ec_host_operation_payloads WHERE operation_id=?').get(inline.id)).toBeUndefined(); diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index 9110afc..f4fab99 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -1,3 +1,4 @@ +import { putFileBlob } from '../../src/lib/server/db/file-blobs.js'; import { createHash } from 'node:crypto'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { join } from 'node:path'; @@ -697,7 +698,7 @@ describe('native instance provisioning', () => { const before = Buffer.from('original bytes on the host'); const after = Buffer.from('new campaign bytes'); db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)").run(new Date().toISOString()); db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')").run(job.name); - const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)").run(job.name, after).lastInsertRowid); + const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after_hash,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)").run(job.name, putFileBlob(after, db())).lastInsertRowid); const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: job.name, kind: 'script', relPath: 'hud.gsc', content: after, inventorySource: 'test', deploymentFileId: fileId }); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); @@ -712,8 +713,9 @@ describe('native instance provisioning', () => { expect(JSON.parse(routing.payload)).toEqual({ stageId: restart.id }); const plan = db().prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { files_json: string }; expect(JSON.parse(plan.files_json)[0]).not.toHaveProperty('contentBase64'); - expect(db().prepare('SELECT payload FROM ec_host_operation_payloads WHERE operation_id=?').get(stage.id)) - .toEqual({ payload: JSON.stringify(stage.payload) }); + const storedPayload = db().prepare('SELECT payload FROM ec_host_operation_payloads WHERE operation_id=?').get(stage.id) as { payload: string }; + expect(JSON.parse(storedPayload.payload).files[0]).toMatchObject({ contentSha256: createHash('sha256').update(after).digest('hex') }); + expect(JSON.parse(storedPayload.payload).files[0]).not.toHaveProperty('contentBase64'); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); const { pendingHostOperations } = await import('../../src/lib/server/hosts/index.js'); @@ -726,13 +728,13 @@ describe('native instance provisioning', () => { remote.players = 0; await advance(); const activation = (await operationFor(restart.id))!; expect(activation).toMatchObject({ kind: 'instance.restart', generation: 1, payload: { stageId: restart.id } }); - expect(db().prepare('SELECT content_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, snapshot_pending: 1 }); + expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, snapshot_pending: 1 }); const files = stage.payload.files as Array<{ id: number; path: string }>; reportHostOperation(hostId, activation.id, { state: 'completed', result: { before: [{ id: files[0].id, path: files[0].path, contentBase64: before.toString('base64') }] } }); await advance(); expect((await detail(restart.id)).state).toBe('done'); expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied'); - expect(db().prepare('SELECT content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex'), snapshot_pending: 0 }); + expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex'), snapshot_pending: 0 }); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex')); }); @@ -827,7 +829,7 @@ describe('native instance provisioning', () => { expect((await detail(restart.id)).state).toBe('failed'); expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('pending'); expect(db().prepare('SELECT COUNT(*) AS count FROM restart_rollout WHERE healthy_at IS NULL').get()).toEqual({ count: 0 }); - expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, sha256_before: null }); + expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, sha256_before: null }); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined(); }); @@ -1277,7 +1279,7 @@ describe('native provisioning cancellation and rollback', () => { expect((await detail(restart.id)).state).toBe('cancelled'); const replacement = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as {provisioning_id:string}; expect(replacement.provisioning_id).not.toBe(restart.id); - expect(db().prepare("SELECT content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?").get(rollback.deploymentId)) + expect(db().prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?").get(rollback.deploymentId)) .toEqual({content:Buffer.from('working')}); await advance(); const stagedRollback = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(replacement.provisioning_id) as {stage_operation_id:string}; @@ -1319,7 +1321,7 @@ describe('cancelled native mutation bookkeeping', () => { await reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('revision two') }]); await activate('revision one'); const rollback = await rollbackBeta(betaId); - expect(db().prepare("SELECT content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(rollback.deploymentId)) + expect(db().prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(rollback.deploymentId)) .toEqual({ content: original === null ? null : Buffer.from(original) }); }); diff --git a/tests/unit/operation-history.test.ts b/tests/unit/operation-history.test.ts index 025830f..aee5152 100644 --- a/tests/unit/operation-history.test.ts +++ b/tests/unit/operation-history.test.ts @@ -12,7 +12,7 @@ describe('persistent activity', () => { const handle = db(); handle.prepare("INSERT INTO servers (name) VALUES ('server')").run(); handle.prepare("INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')").run(); - handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 0, X'')").run(); + handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content_hash) VALUES (1, 1, 1, 'hash', 0, NULL)").run(); handle.prepare("INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'running')").run(); handle.prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage) VALUES (1, 'server', 0, 'deploying')").run(); handle.prepare("UPDATE beta_servers SET stage = 'failed', error = 'write failed' WHERE beta_id = 1").run(); diff --git a/tests/unit/retention.test.ts b/tests/unit/retention.test.ts index 4d4bd28..8a280c2 100644 --- a/tests/unit/retention.test.ts +++ b/tests/unit/retention.test.ts @@ -1,3 +1,4 @@ +import { putFileBlob } from '../../src/lib/server/db/file-blobs.js'; import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; @@ -72,11 +73,11 @@ it('selects resolved failures separately while preserving age, protection and ba it('expires file bodies without silently turning rollback into file deletion', async () => { const a = await api(); a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('p','promote',?,?)").run(old,old); - a.db.prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before,content_after) VALUES('p','s','updated','test.cfg',?,?)").run(Buffer.from('before'),Buffer.from('after')); + a.db.prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before_hash,content_after_hash) VALUES('p','s','updated','test.cfg',?,?)").run(putFileBlob(Buffer.from('before'), a.db),putFileBlob(Buffer.from('after'), a.db)); a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 0,fileContentKeepDays: 1 }); expect(a.retentionPreview().find(p=>p.category==='file-content')?.payloadBytes).toBe(11); expect(a.pruneHistory('file-content')).toBe(1); - expect(a.db.prepare("SELECT content_before,content_after FROM deployment_files").get()).toEqual({ content_before:null,content_after:null }); + expect(a.db.prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,(SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files").get()).toEqual({ content_before:null,content_after:null }); const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); await expect(rollbackPromoteDeployment('p')).rejects.toThrow('expired'); expect(a.pruneHistory('file-content')).toBe(0); @@ -119,7 +120,7 @@ it('deletes old maintenance jobs while preserving update suppression evidence an it('keeps current and referenced artifact versions and honors the minimum version count', async () => { const a = await api(); a.db.prepare("INSERT INTO scripts(id,name,target_rel_path,current_version_id) VALUES(1,'test','test.gsc',1)").run(); - for(let n=1;n<=4;n++) a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content,uploaded_at) VALUES(?,1,?,'sha',1,X'01',?)").run(n,n,old); + for(let n=1;n<=4;n++) a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash,uploaded_at) VALUES(?,1,?,'sha',1,?,?)").run(n,n,putFileBlob(Buffer.from([1]), a.db),old); a.saveFleetSettings({...a.fleetSettings(),versionKeepCount:2}); expect(a.pruneHistory('version')).toBe(1); expect(a.db.prepare('SELECT id FROM script_versions ORDER BY id').all()).toEqual([{id:1},{id:3},{id:4}]); @@ -153,15 +154,15 @@ it('prunes closed matches and audit events without deleting active matches or pr it('keeps active beta evidence while expiring closed snapshots and then closed history', async () => { const a = await api(); a.db.prepare("INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'s','s.gsc')").run(); - a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content) VALUES(1,1,1,'sha',3,X'010203')").run(); + a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash) VALUES(1,1,1,'sha',3,?)").run(putFileBlob(Buffer.from([1,2,3]), a.db)); for (const [id,state] of [[1,'running'],[2,'rolled-back']] as const) { a.db.prepare("INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at,closed_at) VALUES(?,1,1,'test',?,?,?)").run(id,state,old,old); a.db.prepare("INSERT INTO beta_revisions(beta_id,revision,started_at) VALUES(?,1,?)").run(id,old); - a.db.prepare("INSERT INTO beta_revision_files(beta_id,revision,script_id,name,kind,path,version_id,version,sha256,content) VALUES(?,1,1,'s','script','s.gsc',1,1,'sha',X'010203')").run(id); + a.db.prepare("INSERT INTO beta_revision_files(beta_id,revision,script_id,name,kind,path,version_id,version,sha256,content_hash) VALUES(?,1,1,'s','script','s.gsc',1,1,'sha',?)").run(id,putFileBlob(Buffer.from([1,2,3]), a.db)); } a.saveFleetSettings({...a.fleetSettings(),fileContentKeepDays:1}); expect(a.pruneHistory('beta-file-content')).toBe(1); - expect(a.db.prepare('SELECT beta_id,length(content) AS bytes FROM beta_revision_files ORDER BY beta_id').all()).toEqual([{beta_id:1,bytes:3},{beta_id:2,bytes:0}]); + expect(a.db.prepare('SELECT beta_id,COALESCE((SELECT size FROM file_blobs WHERE sha256=content_hash),0) AS bytes FROM beta_revision_files ORDER BY beta_id').all()).toEqual([{beta_id:1,bytes:3},{beta_id:2,bytes:0}]); a.saveFleetSettings({...a.fleetSettings(),betaKeepDays:1}); expect(a.pruneHistory('beta')).toBe(1); expect(a.db.prepare('SELECT id FROM betas').all()).toEqual([{id:1}]); diff --git a/tests/unit/rollback-staging.test.ts b/tests/unit/rollback-staging.test.ts index 8430498..1176fc1 100644 --- a/tests/unit/rollback-staging.test.ts +++ b/tests/unit/rollback-staging.test.ts @@ -1,3 +1,4 @@ +import { putFileBlob } from '../../src/lib/server/db/file-blobs.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { existsSync } from 'node:fs'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; @@ -31,8 +32,8 @@ async function promotion(kind: 'script' | 'plugin' | 'baseline', staged: boolean db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('promote', ?, 1, 'pending')").run(server); db().prepare("INSERT INTO deployment_scripts (deployment_id, script_id, script_version_id, prior_version_id, kind) VALUES ('promote', ?, ?, ?, ?)") .run(script.id, version.id, prior.id, kind); - const file = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) - VALUES ('promote', ?, 'updated', ?, ?, ?, ?, ?)`).run(server, relPath, sha256Buffer(before), sha256Buffer(after), before, after); + const file = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash) + VALUES ('promote', ?, 'updated', ?, ?, ?, ?, ?)`).run(server, relPath, sha256Buffer(before), sha256Buffer(after), putFileBlob(before, db()), putFileBlob(after, db())); const path = join(destOfKind(server, kind), relPath); await mkdir(dirname(path), { recursive: true }); await writeFile(path, staged ? before : after); @@ -86,8 +87,8 @@ describe('deferred promotion rollback', () => { const { db } = await import('../../src/lib/server/db/index.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); - const pending = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after) - VALUES ('promote', ?, 'added', 'never.gsc', ?)`).run(server, Buffer.from('queued')); + const pending = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after_hash) + VALUES ('promote', ?, 'added', 'never.gsc', ?)`).run(server, putFileBlob(Buffer.from('queued'), db())); queueFileChange({ sourceKind: 'campaign', sourceId: 'promote', server, kind: 'script', relPath: 'never.gsc', content: Buffer.from('queued'), inventorySource: 'promote:promote', deploymentFileId: Number(pending.lastInsertRowid) }); const result = await rollbackPromoteDeployment('promote'); diff --git a/tests/unit/server-history-storage.test.ts b/tests/unit/server-history-storage.test.ts index aee18f3..8c7d4e4 100644 --- a/tests/unit/server-history-storage.test.ts +++ b/tests/unit/server-history-storage.test.ts @@ -21,8 +21,9 @@ function database(legacy = false) { function seedHistory(db: Database.Database) { db.exec(`INSERT INTO servers (name, game_port) VALUES ('server', 28960); + INSERT INTO file_blobs(sha256,size,content) VALUES('hash',4,X'74657374'); INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'script', 'test.gsc'); - INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 4, X'74657374'); + INSERT INTO script_versions (id, script_id, version, sha256, size, content_hash) VALUES (1, 1, 1, 'hash', 4, 'hash'); INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'promoted'); INSERT INTO beta_servers (beta_id, server, had_original, stage, healthy_at) VALUES (1, 'server', 0, 'ready', 'healthy'); INSERT INTO beta_server_scripts (beta_id, server, script_id, had_original) VALUES (1, 'server', 1, 0); @@ -32,7 +33,7 @@ function seedHistory(db: Database.Database) { INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (1, 'server', 'test.gsc'); INSERT INTO deployments (id, action, started_at, finished_at) VALUES ('campaign', 'promote', 'before', 'after'); INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, restarted_at) VALUES ('campaign', 'server', 1, 'done', 'restarted'); - INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after) VALUES ('campaign', 'server', 'updated', 'test.gsc', X'74657374'); + INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after_hash) VALUES ('campaign', 'server', 'updated', 'test.gsc', 'hash'); INSERT INTO server_deployments (id, server, action, state, stage, started_at, finished_at, triggered_by) VALUES ('created', 'server', 'create', 'done', 'Ready', 'before', 'after', 'ui'); INSERT INTO maintenance_jobs (server, state, requested_at, finished_at, detail) VALUES ('server', 'done', 'before', 'after', 'Updated'); INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('incident', 'server', 'health', 'attention', 'Unhealthy', 'before', 'after'); @@ -76,7 +77,7 @@ describe('durable server history', () => { expect(JSON.parse(retained.retained_server_deletions)).toEqual({ server: expect.any(String) }); expect(db.prepare('SELECT stage, healthy_at FROM beta_servers').get()).toEqual({ stage: 'ready', healthy_at: 'healthy' }); expect(db.prepare('SELECT restart_state, restarted_at FROM deployment_servers').get()).toEqual({ restart_state: 'done', restarted_at: 'restarted' }); - expect(db.prepare('SELECT content_after FROM deployment_files').get()).toEqual({ content_after: Buffer.from('test') }); + expect(db.prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files').get()).toEqual({ content_after: Buffer.from('test') }); expect(db.prepare('SELECT status, highest_round, ended_at FROM leaderboard_matches').get()).toEqual({ status: 'interrupted', highest_round: 25, ended_at: expect.any(String) }); db.exec(`INSERT INTO servers (name, game_port) VALUES ('server', 28960); INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('server', 'queued', 'new'); diff --git a/tests/unit/server-history-views.test.ts b/tests/unit/server-history-views.test.ts index 92180ba..c7329f4 100644 --- a/tests/unit/server-history-views.test.ts +++ b/tests/unit/server-history-views.test.ts @@ -1,3 +1,4 @@ +import { putFileBlob } from '../../src/lib/server/db/file-blobs.js'; import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; @@ -20,8 +21,8 @@ it('exposes retained campaign and provisioning history without rebinding links t finishDeployment(operation, { ok: true }); db().prepare("INSERT INTO deployments (id, action, started_at, finished_at) VALUES ('history-campaign', 'promote', ?, ?)").run(at, at); db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('history-campaign', ?, 1, 'done')").run(server); - db().prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_before, content_after) VALUES ('history-campaign', ?, 'updated', 'script.gsc', ?, ?)") - .run(server, Buffer.from('before'), Buffer.from('after')); + db().prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_before_hash, content_after_hash) VALUES ('history-campaign', ?, 'updated', 'script.gsc', ?, ?)") + .run(server, putFileBlob(Buffer.from('before'), db()), putFileBlob(Buffer.from('after'), db())); db().prepare('DELETE FROM servers WHERE name = ?').run(server); await seedServerRegistry(); closeDb(); diff --git a/tests/unit/staged-files.test.ts b/tests/unit/staged-files.test.ts index 09f50e9..9ded8f5 100644 --- a/tests/unit/staged-files.test.ts +++ b/tests/unit/staged-files.test.ts @@ -1,3 +1,4 @@ +import { putFileBlob } from '../../src/lib/server/db/file-blobs.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { chmod, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { existsSync, writeFileSync } from 'node:fs'; @@ -104,8 +105,8 @@ describe.each(['campaign', 'beta'])('%s staged file activation', (source it('applies the exact queued bytes even if artifact storage changes while waiting', async () => { const { replacement } = await queueChange(source); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('UPDATE script_versions SET content = ? WHERE id = ?') - .run(Buffer.from('changed after review'), replacement.id); + db().prepare('UPDATE script_versions SET content_hash = ? WHERE id = ?') + .run(putFileBlob(Buffer.from('changed after review'), db()), replacement.id); await tick(source); expect(await readFile(destination())).toEqual(after); });