Deduplicate artifact bytes and deliver files by authenticated hash

src/lib/server/db/file-blobs.ts

@@ -0,0 +1,123 @@
+import { createHash } from 'node:crypto';
+import type Database from 'better-sqlite3';
+import { db } from './index.js';
+
+export function ensureFileBlobSchema(handle: Database.Database): void {
+    handle.exec(`CREATE TABLE IF NOT EXISTS file_blobs(sha256 TEXT PRIMARY KEY,size INTEGER NOT NULL,content BLOB NOT NULL);
+        CREATE TABLE IF NOT EXISTS file_blob_references(owner_type TEXT NOT NULL,owner_id TEXT NOT NULL,
+            sha256 TEXT NOT NULL REFERENCES file_blobs(sha256),PRIMARY KEY(owner_type,owner_id,sha256));
+        CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256);`);
+}
+
+export function putFileBlob(content: Uint8Array | null, handle: Database.Database = db()): string | null {
+    if (content === null) return null;
+    const sha256 = createHash('sha256').update(content).digest('hex');
+    handle.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)').run(sha256, content.byteLength, content);
+    return sha256;
+}
+
+export function getFileBlob(sha256: string, handle: Database.Database = db()): Buffer {
+    const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as { content: Buffer } | undefined;
+    if (!row) throw new Error('File content is unavailable or has expired.');
+    return row.content;
+}
+
+/** SQL fragments are for fixed schema identifiers, never user input. */
+export const blobContent = (column: string) => `(SELECT content FROM file_blobs WHERE sha256=${column})`;
+export const blobSize = (column: string) => `(SELECT size FROM file_blobs WHERE sha256=${column})`;
+
+export function referenceBlobs(ownerType: string, ownerId: string, hashes: Iterable<string>, handle: Database.Database = db()): void {
+    handle.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?').run(ownerType, ownerId);
+    const insert = handle.prepare('INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)');
+    for (const hash of hashes) insert.run(ownerType, ownerId, hash);
+}
+
+/** Immutable file bytes in JSON are replaced with hash references, including before images. */
+export function packFileJson(value: unknown, handle: Database.Database = db(), hashes = new Set<string>()): { value: unknown; hashes: Set<string> } {
+    if (Array.isArray(value)) return { value: value.map(item => packFileJson(item, handle, hashes).value), hashes };
+    if (!value || typeof value !== 'object') return { value, hashes };
+    const result: Record<string, unknown> = {};
+    for (const [key, item] of Object.entries(value)) {
+        if ((key === 'contentBase64' || key === 'content' && 'path' in value && 'sha256' in value) && typeof item === 'string') {
+            const hash = putFileBlob(Buffer.from(item, 'base64'), handle)!;
+            result[key === 'content' ? 'contentHash' : 'contentSha256'] = hash;
+            hashes.add(hash);
+        } else {
+            result[key] = packFileJson(item, handle, hashes).value;
+            if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') hashes.add(item);
+        }
+    }
+    return { value: result, hashes };
+}
+
+export function unpackFileJson(value: unknown, handle: Database.Database = db()): any {
+    if (Array.isArray(value)) return value.map(item => unpackFileJson(item, handle));
+    if (!value || typeof value !== 'object') return value;
+    const result: Record<string, unknown> = {};
+    for (const [key, item] of Object.entries(value)) {
+        if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') {
+            result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(item, handle).toString('base64');
+        } else result[key] = unpackFileJson(item, handle);
+    }
+    return result;
+}
+
+export const blobColumns = [
+    ['script_versions', 'content'], ['staged_file_changes', 'content'],
+    ['deployment_files', 'content_before'], ['deployment_files', 'content_after'],
+    ['beta_file_snapshots', 'prior_content'], ['beta_revision_files', 'content']
+] as const;
+
+/** Finish each SELECT before writing; better-sqlite3 iterators hold an active cursor. */
+export function* migrationRows<T>(handle: Database.Database, sql: string): Generator<T & { rid: number }> {
+    const select = handle.prepare(sql);
+    let cursor = -Number.MAX_SAFE_INTEGER;
+    for (;;) {
+        const row = select.get(cursor) as (T & { rid: number }) | undefined;
+        if (!row) return;
+        cursor = row.rid;
+        yield row;
+    }
+}
+
+/** Backfill one row at a time so large libraries do not exhaust startup memory. */
+export function migrateFileBlobs(handle: Database.Database): void {
+    for (const [table, column] of blobColumns) {
+        if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue;
+        handle.transaction(() => {
+            handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`);
+            const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
+            for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) {
+                update.run(putFileBlob(row.content, handle), row.rid);
+            }
+            handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
+        })();
+    }
+    for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`);
+    if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) {
+        handle.transaction(() => {
+            for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle,
+                'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) {
+                const packed = packFileJson(JSON.parse(row.recipe), handle);
+                handle.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(packed.value), row.deployment_id);
+                referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
+            }
+            handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
+        })();
+    }
+    handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN
+        DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`);
+    for (const action of ['INSERT', 'UPDATE OF recipe']) handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'}
+        AFTER ${action} ON server_launch_recipes BEGIN
+        DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=NEW.deployment_id;
+        INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256)
+            SELECT 'recipe',NEW.deployment_id,json_extract(value,'$.contentHash') FROM json_each(NEW.recipe,'$.files')
+            WHERE json_extract(value,'$.contentHash') IS NOT NULL; END;`);
+}
+
+export function pruneFileBlobs(limit = 100): number {
+    return db().prepare(`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b
+        WHERE NOT EXISTS (SELECT 1 FROM file_blob_references r WHERE r.sha256=b.sha256)
+        ${blobColumns.map(([table, column]) => `AND NOT EXISTS (SELECT 1 FROM ${table} WHERE ${column}_hash=b.sha256)`).join('\n')}
+        LIMIT ?)`).run(limit).changes;
+}

src/lib/server/db/index.ts

@@ -12,6 +12,7 @@ import { migrateServerHistory } from './server-history.js';
 import { migrateNativeInstances } from '../backends/schema.js';
 import { installPerformanceIndexes } from './performance-indexes.js';
 import { migrateVersionLayout } from './version-layout.js';
+import { migrateFileBlobs } from './file-blobs.js';
 import { tuneConnection } from './connection.js';
 import { timeStatement } from './slow-statements.js';
 
@@ -393,6 +394,7 @@ export function db(): Database.Database {
         initializeFleetSettings(handle);
         migrateNativeInstances(handle);
         migrateVersionLayout(handle);
+        migrateFileBlobs(handle);
         // Lifecycle history is retained indefinitely. Admission and status reads
         // must narrow by server or active state instead of scanning that history.
         // Install after migrations that rebuild this table or add server_deleted_at.

src/lib/server/db/schema.sql

@@ -1,3 +1,13 @@
+-- Immutable file bodies are shared across versions, campaigns, betas and host commands.
+CREATE TABLE IF NOT EXISTS file_blobs (
+    sha256 TEXT PRIMARY KEY, size INTEGER NOT NULL, content BLOB NOT NULL
+);
+CREATE TABLE IF NOT EXISTS file_blob_references (
+    owner_type TEXT NOT NULL, owner_id TEXT NOT NULL, sha256 TEXT NOT NULL REFERENCES file_blobs(sha256),
+    PRIMARY KEY(owner_type,owner_id,sha256)
+);
+CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256);
+
 -- Electric Cherry schema. Idempotent — safe to run at every boot via migrate.ts.
 -- The old betas/beta_scripts/beta_deploys tables are dropped in-code before this
 -- runs (see dropLegacyBetaTables in db/index.ts); the new script-centric model
@@ -134,7 +144,7 @@ CREATE TABLE IF NOT EXISTS script_versions (
     state        TEXT NOT NULL DEFAULT 'untested',   -- 'untested' | 'tested' | 'test-failed'
     kind         TEXT NOT NULL DEFAULT 'script',     -- denormalized from scripts.kind for join-free filtering
     uploaded_at  TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
-    content      BLOB NOT NULL,
+    content_hash      TEXT REFERENCES file_blobs(sha256),
     UNIQUE(script_id, version)
 );
 CREATE INDEX IF NOT EXISTS idx_script_versions_script ON script_versions(script_id);
@@ -201,7 +211,7 @@ CREATE TABLE IF NOT EXISTS beta_revision_files (
     version INTEGER NOT NULL,
     release_version TEXT,
     sha256 TEXT NOT NULL,
-    content BLOB NOT NULL,
+    content_hash TEXT REFERENCES file_blobs(sha256),
     PRIMARY KEY (beta_id, revision, script_id),
     FOREIGN KEY (beta_id, revision) REFERENCES beta_revisions(beta_id, revision) ON DELETE CASCADE
 );
@@ -306,7 +316,7 @@ CREATE TABLE IF NOT EXISTS beta_file_snapshots (
     beta_id       INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
     server        TEXT NOT NULL REFERENCES servers(name) ON DELETE CASCADE,
     rel_path      TEXT NOT NULL,
-    prior_content BLOB NOT NULL,
+    prior_content_hash TEXT REFERENCES file_blobs(sha256),
     prior_sha256  TEXT NOT NULL,
     prior_size    INTEGER NOT NULL,
     PRIMARY KEY (beta_id, server, rel_path)
@@ -364,8 +374,8 @@ CREATE TABLE IF NOT EXISTS deployment_files (
     rel_path       TEXT NOT NULL,
     sha256_before  TEXT,
     sha256_after   TEXT,
-    content_before BLOB,
-    content_after  BLOB,
+    content_before_hash TEXT REFERENCES file_blobs(sha256),
+    content_after_hash  TEXT REFERENCES file_blobs(sha256),
     -- An unknown before image is distinct from a confirmed absent file.
     snapshot_pending INTEGER NOT NULL DEFAULT 0
 );
@@ -382,7 +392,7 @@ CREATE TABLE IF NOT EXISTS staged_file_changes (
     server TEXT NOT NULL,
     kind TEXT NOT NULL,
     rel_path TEXT NOT NULL,
-    content BLOB,
+    content_hash TEXT REFERENCES file_blobs(sha256),
     script_version_id INTEGER,
     inventory_source TEXT NOT NULL,
     deployment_file_id INTEGER REFERENCES deployment_files(id) ON DELETE CASCADE,

src/lib/server/db/version-layout.ts

@@ -3,7 +3,7 @@ import type Database from 'better-sqlite3';
 /** Keep metadata ahead of overflow pages, preserving IDs and all references. */
 export function migrateVersionLayout(handle: Database.Database): void {
     const columns = handle.pragma('table_info(script_versions)') as { name: string }[];
-    if (columns.at(-1)?.name === 'content') return;
+    if (!columns.some(column => column.name === 'content') || columns.at(-1)?.name === 'content') return;
     const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'").get() as { sql: string };
     const content = /\bcontent\s+BLOB\s+NOT\s+NULL\s*,/i;
     if (!content.test(sql)) throw new Error('Unrecognized script version content layout');

src/lib/server/fleet/beta-revisions.ts

@@ -12,8 +12,8 @@ export function archiveBetaRevision(id: number, revision: number, startedAt: str
     db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)')
         .run(id, revision, startedAt, notes);
     db().prepare(`INSERT OR IGNORE INTO beta_revision_files
-        (beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content)
-        SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content
+        (beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content_hash)
+        SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content_hash
         FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id
         WHERE bs.beta_id = ?`).run(revision, id);
 }

src/lib/server/fleet/beta.ts

@@ -1,3 +1,4 @@
+import { putFileBlob, blobContent } from '../db/file-blobs.js';
 import { getVersionContent } from './scripts.js';
 import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
 import { storedPlan, testView, activity, setOutcome } from './beta-tests.js';
@@ -734,7 +735,7 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
     const hasCollapsing = bundle.some((b) => isCollapsingKind(b.script.kind));
 
     const getSnapshot = db().prepare(
-        `SELECT prior_content, prior_sha256, prior_size FROM beta_file_snapshots
+        `SELECT ${blobContent('prior_content_hash')} AS prior_content, prior_sha256, prior_size FROM beta_file_snapshots
           WHERE beta_id = ? AND server = ? AND rel_path = ?`
     );
 
@@ -797,10 +798,10 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
             }
             for (const change of restorations) {
                 const file = db().prepare(`INSERT INTO deployment_files
-                    (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
+                    (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
                     VALUES (?, ?, ?, ?, NULL, ?, NULL, ?)`).run(deploymentId, change.server,
                         change.content === null ? 'removed' : 'updated', change.relPath,
-                        change.content === null ? null : sha256Buffer(change.content), change.content);
+                        change.content === null ? null : sha256Buffer(change.content), putFileBlob(change.content));
                 queueFileChange({ sourceKind: 'campaign', sourceId: deploymentId, server: change.server,
                     kind: change.kind, relPath: change.relPath, content: change.content,
                     scriptVersionId: change.versionId, inventorySource: `rollback:${betaId}`,
@@ -969,7 +970,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
     const plans: Plan[] = [];
 
     const getSnapshotForPromote = db().prepare(
-        `SELECT prior_content, prior_sha256 FROM beta_file_snapshots
+        `SELECT ${blobContent('prior_content_hash')} AS prior_content, prior_sha256 FROM beta_file_snapshots
           WHERE beta_id = ? AND server = ? AND rel_path = ?`
     );
 
@@ -1096,7 +1097,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
     );
     const insertDepFile = db().prepare(
         `INSERT INTO deployment_files
-         (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending)
+         (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash, snapshot_pending)
          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
     );
 
@@ -1121,8 +1122,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
                 p.script.target_rel_path,
                 p.priorSha,
                 p.version.sha256,
-                p.priorContent,
-                getVersionContent(p.version.id),
+                putFileBlob(p.priorContent),
+                putFileBlob(getVersionContent(p.version.id)),
                 betaServerSet.has(p.server) ? 0 : 1
             );
             if (plansToActivate.has(p)) {
@@ -1141,8 +1142,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
                 configFileOf(cp.server),
                 cp.priorSha,
                 cp.composedSha,
-                cp.priorContent,
-                cp.composedContent,
+                putFileBlob(cp.priorContent),
+                putFileBlob(cp.composedContent),
                 betaServerSet.has(cp.server) ? 0 : 1
             );
             if (cp.requiresActivation) {

src/lib/server/fleet/drift.ts

@@ -1,3 +1,4 @@
+import { putFileBlob } from '../db/file-blobs.js';
 import { getVersionContent } from './scripts.js';
 import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js';
 import { randomUUID } from 'node:crypto';
@@ -401,11 +402,11 @@ export function createDriftRebuildCampaign(
         db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')")
             .run(id, new Date().toISOString());
         const insertFile = db().prepare(`INSERT INTO deployment_files
-            (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
+            (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
         for (const { file, before, after } of prepared.files) {
             insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated',
-                file.relPath, file.beforeSha256, file.afterSha256, before, after);
+                file.relPath, file.beforeSha256, file.afterSha256, putFileBlob(before), putFileBlob(after));
         }
         db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
             VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId);
@@ -480,7 +481,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
     );
     const insertFile = db().prepare(
         `INSERT INTO deployment_files
-         (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
+         (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
          VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
     );
 
@@ -513,7 +514,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
                     const content = await readFile(abs);
                     if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
                     db().transaction(() => {
-                        const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, content, null);
+                        const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, putFileBlob(content), null);
                         queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
                             kind: entry.kind, relPath: entry.relPath, content: null,
                             inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
@@ -538,7 +539,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
                     if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
                     db().transaction(() => {
                         const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath,
-                            before ? sha256Buffer(before) : null, sha256Buffer(written), before, written);
+                            before ? sha256Buffer(before) : null, sha256Buffer(written), putFileBlob(before), putFileBlob(written));
                         queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
                             kind: entry.kind, relPath: entry.relPath, content: written,
                             scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null,

src/lib/server/fleet/launch-recipes.ts

@@ -1,3 +1,4 @@
+import { packFileJson, unpackFileJson } from '../db/file-blobs.js';
 import { randomUUID } from 'node:crypto';
 import type { BlueprintSummary, LaunchRecipeSummary, LaunchFileSummary } from '$lib/deployment-plan.js';
 import { config, configFileForGame } from '../config.js';
@@ -118,7 +119,7 @@ export function summarizeRecipe(recipe: LaunchRecipePlan, fingerprint = sha256Bu
 /** Keep the small detail response beside the immutable attempted file bytes. */
 export function recipeStorage(recipe: LaunchRecipePlan): { recipe: string; recipeSummary: string } {
     const encoded = JSON.stringify(recipe);
-    return { recipe: encoded, recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) };
+    return { recipe: JSON.stringify(packFileJson(recipe).value), recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) };
 }
 
 export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
@@ -137,7 +138,7 @@ export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSu
 export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
     const row = db().prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
         .get(id) as { recipe: string; blueprint_name: string | null } | undefined;
-    return row ? { recipe: JSON.parse(row.recipe), blueprintName: row.blueprint_name } : null;
+    return row ? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name } : null;
 }
 
 export function blueprintRecipe(id: string): { recipe: LaunchRecipePlan; blueprintName: string } {

src/lib/server/fleet/rollback.ts

@@ -1,3 +1,4 @@
+import { putFileBlob, blobContent } from '../db/file-blobs.js';
 import type { ArtifactKind } from '../config.js';
 import { db } from '../db/index.js';
 import { emit } from '../pipeline/bus.js';
@@ -101,7 +102,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
 
     const fileRows = db()
         .prepare(
-            `SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, f.content_before, f.content_after, f.snapshot_pending
+            `SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, ${blobContent('f.content_before_hash')} AS content_before, ${blobContent('f.content_after_hash')} AS content_after, f.snapshot_pending
              FROM deployment_files f JOIN deployment_servers ds ON ds.deployment_id = f.deployment_id AND ds.server = f.server
              JOIN servers s ON s.name = f.server
              WHERE f.deployment_id = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL ORDER BY f.server, f.rel_path`
@@ -184,7 +185,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
     );
     const insertDepFile = db().prepare(
         `INSERT INTO deployment_files
-         (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
+         (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
          VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
     );
 
@@ -226,8 +227,8 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
                     f.rel_path,
                     f.sha256_after,
                     f.sha256_before,
-                    f.content_after,
-                    f.content_before
+                    putFileBlob(f.content_after),
+                    putFileBlob(f.content_before)
                 );
                 const owningScript = scriptByTarget.get(f.rel_path);
                 const priorVersion = owningScript?.prior_version_id ? getVersion(owningScript.prior_version_id) : null;

src/lib/server/fleet/scripts.ts

@@ -1,3 +1,4 @@
+import { putFileBlob, blobContent } from '../db/file-blobs.js';
 import { createHash } from 'node:crypto';
 import { db } from '../db/index.js';
 import { configFileForGame, type ArtifactKind } from '../config.js';
@@ -139,8 +140,8 @@ export function listPromotedOverlaysForScope(
 export const VERSION_COLUMNS = 'id, script_id, version, release_version, sha256, size, notes, state, kind, uploaded_at';
 
 export function getVersionContent(id: number): Buffer {
-    const row = db().prepare('SELECT content FROM script_versions WHERE id = ?').get(id) as { content: Buffer } | undefined;
-    if (!row) throw new ScriptError(`no such version (id=${id})`, 'no-such-version');
+    const row = db().prepare(`SELECT ${blobContent('content_hash')} AS content FROM script_versions WHERE id = ?`).get(id) as { content: Buffer } | undefined;
+    if (!row?.content) throw new ScriptError(`version content is unavailable (id=${id})`, 'no-such-version');
     return row.content;
 }
 
@@ -241,7 +242,7 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
          VALUES (?, ?, ?, ?, ?)`
     );
     const insertVersion = db().prepare(
-        `INSERT INTO script_versions (script_id, version, sha256, size, content, notes, kind, release_version)
+        `INSERT INTO script_versions (script_id, version, sha256, size, content_hash, notes, kind, release_version)
          VALUES (?, 1, ?, ?, ?, ?, ?, ?)`
     );
 
@@ -250,7 +251,7 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
     db().transaction(() => {
         const s = insertScript.run(input.name, targetRelPath, input.scopeType ?? null, input.scopeValue ?? null, kind);
         scriptId = Number(s.lastInsertRowid);
-        const v = insertVersion.run(scriptId, sha, size, input.content, input.notes ?? null, kind, releaseVersion);
+        const v = insertVersion.run(scriptId, sha, size, putFileBlob(input.content), input.notes ?? null, kind, releaseVersion);
         versionId = Number(v.lastInsertRowid);
     })();
 
@@ -281,10 +282,10 @@ export function uploadNewVersion(scriptId: number, content: Buffer, notes?: stri
 
     const res = db()
         .prepare(
-            `INSERT INTO script_versions (script_id, version, sha256, size, content, notes, kind, release_version)
+            `INSERT INTO script_versions (script_id, version, sha256, size, content_hash, notes, kind, release_version)
              VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
         )
-        .run(scriptId, nextVersion, sha, content.byteLength, content, notes ?? null, script.kind, release);
+        .run(scriptId, nextVersion, sha, content.byteLength, putFileBlob(content), notes ?? null, script.kind, release);
     return getVersion(Number(res.lastInsertRowid))!;
 }
 

src/lib/server/fleet/staged-files.ts

@@ -1,3 +1,4 @@
+import { putFileBlob, blobContent } from '../db/file-blobs.js';
 import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { dirname, relative, resolve, isAbsolute } from 'node:path';
 import { destOfKind, serverDir, type ArtifactKind } from '../config.js';
@@ -39,22 +40,22 @@ function targetPath(server: string, kind: ArtifactKind, relPath: string): string
 /** Capture immutable desired bytes without changing the server or its inventory. */
 export function queueFileChange(change: FileChange): void {
     targetPath(change.server, change.kind, change.relPath);
-    db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content,
+    db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content_hash,
         script_version_id, inventory_source, deployment_file_id, preserve_before_image, check_before, expected_before_sha)
         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         ON CONFLICT(source_kind, source_id, server, kind, rel_path) DO UPDATE SET
-            content = excluded.content, script_version_id = excluded.script_version_id,
+            content_hash = excluded.content_hash, script_version_id = excluded.script_version_id,
             inventory_source = excluded.inventory_source, deployment_file_id = excluded.deployment_file_id,
             preserve_before_image = excluded.preserve_before_image, check_before = excluded.check_before,
             expected_before_sha = excluded.expected_before_sha
         WHERE staged_file_changes.state = 'pending'`)
-        .run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, change.content,
+        .run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, putFileBlob(change.content),
             change.scriptVersionId ?? null, change.inventorySource, change.deploymentFileId ?? null,
             change.preserveBeforeImage ? 1 : 0, change.expectedBeforeSha !== undefined ? 1 : 0, change.expectedBeforeSha ?? null);
 }
 
 export function stagedFileChanges(sourceKind: SourceKind, sourceId: string, server?: string): StagedFileChange[] {
-    return db().prepare(`SELECT * FROM staged_file_changes WHERE source_kind = ? AND source_id = ?
+    return db().prepare(`SELECT *, ${blobContent('content_hash')} AS content FROM staged_file_changes WHERE source_kind = ? AND source_id = ?
         ${server === undefined ? '' : 'AND server = ?'} ORDER BY id`)
         .all(sourceKind, sourceId, ...(server === undefined ? [] : [server])) as StagedFileChange[];
 }
@@ -81,13 +82,13 @@ function readExisting(path: string): Buffer | null {
 }
 
 /** The first activation owns the rollback image, including an absent file. */
-export function captureBetaBeforeImage(row: StagedFileChange, before: Buffer | null): void {
+export function captureBetaBeforeImage(row: Pick<StagedFileChange, 'source_id' | 'server' | 'rel_path' | 'kind'>, before: Buffer | null): void {
     if (db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?')
         .get(row.source_id, row.server, row.rel_path)) return;
     const prior = getServerScript(row.server, row.rel_path);
     if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots
-        (beta_id, server, rel_path, prior_content, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
-        .run(row.source_id, row.server, row.rel_path, before, sha256Buffer(before), before.length);
+        (beta_id, server, rel_path, prior_content_hash, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
+        .run(row.source_id, row.server, row.rel_path, putFileBlob(before), sha256Buffer(before), before.length);
     db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
         AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`)
         .run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, row.server, row.rel_path, row.kind);
@@ -100,7 +101,7 @@ export function captureBetaBeforeImage(row: StagedFileChange, before: Buffer | n
  * from interleaving between the final gates and dispatching the restart.
  */
 export function applyStagedFilesForOperation(operationId: string, server: string): void {
-    const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s
+    const rows = db().prepare(`SELECT f.*, ${blobContent('f.content_hash')} AS content FROM staged_file_changes f JOIN provisioning_sources s
         ON s.kind = f.source_kind AND s.source_id = f.source_id AND s.server = f.server
         WHERE s.operation_id = ? AND f.server = ? AND f.state IN ('pending','applying') ORDER BY f.id`)
         .all(operationId, server) as StagedFileChange[];
@@ -132,8 +133,8 @@ export function applyStagedFilesForOperation(operationId: string, server: string
             }
             if (row.deployment_file_id && !row.preserve_before_image) {
                 db().prepare(`UPDATE deployment_files SET
-                    verb = ?, content_before = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`)
-                    .run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before,
+                    verb = ?, content_before_hash = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`)
+                    .run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', putFileBlob(before),
                         before === null ? null : sha256Buffer(before), row.deployment_file_id);
                 if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
                     .run(row.source_id, `${server}: captured rollback snapshot for ${row.rel_path}`);

src/lib/server/hosts/index.ts

@@ -1,3 +1,4 @@
+import { packFileJson, unpackFileJson, referenceBlobs } from '../db/file-blobs.js';
 import { pruneHistory } from '../retention.js';
 import { recordAgentReceipt } from '../provisioning/agent-events.js';
 import { EventEmitter } from 'node:events';
@@ -396,7 +397,7 @@ export type HostOperationStatus = Omit<HostOperation, 'payload'>;
 const operationStatusColumns = 'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at';
 const operationPayloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
 function operationStatusFromRow(row: Omit<OperationRow, 'payload'>): HostOperationStatus { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
-function operationFromRow(row: OperationRow): HostOperation { return { ...operationStatusFromRow(row), payload: JSON.parse(row.payload) }; }
+function operationFromRow(row: OperationRow): HostOperation { const status = operationStatusFromRow(row); return { ...status, result: unpackFileJson(status.result), payload: unpackFileJson(JSON.parse(row.payload)) }; }
 export function getHostOperation(id: string): HostOperation | null {
     const row = database().prepare(`SELECT ${operationStatusColumns},${operationPayloadSql} AS payload FROM ec_host_operations o WHERE o.id=?`)
         .get(id) as OperationRow | undefined;
@@ -474,20 +475,24 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
                 FROM ec_host_operations o WHERE o.host_id=? AND o.idempotency_key=?`).get(payload, hostId, options.idempotencyKey) as
                 { id: string; kind: string; instance_id: string | null; generation: number | null; history_expired_at: string | null; request_hash: string | null; same_payload: number } | undefined;
             if (existing) {
-                const samePayload = existing.history_expired_at ? existing.request_hash === createHash('sha256').update(payload).digest('hex') : !!existing.same_payload;
+                const samePayload = existing.request_hash ? existing.request_hash === createHash('sha256').update(payload).digest('hex') : !!existing.same_payload;
                 if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || !samePayload) throw new Error('Operation idempotency key is already associated with a different request.');
                 return getHostOperationStatus(existing.id)!;
             }
         }
         if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
         if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.');
+        const requestHash = createHash('sha256').update(payload).digest('hex');
+        const packed = packFileJson(JSON.parse(payload.toString()));
+        payload = JSON.stringify(packed.value);
         const id = options.id ?? randomUUID(); const timestamp = now();
         const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?')
             .get(options.instanceId) as { server: string } | undefined)?.server ?? null : null;
-        database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name)
-            VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?)`)
+        database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name,request_hash)
+            VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?,?)`)
             .run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null,
-                routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName);
+                routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName, requestHash);
+        referenceBlobs('operation', id, packed.hashes);
         if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload);
         database().prepare(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END,
             target_path=${operationTargetSql} WHERE o.id=?`).run(id);
@@ -562,7 +567,8 @@ export function getHostOperationPayload(hostId: string, id: string): Buffer {
     const row = database().prepare(`SELECT CAST(${operationPayloadSql} AS BLOB) AS payload FROM ec_host_operations o WHERE o.id=? AND o.host_id=?`)
         .get(id, hostId) as { payload: Buffer } | undefined;
     if (!row) throw new Error('Host operation was not found for this host.');
-    return row.payload;
+    if (!row.payload.includes('"contentSha256"') || getHost(hostId)?.inventory.capabilities?.includes('files.blobs')) return row.payload;
+    return Buffer.from(JSON.stringify(unpackFileJson(JSON.parse(row.payload.toString()))));
 }
 export function reportHostOperationStatus(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperationStatus {
     const operation = database().transaction(() => {
@@ -575,8 +581,13 @@ export function reportHostOperationStatus(hostId: string, id: string, input: { s
         const state = input.state as HostOperationState;
         if (operation.state === 'running' && state === 'accepted') return operation;
         const error = input.error === undefined || input.error === null ? null : String(input.error).slice(0, 4000);
-        const result = input.result === undefined ? null : JSON.stringify(input.result);
+        let result = input.result === undefined ? null : JSON.stringify(input.result);
         if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
+        if (result !== null) {
+            const packed = packFileJson(input.result);
+            result = JSON.stringify(packed.value);
+            referenceBlobs('operation-result', id, packed.hashes);
+        }
         const observation = state === 'completed' && !error && successfulObservation(operation.kind,
             operation.kind === 'rcon.execute' ? JSON.parse(getHostOperationPayload(hostId, id).toString('utf8')) : {}, input.result);
         database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=?,observation_success=? WHERE id=?')

src/lib/server/hosts/schema.ts

@@ -1,3 +1,5 @@
+import { createHash } from 'node:crypto';
+import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js';
 import type Database from 'better-sqlite3';
 import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js';
 
@@ -6,6 +8,7 @@ export function ensureHostSchema(handle: Database.Database): void {
     if (initialized.has(handle)) return;
     // Background readers open only after controller migrations have completed.
     if (handle.readonly) { initialized.add(handle); return; }
+    ensureFileBlobSchema(handle);
     handle.exec(`
         CREATE TABLE IF NOT EXISTS ec_hosts (
             id TEXT PRIMARY KEY, name TEXT NOT NULL, hostname TEXT NOT NULL DEFAULT '',
@@ -150,5 +153,37 @@ export function ensureHostSchema(handle: Database.Database): void {
             handle.exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)');
         })();
     }
+    if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('operation-file-blobs')) {
+        handle.transaction(() => {
+            for (const row of migrationRows<{ id: string; payload: string; result: string | null }>(handle,
+                `SELECT o.rowid AS rid,o.id,COALESCE(p.payload,o.payload) payload,o.result
+                FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id
+                WHERE o.rowid>? AND o.kind IN ('files.stage','instance.create','instance.rebuild','instance.restart','instance.start','files.read')
+                ORDER BY o.rowid LIMIT 1`)) {
+                const packed = packFileJson(JSON.parse(row.payload), handle);
+                handle.prepare('UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?) WHERE id=?')
+                    .run(createHash('sha256').update(row.payload).digest('hex'), row.id);
+                if (packed.hashes.size) {
+                    handle.prepare(`INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)
+                        ON CONFLICT(operation_id) DO UPDATE SET payload=excluded.payload`).run(row.id, JSON.stringify(packed.value));
+                    handle.prepare("UPDATE ec_host_operations SET payload=json_remove(payload,'$.files') WHERE id=?").run(row.id);
+                    referenceBlobs('operation', row.id, packed.hashes, handle);
+                }
+                if (row.result) {
+                    const result = packFileJson(JSON.parse(row.result), handle);
+                    if (result.hashes.size) {
+                        handle.prepare('UPDATE ec_host_operations SET result=? WHERE id=?').run(JSON.stringify(result.value), row.id);
+                        referenceBlobs('operation-result', row.id, result.hashes, handle);
+                    }
+                }
+            }
+            handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('operation-file-blobs');
+        })();
+    }
+    handle.exec(`CREATE TRIGGER IF NOT EXISTS release_operation_blobs AFTER DELETE ON ec_host_operations BEGIN
+        DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=OLD.id; END;
+        CREATE TRIGGER IF NOT EXISTS expire_operation_blobs AFTER UPDATE OF history_expired_at ON ec_host_operations
+        WHEN NEW.history_expired_at IS NOT NULL BEGIN
+        DELETE FROM file_blob_references WHERE owner_type IN ('operation','operation-result') AND owner_id=NEW.id; END;`);
     initialized.add(handle);
 }

src/lib/server/provisioning/launch-preparation.ts

@@ -20,6 +20,7 @@ export interface NativeLaunchInput {
 export interface PreparedNativeLaunch {
     recipe: LaunchRecipePlan;
     recipeSummary: string;
+    blobs?: Array<{ sha256: string; content: Uint8Array }>;
     recipeJson: Uint8Array;
     payloadJson: Uint8Array;
 }

src/lib/server/provisioning/launch-worker.ts

@@ -29,17 +29,20 @@ parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeL
                 PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`,
                 PLUTO_SERVER_KEY: input.serverKey },
             files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path),
-                contentBase64: file.content, sha256: file.sha256 })),
+                contentSha256: file.sha256, sha256: file.sha256 })),
             ...(input.rebuild ? { nextGeneration: instance.generation + 1, ...(input.safetyOverride ? { safetyOverride: true } : {}) } : {})
         };
         const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value));
-        const recipeJson = encode(recipe);
+        const blobs = [...new Map(recipe.files.map(file => [file.sha256,
+            { sha256: file.sha256, content: new Uint8Array(Buffer.from(file.content, 'base64')) }])).values()];
+        const storedRecipe = { ...recipe, files: recipe.files.map(({ content, ...file }) => ({ ...file, contentHash: file.sha256 })) };
+        const recipeJson = encode(storedRecipe);
         const payloadJson = encode(payload);
         const plan = { ...recipe, files: recipe.files.map(({ content: _content, ...file }) => file) };
-        const recipeSummary = JSON.stringify(summarizeRecipe(plan, sha256Buffer(Buffer.from(recipeJson.buffer))));
+        const recipeSummary = JSON.stringify(summarizeRecipe(plan, sha256Buffer(Buffer.from(JSON.stringify(recipe)))));
         // Transfer ownership of the encoded bytes; do not clone giant strings or objects.
-        parentPort!.postMessage({ id, value: { recipe: plan, recipeSummary, recipeJson, payloadJson } },
-            [recipeJson.buffer, payloadJson.buffer]);
+        parentPort!.postMessage({ id, value: { recipe: plan, recipeSummary, recipeJson, payloadJson, blobs } },
+            [recipeJson.buffer, payloadJson.buffer, ...blobs.map(blob => blob.content.buffer)]);
     } catch (error) {
         parentPort!.postMessage({ id, error: error instanceof Error ? error.message : 'Launch preparation failed.' });
     }

src/lib/server/provisioning/native.ts

@@ -1,3 +1,4 @@
+import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js';
 import { concurrencyBlocker, preparingJobs } from './concurrency.js';
 import { isIP } from 'node:net';
 import { setImmediate as yieldToController } from 'node:timers/promises';
@@ -215,13 +216,14 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer
     } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
 }
 
-function filesFor(job: Job): Array<FilePlan & { contentBase64: string | null }> {
-    const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s
+function filesFor(job: Job): FilePlan[] {
+    const rows = db().prepare(`SELECT f.id,f.kind,f.rel_path,f.content_hash,f.check_before,f.expected_before_sha
+        FROM staged_file_changes f JOIN provisioning_sources s
         ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
-        WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as StagedFileChange[];
+        WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as Array<
+            Pick<StagedFileChange, 'id' | 'kind' | 'rel_path' | 'check_before' | 'expected_before_sha'> & { content_hash: string | null }>;
     return rows.map(row => ({ id: row.id, path: posix.join(relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), row.rel_path),
-        contentBase64: row.content?.toString('base64') ?? null, sha256: row.content ? sha256Buffer(row.content) : null,
-        checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha }));
+        sha256: row.content_hash, checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha }));
 }
 
 function fileMetadata({ contentBase64, ...file }: FilePlan & { contentBase64?: string | null }): FilePlan { return file; }
@@ -236,27 +238,29 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
     if (job.applied_at) return;
     const plans = storedFiles(job);
     if (!plans.length) return;
-    const before = result.before as { id?: number; path: string; contentBase64: string | null }[] | undefined;
+    const before = result.before as { id?: number; path: string; contentBase64: string | null; contentSha256?: string }[] | undefined;
     db().transaction(() => {
         for (const plan of plans) {
             const snapshot = before?.find(item => item.id === plan.id || item.path === plan.path);
             if (!snapshot) throw new Error('Agent activation receipt is missing a before image.');
-            const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange;
+            const row = db().prepare(`SELECT *, ${blobSize('content_hash')} AS size FROM staged_file_changes WHERE id=?`).get(plan.id) as
+                Omit<StagedFileChange, 'content'> & { content_hash: string | null; size: number };
             if (!row || row.state === 'applied') continue;
-            const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64');
+            const bytes = snapshot.contentSha256 ? getFileBlob(snapshot.contentSha256)
+                : snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64');
             if (row.source_kind === 'beta') {
                 captureBetaBeforeImage(row, bytes);
                 db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server);
             }
             if (row.deployment_file_id && !row.preserve_before_image) {
-                db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=?,snapshot_pending=0 WHERE id=?')
-                    .run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
+                db().prepare('UPDATE deployment_files SET verb=?,content_before_hash=?,sha256_before=?,snapshot_pending=0 WHERE id=?')
+                    .run(row.content_hash === null ? 'removed' : bytes === null ? 'added' : 'updated', putFileBlob(bytes), bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
                 if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
                     .run(row.source_id, `${job.server}: captured rollback snapshot for ${row.rel_path}`);
             }
-            if (row.content === null) removeScript(job.server, row.rel_path);
+            if (row.content_hash === null) removeScript(job.server, row.rel_path);
             else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null,
-                sha256Buffer(row.content), row.content.length, row.inventory_source);
+                row.content_hash, row.size, row.inventory_source);
             db().prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?").run(new Date().toISOString(), row.id);
         }
         db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
@@ -596,7 +600,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     const files = job.stage_operation_id ? storedFiles(job) : filesFor(job);
     if (files.length && !job.stage_operation_id) {
         db().transaction(() => {
-            const stage = queueHostOperationSerialized(instance.host_id, 'files.stage', Buffer.from(JSON.stringify({ stageId: job.id, files })),
+            const stage = queueHostOperationSerialized(instance.host_id, 'files.stage', Buffer.from(JSON.stringify({ stageId: job.id, files: files.map(file => ({ ...file, ...(file.sha256 ? { contentSha256: file.sha256 } : { contentBase64: null }) })) })),
                 { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:stage` }, { stageId: job.id });
             db().prepare('UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?')
                 .run(stage.id, JSON.stringify(files.map(fileMetadata)), job.id);
@@ -713,6 +717,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (prepared) {
                 deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
                     checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
+                for (const blob of prepared.blobs ?? []) putFileBlob(blob.content);
                 const recipe = prepared.recipe;
                 db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?')
                     .run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id);

src/lib/server/retention.ts

@@ -1,3 +1,4 @@
+import { blobSize, pruneFileBlobs } from './db/file-blobs.js';
 import { createHash } from 'node:crypto';
 import { db } from './db/index.js';
 import { fleetSettings } from './settings.js';
@@ -42,7 +43,7 @@ function policies(): Policy[] {
     return [
         { category: 'campaign', label: 'Campaigns', table: 'deployments', id: 'id', clock: 'started_at', days: 'deployKeepDays',
             eligible: `${campaignSafe} AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='file-content' AND c.record_id=o.id AND c.protected=1)`, failed: failedCampaign,
-            bytes: `(SELECT COALESCE(SUM(COALESCE(length(content_before),0)+COALESCE(length(content_after),0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
+            bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
         { category: 'provisioning', label: 'Provisioning workflows', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'provisioningKeepDays', failed: "o.state='failed'", eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
             AND NOT EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=o.id AND r.healthy_at IS NULL)
             AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id) WHERE n.deployment_id=o.id AND a.state NOT IN ('completed','failed'))
@@ -70,22 +71,22 @@ function policies(): Policy[] {
         { category: 'beta', label: 'Completed betas', table: 'betas', id: 'id', clock: 'closed_at', days: 'betaKeepDays',
             eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta-file-content' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id)
                 AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
-            bytes: '(SELECT COALESCE(SUM(length(content)),0) FROM beta_revision_files WHERE beta_id=o.id)' },
+            bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` },
         { category: 'file-content', label: 'Campaign file bodies', table: 'deployments', id: 'id', clock: 'finished_at', days: 'fileContentKeepDays',
             eligible: `${campaignSafe} AND NOT (${failedCampaign})
                 AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=o.id AND c.protected=1)
                 AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`,
-            bytes: '(SELECT COALESCE(SUM(COALESCE(length(content_before),0)+COALESCE(length(content_after),0)),0) FROM deployment_files WHERE deployment_id=o.id)' },
+            bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` },
         { category: 'beta-file-content', label: 'Closed beta file snapshots', table: 'betas', id: 'id', clock: 'closed_at', days: 'fileContentKeepDays',
             eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id)
                 AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1)
                 AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT))
                 AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`,
-            bytes: '(SELECT COALESCE(SUM(length(content)),0) FROM beta_revision_files WHERE beta_id=o.id)' },
+            bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` },
         { category: 'version', label: 'Unused artifact versions', table: 'script_versions', id: 'id', clock: 'uploaded_at', days: 'versionKeepDays', minimum: false,
             eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id)
                 AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id)
-                AND ${versionReferences()}`, bytes: 'length(o.content)' },
+                AND ${versionReferences()}`, bytes: `${blobSize('o.content_hash')}` },
         { category: 'audit', label: 'Authentication audit events', table: 'auth_audit', id: 'id', clock: 'created_at', days: 'auditKeepDays', eligible: '1', bytes: 'length(o.details)' }
     ];
 }
@@ -181,18 +182,18 @@ export function pruneHistory(only?: string): number {
                     db().prepare('DELETE FROM ec_host_operation_events WHERE operation_id=?').run(row.id);
                     if (policy.category === 'operation') {
                         const payload = db().prepare(`SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(row.id) as { payload: string };
-                        db().prepare(`UPDATE ec_host_operations SET request_hash=?,payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`)
+                        db().prepare(`UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?),payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`)
                             .run(createHash('sha256').update(payload.payload).digest('hex'),new Date().toISOString(),row.id);
                     }
                     db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(row.id);
                 }
                 if (policy.category === 'beta-file-content') {
-                    db().prepare("UPDATE beta_revision_files SET content=X'' WHERE beta_id=?").run(row.id);
+                    db().prepare("UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?").run(row.id);
                     db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id=?').run(row.id);
                     db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
                 } else if (policy.category === 'file-content') {
-                    db().prepare('UPDATE deployment_files SET content_before=NULL,content_after=NULL WHERE deployment_id=?').run(row.id);
-                    db().prepare("UPDATE staged_file_changes SET content=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')").run(row.id);
+                    db().prepare('UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?').run(row.id);
+                    db().prepare("UPDATE staged_file_changes SET content_hash=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')").run(row.id);
                     db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString());
                 } else if (policy.category !== 'operation') {
                     db().prepare(`DELETE FROM ${policy.table} WHERE rowid=?`).run(row.rid);
@@ -202,6 +203,7 @@ export function pruneHistory(only?: string): number {
             return rows.length;
         })();
     }
+    pruneFileBlobs();
     return removed;
 }
 

src/routes/api/agent/blobs/[sha256]/+server.ts

@@ -0,0 +1,17 @@
+import { error } from '@sveltejs/kit';
+import { authenticateAgent } from '$lib/server/hosts/http.js';
+import { db } from '$lib/server/db/index.js';
+
+export function GET({ request, params }: { request: Request; params: { sha256: string } }) {
+    const host = authenticateAgent(request);
+    if (!/^[a-f0-9]{64}$/.test(params.sha256)) error(404, 'File not found.');
+    const row = db().prepare(`SELECT b.content,b.size FROM file_blobs b WHERE b.sha256=? AND EXISTS (
+        SELECT 1 FROM file_blob_references r JOIN ec_host_operations o ON o.id=r.owner_id
+        WHERE r.sha256=b.sha256 AND r.owner_type='operation' AND o.host_id=?)`).get(params.sha256, host.id) as
+        { content: Buffer; size: number } | undefined;
+    if (!row) error(404, 'File not found.');
+    return new Response(new Uint8Array(row.content), { headers: {
+        'Content-Type': 'application/octet-stream', 'Content-Length': String(row.size),
+        'Cache-Control': 'private, no-store', ETag: `"${params.sha256}"`
+    } });
+}

src/routes/api/betas/[id]/revisions/[revision]/files/[scriptId]/+server.ts

@@ -1,3 +1,4 @@
+import { blobContent } from '$lib/server/db/file-blobs.js';
 import { db } from '$lib/server/db/index.js';
 
 /** Current files can be reviewed before the first revision is archived. */
@@ -5,12 +6,12 @@ export async function GET({ params }: { params: { id: string; revision: string;
     const values = [params.id, params.revision, params.scriptId].map(Number);
     if (values.some(value => !Number.isSafeInteger(value) || value < 1)) return new Response('Invalid file', { status: 400 });
     const [id, revision, scriptId] = values;
-    const file = db().prepare('SELECT content FROM beta_revision_files WHERE beta_id = ? AND revision = ? AND script_id = ?')
+    const file = db().prepare(`SELECT ${blobContent('content_hash')} AS content FROM beta_revision_files WHERE beta_id = ? AND revision = ? AND script_id = ?`)
         .get(id, revision, scriptId) as { content: Buffer } | undefined
-        ?? (revision === 1 ? db().prepare(`SELECT v.content FROM beta_scripts bs JOIN script_versions v ON v.id = bs.script_version_id
+        ?? (revision === 1 ? db().prepare(`SELECT ${blobContent('v.content_hash')} AS content FROM beta_scripts bs JOIN script_versions v ON v.id = bs.script_version_id
             WHERE bs.beta_id = ? AND bs.script_id = ? AND NOT EXISTS (SELECT 1 FROM beta_revisions WHERE beta_id = ?)`)
             .get(id, scriptId, id) as { content: Buffer } | undefined : undefined);
-    if (!file) return new Response('File not found', { status: 404 });
+    if (!file?.content) return new Response('File not found', { status: 404 });
     return new Response(new Uint8Array(file.content), { headers: { 'Content-Type': 'application/octet-stream',
         'Content-Disposition': 'attachment', 'Cache-Control': 'no-store' } });
 }

src/routes/api/campaigns/[id]/+server.ts

@@ -1,3 +1,4 @@
+import { blobSize } from '$lib/server/db/file-blobs.js';
 import { json, error } from '@sveltejs/kit';
 import { db } from '$lib/server/db/index.js';
 
@@ -16,8 +17,8 @@ export async function GET({ params }: { params: { id: string } }) {
     const files = db()
         .prepare(
             `SELECT id, server, verb, rel_path, sha256_before, sha256_after, snapshot_pending,
-                    LENGTH(content_before) AS size_before,
-                    LENGTH(content_after)  AS size_after,
+                    ${blobSize('content_before_hash')} AS size_before,
+                    ${blobSize('content_after_hash')}  AS size_after,
                     COALESCE((SELECT state FROM staged_file_changes WHERE deployment_file_id = deployment_files.id), 'applied') AS apply_state
              FROM deployment_files
              WHERE deployment_id = ?

src/routes/api/campaigns/[id]/file/[fileId]/+server.ts

@@ -1,3 +1,4 @@
+import { blobContent } from '$lib/server/db/file-blobs.js';
 import { json, error } from '@sveltejs/kit';
 import { db } from '$lib/server/db/index.js';
 
@@ -10,7 +11,7 @@ export async function GET({ params }: { params: { id: string; fileId: string } }
     }
     const row = db()
         .prepare(
-            `SELECT verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending
+            `SELECT verb, rel_path, sha256_before, sha256_after, ${blobContent('content_before_hash')} AS content_before, ${blobContent('content_after_hash')} AS content_after, snapshot_pending
              FROM deployment_files
              WHERE deployment_id = ? AND id = ?`
         )

src/routes/api/deployments/[id]/+server.ts

@@ -1,3 +1,4 @@
+import { blobSize } from '$lib/server/db/file-blobs.js';
 import { json, error } from '@sveltejs/kit';
 import { db } from '$lib/server/db/index.js';
 import { getDeployment } from '$lib/server/provisioning/index.js';
@@ -19,8 +20,8 @@ export async function GET({ params, url }: { params: { id: string }; url?: URL }
     const files = db()
         .prepare(
             `SELECT id, server, verb, rel_path, sha256_before, sha256_after,
-                    LENGTH(content_before) AS size_before,
-                    LENGTH(content_after)  AS size_after
+                    ${blobSize('content_before_hash')} AS size_before,
+                    ${blobSize('content_after_hash')}  AS size_after
              FROM deployment_files
              WHERE deployment_id = ?
              ORDER BY server, rel_path`

src/routes/api/deployments/[id]/file/[fileId]/+server.ts

@@ -1,3 +1,4 @@
+import { blobContent } from '$lib/server/db/file-blobs.js';
 import { json, error } from '@sveltejs/kit';
 import { db } from '$lib/server/db/index.js';
 
@@ -7,7 +8,7 @@ import { db } from '$lib/server/db/index.js';
 export async function GET({ params }: { params: { id: string; fileId: string } }) {
     const row = db()
         .prepare(
-            `SELECT verb, rel_path, sha256_before, sha256_after, content_before, content_after
+            `SELECT verb, rel_path, sha256_before, sha256_after, ${blobContent('content_before_hash')} AS content_before, ${blobContent('content_after_hash')} AS content_after
              FROM deployment_files
              WHERE deployment_id = ? AND id = ?`
         )