josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Deduplicate artifact bytes and deliver files by authenticated hash

Browse Source
This commit is contained in:
josh
2026-10-06 14:35:13 -04:00
parent 18283c63a7
commit 62df14b180
45 changed files with 510 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/db/file-blobs.ts
+123
View File
@@ -0,0 +1,123 @@
import { createHash } from 'node:crypto';
import type Database from 'better-sqlite3';
import { db } from './index.js';
export function ensureFileBlobSchema(handle: Database.Database): void {
handle.exec(`CREATE TABLE IF NOT EXISTS file_blobs(sha256 TEXT PRIMARY KEY,size INTEGER NOT NULL,content BLOB NOT NULL);
CREATE TABLE IF NOT EXISTS file_blob_references(owner_type TEXT NOT NULL,owner_id TEXT NOT NULL,
sha256 TEXT NOT NULL REFERENCES file_blobs(sha256),PRIMARY KEY(owner_type,owner_id,sha256));
CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256);`);
}
export function putFileBlob(content: Uint8Array | null, handle: Database.Database = db()): string | null {
if (content === null) return null;
const sha256 = createHash('sha256').update(content).digest('hex');
handle.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)').run(sha256, content.byteLength, content);
return sha256;
}
export function getFileBlob(sha256: string, handle: Database.Database = db()): Buffer {
const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as { content: Buffer } | undefined;
if (!row) throw new Error('File content is unavailable or has expired.');
return row.content;
}
/** SQL fragments are for fixed schema identifiers, never user input. */
export const blobContent = (column: string) => `(SELECT content FROM file_blobs WHERE sha256=${column})`;
export const blobSize = (column: string) => `(SELECT size FROM file_blobs WHERE sha256=${column})`;
export function referenceBlobs(ownerType: string, ownerId: string, hashes: Iterable<string>, handle: Database.Database = db()): void {
handle.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?').run(ownerType, ownerId);
const insert = handle.prepare('INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)');
for (const hash of hashes) insert.run(ownerType, ownerId, hash);
}
/** Immutable file bytes in JSON are replaced with hash references, including before images. */
export function packFileJson(value: unknown, handle: Database.Database = db(), hashes = new Set<string>()): { value: unknown; hashes: Set<string> } {
if (Array.isArray(value)) return { value: value.map(item => packFileJson(item, handle, hashes).value), hashes };
if (!value || typeof value !== 'object') return { value, hashes };
const result: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) {
if ((key === 'contentBase64' || key === 'content' && 'path' in value && 'sha256' in value) && typeof item === 'string') {
const hash = putFileBlob(Buffer.from(item, 'base64'), handle)!;
result[key === 'content' ? 'contentHash' : 'contentSha256'] = hash;
hashes.add(hash);
} else {
result[key] = packFileJson(item, handle, hashes).value;
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') hashes.add(item);
}
}
return { value: result, hashes };
}
export function unpackFileJson(value: unknown, handle: Database.Database = db()): any {
if (Array.isArray(value)) return value.map(item => unpackFileJson(item, handle));
if (!value || typeof value !== 'object') return value;
const result: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) {
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') {
result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(item, handle).toString('base64');
} else result[key] = unpackFileJson(item, handle);
}
return result;
}
export const blobColumns = [
['script_versions', 'content'], ['staged_file_changes', 'content'],
['deployment_files', 'content_before'], ['deployment_files', 'content_after'],
['beta_file_snapshots', 'prior_content'], ['beta_revision_files', 'content']
] as const;
/** Finish each SELECT before writing; better-sqlite3 iterators hold an active cursor. */
export function* migrationRows<T>(handle: Database.Database, sql: string): Generator<T & { rid: number }> {
const select = handle.prepare(sql);
let cursor = -Number.MAX_SAFE_INTEGER;
for (;;) {
const row = select.get(cursor) as (T & { rid: number }) | undefined;
if (!row) return;
cursor = row.rid;
yield row;
}
}
/** Backfill one row at a time so large libraries do not exhaust startup memory. */
export function migrateFileBlobs(handle: Database.Database): void {
for (const [table, column] of blobColumns) {
if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue;
handle.transaction(() => {
handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`);
const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) {
update.run(putFileBlob(row.content, handle), row.rid);
}
handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
})();
}
for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`);
if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) {
handle.transaction(() => {
for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle,
'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) {
const packed = packFileJson(JSON.parse(row.recipe), handle);
handle.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(packed.value), row.deployment_id);
referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
}
handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
})();
}
handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN
DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`);
for (const action of ['INSERT', 'UPDATE OF recipe']) handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'}
AFTER ${action} ON server_launch_recipes BEGIN
DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=NEW.deployment_id;
INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256)
SELECT 'recipe',NEW.deployment_id,json_extract(value,'$.contentHash') FROM json_each(NEW.recipe,'$.files')
WHERE json_extract(value,'$.contentHash') IS NOT NULL; END;`);
}
export function pruneFileBlobs(limit = 100): number {
return db().prepare(`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b
WHERE NOT EXISTS (SELECT 1 FROM file_blob_references r WHERE r.sha256=b.sha256)
${blobColumns.map(([table, column]) => `AND NOT EXISTS (SELECT 1 FROM ${table} WHERE ${column}_hash=b.sha256)`).join('\n')}
LIMIT ?)`).run(limit).changes;
}
src/lib/server/db/index.ts
+2
View File
@@ -12,6 +12,7 @@ import { migrateServerHistory } from './server-history.js';
import { migrateNativeInstances } from '../backends/schema.js';
import { installPerformanceIndexes } from './performance-indexes.js';
import { migrateVersionLayout } from './version-layout.js';
import { migrateFileBlobs } from './file-blobs.js';
import { tuneConnection } from './connection.js';
import { timeStatement } from './slow-statements.js';
@@ -393,6 +394,7 @@ export function db(): Database.Database {
initializeFleetSettings(handle);
migrateNativeInstances(handle);
migrateVersionLayout(handle);
migrateFileBlobs(handle);
// Lifecycle history is retained indefinitely. Admission and status reads
// must narrow by server or active state instead of scanning that history.
// Install after migrations that rebuild this table or add server_deleted_at.
src/lib/server/db/schema.sql
+16 -6
View File
@@ -1,3 +1,13 @@
-- Immutable file bodies are shared across versions, campaigns, betas and host commands.
CREATE TABLE IF NOT EXISTS file_blobs (
sha256 TEXT PRIMARY KEY, size INTEGER NOT NULL, content BLOB NOT NULL
);
CREATE TABLE IF NOT EXISTS file_blob_references (
owner_type TEXT NOT NULL, owner_id TEXT NOT NULL, sha256 TEXT NOT NULL REFERENCES file_blobs(sha256),
PRIMARY KEY(owner_type,owner_id,sha256)
);
CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256);
-- Electric Cherry schema. Idempotent — safe to run at every boot via migrate.ts.
-- The old betas/beta_scripts/beta_deploys tables are dropped in-code before this
-- runs (see dropLegacyBetaTables in db/index.ts); the new script-centric model
@@ -134,7 +144,7 @@ CREATE TABLE IF NOT EXISTS script_versions (
state TEXT NOT NULL DEFAULT 'untested', -- 'untested' | 'tested' | 'test-failed'
kind TEXT NOT NULL DEFAULT 'script', -- denormalized from scripts.kind for join-free filtering
uploaded_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
content BLOB NOT NULL,
content_hash TEXT REFERENCES file_blobs(sha256),
UNIQUE(script_id, version)
);
CREATE INDEX IF NOT EXISTS idx_script_versions_script ON script_versions(script_id);
@@ -201,7 +211,7 @@ CREATE TABLE IF NOT EXISTS beta_revision_files (
version INTEGER NOT NULL,
release_version TEXT,
sha256 TEXT NOT NULL,
content BLOB NOT NULL,
content_hash TEXT REFERENCES file_blobs(sha256),
PRIMARY KEY (beta_id, revision, script_id),
FOREIGN KEY (beta_id, revision) REFERENCES beta_revisions(beta_id, revision) ON DELETE CASCADE
);
@@ -306,7 +316,7 @@ CREATE TABLE IF NOT EXISTS beta_file_snapshots (
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
server TEXT NOT NULL REFERENCES servers(name) ON DELETE CASCADE,
rel_path TEXT NOT NULL,
prior_content BLOB NOT NULL,
prior_content_hash TEXT REFERENCES file_blobs(sha256),
prior_sha256 TEXT NOT NULL,
prior_size INTEGER NOT NULL,
PRIMARY KEY (beta_id, server, rel_path)
@@ -364,8 +374,8 @@ CREATE TABLE IF NOT EXISTS deployment_files (
rel_path TEXT NOT NULL,
sha256_before TEXT,
sha256_after TEXT,
content_before BLOB,
content_after BLOB,
content_before_hash TEXT REFERENCES file_blobs(sha256),
content_after_hash TEXT REFERENCES file_blobs(sha256),
-- An unknown before image is distinct from a confirmed absent file.
snapshot_pending INTEGER NOT NULL DEFAULT 0
);
@@ -382,7 +392,7 @@ CREATE TABLE IF NOT EXISTS staged_file_changes (
server TEXT NOT NULL,
kind TEXT NOT NULL,
rel_path TEXT NOT NULL,
content BLOB,
content_hash TEXT REFERENCES file_blobs(sha256),
script_version_id INTEGER,
inventory_source TEXT NOT NULL,
deployment_file_id INTEGER REFERENCES deployment_files(id) ON DELETE CASCADE,
src/lib/server/db/version-layout.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import type Database from 'better-sqlite3';
/** Keep metadata ahead of overflow pages, preserving IDs and all references. */
export function migrateVersionLayout(handle: Database.Database): void {
const columns = handle.pragma('table_info(script_versions)') as { name: string }[];
if (columns.at(-1)?.name === 'content') return;
if (!columns.some(column => column.name === 'content') || columns.at(-1)?.name === 'content') return;
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'").get() as { sql: string };
const content = /\bcontent\s+BLOB\s+NOT\s+NULL\s*,/i;
if (!content.test(sql)) throw new Error('Unrecognized script version content layout');