josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Deduplicate artifact bytes and deliver files by authenticated hash

Browse Source
This commit is contained in:
josh
2026-10-06 14:35:13 -04:00
parent 18283c63a7
commit 62df14b180
45 changed files with 510 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/beta-revisions.ts
+2 -2
View File
@@ -12,8 +12,8 @@ export function archiveBetaRevision(id: number, revision: number, startedAt: str
db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)')
.run(id, revision, startedAt, notes);
db().prepare(`INSERT OR IGNORE INTO beta_revision_files
(beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content)
SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content
(beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content_hash)
SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content_hash
FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id
WHERE bs.beta_id = ?`).run(revision, id);
}
src/lib/server/fleet/beta.ts
+10 -9
View File
@@ -1,3 +1,4 @@
import { putFileBlob, blobContent } from '../db/file-blobs.js';
import { getVersionContent } from './scripts.js';
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
import { storedPlan, testView, activity, setOutcome } from './beta-tests.js';
@@ -734,7 +735,7 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
const hasCollapsing = bundle.some((b) => isCollapsingKind(b.script.kind));
const getSnapshot = db().prepare(
`SELECT prior_content, prior_sha256, prior_size FROM beta_file_snapshots
`SELECT ${blobContent('prior_content_hash')} AS prior_content, prior_sha256, prior_size FROM beta_file_snapshots
WHERE beta_id = ? AND server = ? AND rel_path = ?`
);
@@ -797,10 +798,10 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
}
for (const change of restorations) {
const file = db().prepare(`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
VALUES (?, ?, ?, ?, NULL, ?, NULL, ?)`).run(deploymentId, change.server,
change.content === null ? 'removed' : 'updated', change.relPath,
change.content === null ? null : sha256Buffer(change.content), change.content);
change.content === null ? null : sha256Buffer(change.content), putFileBlob(change.content));
queueFileChange({ sourceKind: 'campaign', sourceId: deploymentId, server: change.server,
kind: change.kind, relPath: change.relPath, content: change.content,
scriptVersionId: change.versionId, inventorySource: `rollback:${betaId}`,
@@ -969,7 +970,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
const plans: Plan[] = [];
const getSnapshotForPromote = db().prepare(
`SELECT prior_content, prior_sha256 FROM beta_file_snapshots
`SELECT ${blobContent('prior_content_hash')} AS prior_content, prior_sha256 FROM beta_file_snapshots
WHERE beta_id = ? AND server = ? AND rel_path = ?`
);
@@ -1096,7 +1097,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
);
const insertDepFile = db().prepare(
`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending)
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash, snapshot_pending)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
);
@@ -1121,8 +1122,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
p.script.target_rel_path,
p.priorSha,
p.version.sha256,
p.priorContent,
getVersionContent(p.version.id),
putFileBlob(p.priorContent),
putFileBlob(getVersionContent(p.version.id)),
betaServerSet.has(p.server) ? 0 : 1
);
if (plansToActivate.has(p)) {
@@ -1141,8 +1142,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
configFileOf(cp.server),
cp.priorSha,
cp.composedSha,
cp.priorContent,
cp.composedContent,
putFileBlob(cp.priorContent),
putFileBlob(cp.composedContent),
betaServerSet.has(cp.server) ? 0 : 1
);
if (cp.requiresActivation) {
src/lib/server/fleet/drift.ts
+6 -5
View File
@@ -1,3 +1,4 @@
import { putFileBlob } from '../db/file-blobs.js';
import { getVersionContent } from './scripts.js';
import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js';
import { randomUUID } from 'node:crypto';
@@ -401,11 +402,11 @@ export function createDriftRebuildCampaign(
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')")
.run(id, new Date().toISOString());
const insertFile = db().prepare(`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
for (const { file, before, after } of prepared.files) {
insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated',
file.relPath, file.beforeSha256, file.afterSha256, before, after);
file.relPath, file.beforeSha256, file.afterSha256, putFileBlob(before), putFileBlob(after));
}
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId);
@@ -480,7 +481,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
);
const insertFile = db().prepare(
`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
);
@@ -513,7 +514,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
const content = await readFile(abs);
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, content, null);
const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, putFileBlob(content), null);
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
@@ -538,7 +539,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath,
before ? sha256Buffer(before) : null, sha256Buffer(written), before, written);
before ? sha256Buffer(before) : null, sha256Buffer(written), putFileBlob(before), putFileBlob(written));
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: written,
scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null,
src/lib/server/fleet/launch-recipes.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { packFileJson, unpackFileJson } from '../db/file-blobs.js';
import { randomUUID } from 'node:crypto';
import type { BlueprintSummary, LaunchRecipeSummary, LaunchFileSummary } from '$lib/deployment-plan.js';
import { config, configFileForGame } from '../config.js';
@@ -118,7 +119,7 @@ export function summarizeRecipe(recipe: LaunchRecipePlan, fingerprint = sha256Bu
/** Keep the small detail response beside the immutable attempted file bytes. */
export function recipeStorage(recipe: LaunchRecipePlan): { recipe: string; recipeSummary: string } {
const encoded = JSON.stringify(recipe);
return { recipe: encoded, recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) };
return { recipe: JSON.stringify(packFileJson(recipe).value), recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) };
}
export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
@@ -137,7 +138,7 @@ export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSu
export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
const row = db().prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
.get(id) as { recipe: string; blueprint_name: string | null } | undefined;
return row ? { recipe: JSON.parse(row.recipe), blueprintName: row.blueprint_name } : null;
return row ? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name } : null;
}
export function blueprintRecipe(id: string): { recipe: LaunchRecipePlan; blueprintName: string } {
src/lib/server/fleet/rollback.ts
+5 -4
View File
@@ -1,3 +1,4 @@
import { putFileBlob, blobContent } from '../db/file-blobs.js';
import type { ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
@@ -101,7 +102,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
const fileRows = db()
.prepare(
`SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, f.content_before, f.content_after, f.snapshot_pending
`SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, ${blobContent('f.content_before_hash')} AS content_before, ${blobContent('f.content_after_hash')} AS content_after, f.snapshot_pending
FROM deployment_files f JOIN deployment_servers ds ON ds.deployment_id = f.deployment_id AND ds.server = f.server
JOIN servers s ON s.name = f.server
WHERE f.deployment_id = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL ORDER BY f.server, f.rel_path`
@@ -184,7 +185,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
);
const insertDepFile = db().prepare(
`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
);
@@ -226,8 +227,8 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
f.rel_path,
f.sha256_after,
f.sha256_before,
f.content_after,
f.content_before
putFileBlob(f.content_after),
putFileBlob(f.content_before)
);
const owningScript = scriptByTarget.get(f.rel_path);
const priorVersion = owningScript?.prior_version_id ? getVersion(owningScript.prior_version_id) : null;
src/lib/server/fleet/scripts.ts
+7 -6
View File
@@ -1,3 +1,4 @@
import { putFileBlob, blobContent } from '../db/file-blobs.js';
import { createHash } from 'node:crypto';
import { db } from '../db/index.js';
import { configFileForGame, type ArtifactKind } from '../config.js';
@@ -139,8 +140,8 @@ export function listPromotedOverlaysForScope(
export const VERSION_COLUMNS = 'id, script_id, version, release_version, sha256, size, notes, state, kind, uploaded_at';
export function getVersionContent(id: number): Buffer {
const row = db().prepare('SELECT content FROM script_versions WHERE id = ?').get(id) as { content: Buffer } | undefined;
if (!row) throw new ScriptError(`no such version (id=${id})`, 'no-such-version');
const row = db().prepare(`SELECT ${blobContent('content_hash')} AS content FROM script_versions WHERE id = ?`).get(id) as { content: Buffer } | undefined;
if (!row?.content) throw new ScriptError(`version content is unavailable (id=${id})`, 'no-such-version');
return row.content;
}
@@ -241,7 +242,7 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
VALUES (?, ?, ?, ?, ?)`
);
const insertVersion = db().prepare(
`INSERT INTO script_versions (script_id, version, sha256, size, content, notes, kind, release_version)
`INSERT INTO script_versions (script_id, version, sha256, size, content_hash, notes, kind, release_version)
VALUES (?, 1, ?, ?, ?, ?, ?, ?)`
);
@@ -250,7 +251,7 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver
db().transaction(() => {
const s = insertScript.run(input.name, targetRelPath, input.scopeType ?? null, input.scopeValue ?? null, kind);
scriptId = Number(s.lastInsertRowid);
const v = insertVersion.run(scriptId, sha, size, input.content, input.notes ?? null, kind, releaseVersion);
const v = insertVersion.run(scriptId, sha, size, putFileBlob(input.content), input.notes ?? null, kind, releaseVersion);
versionId = Number(v.lastInsertRowid);
})();
@@ -281,10 +282,10 @@ export function uploadNewVersion(scriptId: number, content: Buffer, notes?: stri
const res = db()
.prepare(
`INSERT INTO script_versions (script_id, version, sha256, size, content, notes, kind, release_version)
`INSERT INTO script_versions (script_id, version, sha256, size, content_hash, notes, kind, release_version)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(scriptId, nextVersion, sha, content.byteLength, content, notes ?? null, script.kind, release);
.run(scriptId, nextVersion, sha, content.byteLength, putFileBlob(content), notes ?? null, script.kind, release);
return getVersion(Number(res.lastInsertRowid))!;
}
src/lib/server/fleet/staged-files.ts
+11 -10
View File
@@ -1,3 +1,4 @@
import { putFileBlob, blobContent } from '../db/file-blobs.js';
import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
import { dirname, relative, resolve, isAbsolute } from 'node:path';
import { destOfKind, serverDir, type ArtifactKind } from '../config.js';
@@ -39,22 +40,22 @@ function targetPath(server: string, kind: ArtifactKind, relPath: string): string
/** Capture immutable desired bytes without changing the server or its inventory. */
export function queueFileChange(change: FileChange): void {
targetPath(change.server, change.kind, change.relPath);
db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content,
db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content_hash,
script_version_id, inventory_source, deployment_file_id, preserve_before_image, check_before, expected_before_sha)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(source_kind, source_id, server, kind, rel_path) DO UPDATE SET
content = excluded.content, script_version_id = excluded.script_version_id,
content_hash = excluded.content_hash, script_version_id = excluded.script_version_id,
inventory_source = excluded.inventory_source, deployment_file_id = excluded.deployment_file_id,
preserve_before_image = excluded.preserve_before_image, check_before = excluded.check_before,
expected_before_sha = excluded.expected_before_sha
WHERE staged_file_changes.state = 'pending'`)
.run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, change.content,
.run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, putFileBlob(change.content),
change.scriptVersionId ?? null, change.inventorySource, change.deploymentFileId ?? null,
change.preserveBeforeImage ? 1 : 0, change.expectedBeforeSha !== undefined ? 1 : 0, change.expectedBeforeSha ?? null);
}
export function stagedFileChanges(sourceKind: SourceKind, sourceId: string, server?: string): StagedFileChange[] {
return db().prepare(`SELECT * FROM staged_file_changes WHERE source_kind = ? AND source_id = ?
return db().prepare(`SELECT *, ${blobContent('content_hash')} AS content FROM staged_file_changes WHERE source_kind = ? AND source_id = ?
${server === undefined ? '' : 'AND server = ?'} ORDER BY id`)
.all(sourceKind, sourceId, ...(server === undefined ? [] : [server])) as StagedFileChange[];
}
@@ -81,13 +82,13 @@ function readExisting(path: string): Buffer | null {
}
/** The first activation owns the rollback image, including an absent file. */
export function captureBetaBeforeImage(row: StagedFileChange, before: Buffer | null): void {
export function captureBetaBeforeImage(row: Pick<StagedFileChange, 'source_id' | 'server' | 'rel_path' | 'kind'>, before: Buffer | null): void {
if (db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?')
.get(row.source_id, row.server, row.rel_path)) return;
const prior = getServerScript(row.server, row.rel_path);
if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots
(beta_id, server, rel_path, prior_content, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
.run(row.source_id, row.server, row.rel_path, before, sha256Buffer(before), before.length);
(beta_id, server, rel_path, prior_content_hash, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
.run(row.source_id, row.server, row.rel_path, putFileBlob(before), sha256Buffer(before), before.length);
db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`)
.run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, row.server, row.rel_path, row.kind);
@@ -100,7 +101,7 @@ export function captureBetaBeforeImage(row: StagedFileChange, before: Buffer | n
* from interleaving between the final gates and dispatching the restart.
*/
export function applyStagedFilesForOperation(operationId: string, server: string): void {
const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s
const rows = db().prepare(`SELECT f.*, ${blobContent('f.content_hash')} AS content FROM staged_file_changes f JOIN provisioning_sources s
ON s.kind = f.source_kind AND s.source_id = f.source_id AND s.server = f.server
WHERE s.operation_id = ? AND f.server = ? AND f.state IN ('pending','applying') ORDER BY f.id`)
.all(operationId, server) as StagedFileChange[];
@@ -132,8 +133,8 @@ export function applyStagedFilesForOperation(operationId: string, server: string
}
if (row.deployment_file_id && !row.preserve_before_image) {
db().prepare(`UPDATE deployment_files SET
verb = ?, content_before = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`)
.run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before,
verb = ?, content_before_hash = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`)
.run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', putFileBlob(before),
before === null ? null : sha256Buffer(before), row.deployment_file_id);
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${server}: captured rollback snapshot for ${row.rel_path}`);