josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Deduplicate artifact bytes and deliver files by authenticated hash

Browse Source
This commit is contained in:
josh
2026-10-06 14:35:13 -04:00
parent 18283c63a7
commit 62df14b180
45 changed files with 510 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/native.ts
+18 -13
View File
@@ -1,3 +1,4 @@
import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js';
import { concurrencyBlocker, preparingJobs } from './concurrency.js';
import { isIP } from 'node:net';
import { setImmediate as yieldToController } from 'node:timers/promises';
@@ -215,13 +216,14 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
}
function filesFor(job: Job): Array<FilePlan & { contentBase64: string | null }> {
const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s
function filesFor(job: Job): FilePlan[] {
const rows = db().prepare(`SELECT f.id,f.kind,f.rel_path,f.content_hash,f.check_before,f.expected_before_sha
FROM staged_file_changes f JOIN provisioning_sources s
ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as StagedFileChange[];
WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as Array<
Pick<StagedFileChange, 'id' | 'kind' | 'rel_path' | 'check_before' | 'expected_before_sha'> & { content_hash: string | null }>;
return rows.map(row => ({ id: row.id, path: posix.join(relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), row.rel_path),
contentBase64: row.content?.toString('base64') ?? null, sha256: row.content ? sha256Buffer(row.content) : null,
checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha }));
sha256: row.content_hash, checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha }));
}
function fileMetadata({ contentBase64, ...file }: FilePlan & { contentBase64?: string | null }): FilePlan { return file; }
@@ -236,27 +238,29 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
if (job.applied_at) return;
const plans = storedFiles(job);
if (!plans.length) return;
const before = result.before as { id?: number; path: string; contentBase64: string | null }[] | undefined;
const before = result.before as { id?: number; path: string; contentBase64: string | null; contentSha256?: string }[] | undefined;
db().transaction(() => {
for (const plan of plans) {
const snapshot = before?.find(item => item.id === plan.id || item.path === plan.path);
if (!snapshot) throw new Error('Agent activation receipt is missing a before image.');
const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange;
const row = db().prepare(`SELECT *, ${blobSize('content_hash')} AS size FROM staged_file_changes WHERE id=?`).get(plan.id) as
Omit<StagedFileChange, 'content'> & { content_hash: string | null; size: number };
if (!row || row.state === 'applied') continue;
const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64');
const bytes = snapshot.contentSha256 ? getFileBlob(snapshot.contentSha256)
: snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64');
if (row.source_kind === 'beta') {
captureBetaBeforeImage(row, bytes);
db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server);
}
if (row.deployment_file_id && !row.preserve_before_image) {
db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=?,snapshot_pending=0 WHERE id=?')
.run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
db().prepare('UPDATE deployment_files SET verb=?,content_before_hash=?,sha256_before=?,snapshot_pending=0 WHERE id=?')
.run(row.content_hash === null ? 'removed' : bytes === null ? 'added' : 'updated', putFileBlob(bytes), bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${job.server}: captured rollback snapshot for ${row.rel_path}`);
}
if (row.content === null) removeScript(job.server, row.rel_path);
if (row.content_hash === null) removeScript(job.server, row.rel_path);
else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null,
sha256Buffer(row.content), row.content.length, row.inventory_source);
row.content_hash, row.size, row.inventory_source);
db().prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?").run(new Date().toISOString(), row.id);
}
db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
@@ -596,7 +600,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const files = job.stage_operation_id ? storedFiles(job) : filesFor(job);
if (files.length && !job.stage_operation_id) {
db().transaction(() => {
const stage = queueHostOperationSerialized(instance.host_id, 'files.stage', Buffer.from(JSON.stringify({ stageId: job.id, files })),
const stage = queueHostOperationSerialized(instance.host_id, 'files.stage', Buffer.from(JSON.stringify({ stageId: job.id, files: files.map(file => ({ ...file, ...(file.sha256 ? { contentSha256: file.sha256 } : { contentBase64: null }) })) })),
{ instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:stage` }, { stageId: job.id });
db().prepare('UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?')
.run(stage.id, JSON.stringify(files.map(fileMetadata)), job.id);
@@ -713,6 +717,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (prepared) {
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
for (const blob of prepared.blobs ?? []) putFileBlob(blob.content);
const recipe = prepared.recipe;
db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?')
.run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id);