josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Deduplicate artifact bytes and deliver files by authenticated hash

Browse Source
This commit is contained in:
josh
2026-10-06 14:35:13 -04:00
parent 18283c63a7
commit 62df14b180
45 changed files with 510 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/native-provisioning.test.ts
+10 -8
View File
@@ -1,3 +1,4 @@
import { putFileBlob } from '../../src/lib/server/db/file-blobs.js';
import { createHash } from 'node:crypto';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { join } from 'node:path';
@@ -697,7 +698,7 @@ describe('native instance provisioning', () => {
const before = Buffer.from('original bytes on the host'); const after = Buffer.from('new campaign bytes');
db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)").run(new Date().toISOString());
db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')").run(job.name);
const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)").run(job.name, after).lastInsertRowid);
const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after_hash,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)").run(job.name, putFileBlob(after, db())).lastInsertRowid);
const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: job.name, kind: 'script', relPath: 'hud.gsc', content: after, inventorySource: 'test', deploymentFileId: fileId });
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
@@ -712,8 +713,9 @@ describe('native instance provisioning', () => {
expect(JSON.parse(routing.payload)).toEqual({ stageId: restart.id });
const plan = db().prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { files_json: string };
expect(JSON.parse(plan.files_json)[0]).not.toHaveProperty('contentBase64');
expect(db().prepare('SELECT payload FROM ec_host_operation_payloads WHERE operation_id=?').get(stage.id))
.toEqual({ payload: JSON.stringify(stage.payload) });
const storedPayload = db().prepare('SELECT payload FROM ec_host_operation_payloads WHERE operation_id=?').get(stage.id) as { payload: string };
expect(JSON.parse(storedPayload.payload).files[0]).toMatchObject({ contentSha256: createHash('sha256').update(after).digest('hex') });
expect(JSON.parse(storedPayload.payload).files[0]).not.toHaveProperty('contentBase64');
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
const { pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
@@ -726,13 +728,13 @@ describe('native instance provisioning', () => {
remote.players = 0; await advance();
const activation = (await operationFor(restart.id))!;
expect(activation).toMatchObject({ kind: 'instance.restart', generation: 1, payload: { stageId: restart.id } });
expect(db().prepare('SELECT content_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, snapshot_pending: 1 });
expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, snapshot_pending: 1 });
const files = stage.payload.files as Array<{ id: number; path: string }>;
reportHostOperation(hostId, activation.id, { state: 'completed', result: { before: [{ id: files[0].id, path: files[0].path, contentBase64: before.toString('base64') }] } });
await advance();
expect((await detail(restart.id)).state).toBe('done');
expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied');
expect(db().prepare('SELECT content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex'), snapshot_pending: 0 });
expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex'), snapshot_pending: 0 });
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex'));
});
@@ -827,7 +829,7 @@ describe('native instance provisioning', () => {
expect((await detail(restart.id)).state).toBe('failed');
expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('pending');
expect(db().prepare('SELECT COUNT(*) AS count FROM restart_rollout WHERE healthy_at IS NULL').get()).toEqual({ count: 0 });
expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, sha256_before: null });
expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, sha256_before: null });
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined();
});
@@ -1277,7 +1279,7 @@ describe('native provisioning cancellation and rollback', () => {
expect((await detail(restart.id)).state).toBe('cancelled');
const replacement = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as {provisioning_id:string};
expect(replacement.provisioning_id).not.toBe(restart.id);
expect(db().prepare("SELECT content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?").get(rollback.deploymentId))
expect(db().prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?").get(rollback.deploymentId))
.toEqual({content:Buffer.from('working')});
await advance();
const stagedRollback = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(replacement.provisioning_id) as {stage_operation_id:string};
@@ -1319,7 +1321,7 @@ describe('cancelled native mutation bookkeeping', () => {
await reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('revision two') }]);
await activate('revision one');
const rollback = await rollbackBeta(betaId);
expect(db().prepare("SELECT content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(rollback.deploymentId))
expect(db().prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(rollback.deploymentId))
.toEqual({ content: original === null ? null : Buffer.from(original) });
});