josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Defer promotion snapshots to campaign activation

Browse Source
This commit is contained in:
josh
2026-10-06 00:16:43 -04:00
parent d6ff59b129
commit 9fd448339d
15 changed files with 179 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/operation-flow.ts
+12 -5
View File
@@ -29,12 +29,19 @@ export function betaFlow(beta: { state: string; wait_for_health: number; wait_fo
: 'pending' }));
}
export function campaignFlow(servers: { restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean): OperationStep[] {
export function campaignFlow(servers: { server?: string; restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean,
files: { server: string; snapshot_pending: number; apply_state: string }[] = []): OperationStep[] {
const active = servers.filter(s => !['skipped', 'cancelled'].includes(s.restart_state));
const positions = active.map(s => s.restart_state === 'done' ? 4
: s.restarted_at ? 3 : 0);
const current = positions.length ? Math.min(...positions) : finished ? 4 : 0;
return ['Wait for empty servers and restart capacity', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({
const positions = active.map(s => {
if (s.restart_state === 'done') return 5;
if (s.restarted_at) return 4;
const changes = files.filter(f => f.server === s.server && f.apply_state !== 'cancelled');
if (changes.some(f => f.apply_state === 'applying')) return changes.some(f => f.snapshot_pending) ? 1 : 2;
if (changes.length && changes.every(f => f.apply_state === 'applied')) return 3;
return 0;
});
const current = positions.length ? Math.min(...positions) : finished ? 5 : 0;
return ['Wait for empty servers and restart capacity', 'Capture rollback snapshots', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({
label, state: i < current ? 'done' : i !== current ? 'pending'
: active.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed'
: active.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current'
src/lib/server/db/index.ts
+3
View File
@@ -317,6 +317,9 @@ export function db(): Database.Database {
handle.exec(schemaSql);
migrateProvisioningActions(handle);
migrateWatchdogConstraints(handle);
migrateTable(handle, 'deployment_files', [
{ name: 'snapshot_pending', ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0' }
]);
migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
migrateTable(handle, 'server_deployments', [
{ name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
src/lib/server/db/schema.sql
+3 -1
View File
@@ -365,7 +365,9 @@ CREATE TABLE IF NOT EXISTS deployment_files (
sha256_before TEXT,
sha256_after TEXT,
content_before BLOB,
content_after BLOB
content_after BLOB,
-- An unknown before image is distinct from a confirmed absent file.
snapshot_pending INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_deployment_files_dep ON deployment_files(deployment_id);
src/lib/server/fleet/beta.ts
+10 -27
View File
@@ -35,7 +35,7 @@ import {
type ScriptVersionRow
} from './scripts.js';
import { composeServerCfg } from './config-compose.js';
import { getServerScript, removeScript } from './inventory.js';
import { removeScript } from './inventory.js';
import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js';
import { emit } from '../pipeline/bus.js';
import { cancelProvisioningJob } from '../provisioning/cancellation.js';
@@ -976,7 +976,6 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
const targets = reviewed.preview.files.find(file => file.id === script.id)!.servers;
for (const name of targets) {
const dest = destOfKind(name, script.kind);
let plan: Plan;
if (betaServerSet.has(name)) {
const bss = getBetaServerScript(betaId, name, script.id);
@@ -1001,21 +1000,9 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
}
} else {
// Non-beta matching-scope server: current on-disk file (if any)
// is the "before" state for a promote-then-rollback round trip.
const targetPath = join(dest, script.target_rel_path);
if (await exists(targetPath)) {
const priorContent = await readFile(targetPath);
const inv = getServerScript(name, script.target_rel_path);
plan = {
script, version, server: name,
priorVersionId: inv?.script_version_id ?? null,
priorSha: sha256Buffer(priorContent),
priorContent, hadFile: true
};
} else {
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
}
// Campaign activation captures the actual before image just
// before replacing this file. Do not wait on remote reads here.
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
}
plans.push(plan);
}
@@ -1068,12 +1055,6 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
hadFile = true;
}
}
if (!betaServerSet.has(name) && await exists(targetPath)) {
// Capture an initial preview; activation records the actual before image.
priorContent = await readFile(targetPath);
priorSha = sha256Buffer(priorContent);
hadFile = true;
}
const requiresActivation = !betaServerSet.has(name) || !await exists(targetPath)
|| sha256Buffer(await readFile(targetPath)) !== composedSha;
cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation });
@@ -1114,8 +1095,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
);
const insertDepFile = db().prepare(
`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
);
const uniqueServers = Array.from(new Set([
@@ -1140,7 +1121,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
p.priorSha,
p.version.sha256,
p.priorContent,
p.version.content
p.version.content,
betaServerSet.has(p.server) ? 0 : 1
);
if (plansToActivate.has(p)) {
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server,
@@ -1159,7 +1141,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
cp.priorSha,
cp.composedSha,
cp.priorContent,
cp.composedContent
cp.composedContent,
betaServerSet.has(cp.server) ? 0 : 1
);
if (cp.requiresActivation) {
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: cp.server,
src/lib/server/fleet/rollback.ts
+6 -2
View File
@@ -11,7 +11,7 @@ import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges }
// pre-multi-script legacy rows.
export class RollbackError extends Error {
constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back') {
constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back' | 'snapshot-pending') {
super(message);
}
}
@@ -42,6 +42,7 @@ interface FileRow {
sha256_after: string | null;
content_before: Buffer | null;
content_after: Buffer | null;
snapshot_pending: number;
}
interface ScriptTargetRow {
@@ -100,7 +101,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
const fileRows = db()
.prepare(
`SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, f.content_before, f.content_after
`SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, f.content_before, f.content_after, f.snapshot_pending
FROM deployment_files f JOIN deployment_servers ds ON ds.deployment_id = f.deployment_id AND ds.server = f.server
JOIN servers s ON s.name = f.server
WHERE f.deployment_id = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL ORDER BY f.server, f.rel_path`
@@ -154,6 +155,9 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
const preservesBetaSnapshot = staged.some(row => row.server === f.server && row.kind === kind
&& row.rel_path === f.rel_path && row.preserve_before_image);
if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue;
if (f.snapshot_pending) {
throw new RollbackError(`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`, 'snapshot-pending');
}
const arr = perServer.get(f.server);
if (arr) arr.push(f);
else perServer.set(f.server, [f]);
src/lib/server/fleet/staged-files.ts
+8 -4
View File
@@ -130,10 +130,14 @@ export function applyStagedFilesForOperation(operationId: string, server: string
if (row.source_kind === 'beta') {
captureBetaBeforeImage(row, before);
}
if (row.deployment_file_id && !row.preserve_before_image) db().prepare(`UPDATE deployment_files SET
verb = ?, content_before = ?, sha256_before = ? WHERE id = ?`)
.run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before,
before === null ? null : sha256Buffer(before), row.deployment_file_id);
if (row.deployment_file_id && !row.preserve_before_image) {
db().prepare(`UPDATE deployment_files SET
verb = ?, content_before = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`)
.run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before,
before === null ? null : sha256Buffer(before), row.deployment_file_id);
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${server}: captured rollback snapshot for ${row.rel_path}`);
}
db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?").run(row.id);
})();
if (row.content === null) {
src/lib/server/provisioning/native.ts
+6 -2
View File
@@ -240,8 +240,12 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
captureBetaBeforeImage(row, bytes);
db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server);
}
if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?')
.run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
if (row.deployment_file_id && !row.preserve_before_image) {
db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=?,snapshot_pending=0 WHERE id=?')
.run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${job.server}: captured rollback snapshot for ${row.rel_path}`);
}
if (row.content === null) removeScript(job.server, row.rel_path);
else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null,
sha256Buffer(row.content), row.content.length, row.inventory_source);