josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Defer server file changes until guarded restart activation

Browse Source 
Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely.

Fixes #28

Validation: 805 tests passed, Svelte checks clean, production build passed.
This commit is contained in:
josh
2026-09-27 23:06:16 -04:00
parent eb2a852fdc
commit fb48f64de1
36 changed files with 1634 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/rollback.ts
+29 -50
View File
@@ -1,11 +1,8 @@
import { mkdir, rm, writeFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { destOfKind, type ArtifactKind } from '../config.js';
import type { ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
import { recordScript, removeScript } from './inventory.js';
import { getVersion } from './scripts.js';
import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js';
// Roll a promoted deployment back to whatever versions were current before
// the promote landed. Reads deployment_scripts as the source of truth for
@@ -142,58 +139,27 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
// Group files by server so each server gets exactly one deployment_servers row.
const perServer = new Map<string, FileRow[]>();
const staged = stagedFileChanges('campaign', promoteId);
// Composed baseline rows are physically staged as config files. Prefer the
// explicit link instead of inferring their destination from the library.
const kindForFile = (file: FileRow) => staged.find(row => row.deployment_file_id === file.id)?.kind ?? kindForRelPath(file.rel_path);
for (const f of fileRows) {
// A promotion can still be waiting for empty servers. Withdrawing it
// must not restore or remove files that it never installed. Rows from
// older releases have no staging record and retain their audit meaning.
const kind = kindForFile(f);
const preservesBetaSnapshot = staged.some(row => row.server === f.server && row.kind === kind
&& row.rel_path === f.rel_path && row.preserve_before_image);
if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue;
const arr = perServer.get(f.server);
if (arr) arr.push(f);
else perServer.set(f.server, [f]);
}
const affectedServers: string[] = [];
const perServerFailures = new Map<string, string>();
const active = (server: string) => !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server);
for (const [server, files] of perServer) {
if (!active(server)) continue;
try {
for (const f of files) {
if (!active(server)) break;
const kind = kindForRelPath(f.rel_path);
const dest = destOfKind(server, kind);
const abs = join(dest, f.rel_path);
await mkdir(join(dest, dirname(f.rel_path)), { recursive: true });
if (!active(server)) break;
const owningScript = scriptByTarget.get(f.rel_path);
if (f.verb === 'added') {
if (existsSync(abs)) await rm(abs, { force: true });
if (!active(server)) break;
removeScript(server, f.rel_path);
} else if (f.verb === 'updated' && f.content_before) {
await writeFile(abs, f.content_before);
if (!active(server)) break;
const priorVer = owningScript?.prior_version_id
? getVersion(owningScript.prior_version_id)
: null;
recordScript(
server,
f.rel_path,
priorVer?.id ?? null,
f.sha256_before!,
f.content_before.byteLength,
`rollback:${newDepId}`
);
} else if (f.verb === 'removed' && f.content_before) {
await writeFile(abs, f.content_before);
if (!active(server)) break;
recordScript(server, f.rel_path, null, f.sha256_before!, f.content_before.byteLength, `rollback:${newDepId}`);
}
}
if (active(server)) affectedServers.push(server);
} catch (err) {
perServerFailures.set(server, (err as Error).message);
}
}
// Write the new rollback deployment + mark the original.
const insertDep = db().prepare(
`INSERT INTO deployments
@@ -216,6 +182,11 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
);
db().transaction(() => {
cancelStagedFiles('campaign', promoteId);
// Withdrawing this source leaves a shared restart available to other
// campaigns. Provisioning cancels an operation with no eligible sources.
db().prepare(`UPDATE deployment_servers SET restart_state = 'skipped', error = NULL
WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).run(promoteId);
insertDep.run(newDepId, startedAt);
// Mirror the promote's script list — swapped: the "current" version
// after rollback is the promote's prior; the "prior" of the rollback
@@ -231,8 +202,8 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
}
for (const [server, files] of perServer) {
if (!active(server)) continue;
if (perServerFailures.has(server)) {
insertDepServer.run(newDepId, server, 0, 'failed', perServerFailures.get(server)!);
if (files.some(file => file.verb !== 'added' && file.content_before === null)) {
insertDepServer.run(newDepId, server, 0, 'failed', 'Original file snapshot is missing; cannot restore safely');
continue;
}
insertDepServer.run(newDepId, server, files.length, 'pending', null);
@@ -241,7 +212,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
f.verb === 'added' ? 'removed'
: f.verb === 'removed' ? 'added'
: 'updated';
insertDepFile.run(
const inserted = insertDepFile.run(
newDepId,
server,
invertedVerb,
@@ -251,7 +222,15 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
f.content_after,
f.content_before
);
const owningScript = scriptByTarget.get(f.rel_path);
const priorVersion = owningScript?.prior_version_id ? getVersion(owningScript.prior_version_id) : null;
queueFileChange({ sourceKind: 'campaign', sourceId: newDepId, server,
kind: kindForFile(f), relPath: f.rel_path,
content: f.verb === 'added' ? null : f.content_before,
scriptVersionId: f.verb === 'updated' ? priorVersion?.id ?? null : null,
inventorySource: `rollback:${newDepId}`, deploymentFileId: Number(inserted.lastInsertRowid) });
}
affectedServers.push(server);
}
db().prepare(`UPDATE deployments SET rolled_back_by = ? WHERE id = ?`).run(newDepId, promoteId);
// Restore each affected script's current_version_id.