josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve deleted server history

Browse Source 
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.
This commit is contained in:
josh
2026-09-27 21:48:55 -04:00
parent 53b29f726c
commit 981fb27c1c
61 changed files with 929 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/beta-plan-runner.ts
+19 -5
View File
@@ -1,5 +1,5 @@
import { db } from '../db/index.js';
import { betaOperationInFlight, deployBetaServers, getBeta, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { betaOperationInFlight, deployBetaServers, getBeta, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
@@ -7,9 +7,9 @@ import { emit, type BetaStage } from './bus.js';
import { requestRestart, releaseRestartRequest } from '../provisioning/index.js';
function stage(id: number, server: string, next: BetaStage, error: string | null = null): void {
db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ?')
const result = db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL')
.run(next, error, new Date().toISOString(), id, server);
emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error });
if (result.changes) emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error });
}
export function betaState(id: number, ready: boolean): void {
@@ -25,14 +25,16 @@ export function betaState(id: number, ready: boolean): void {
}
export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void> {
const servers = listBetaServers(id).filter(s => s.deployed_at);
const servers = listBetaServers(id).filter(s => s.deployed_at && isActiveBetaServer(id, s.server));
const result = await rollbackBeta(id);
if (plan.rollbackRestart === 'when-empty' && servers.length) {
const deploymentId = `dep-beta-rollback-${id}-${Date.now()}`;
db().transaction(() => {
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, 'beta-test-plan', ?)")
.run(deploymentId, new Date().toISOString(), id);
for (const server of servers) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server);
for (const server of servers) {
if (isActiveBetaServer(id, server.server)) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server);
}
})();
activity(id, 'Restored files; rolling restarts will wait for empty servers');
} else activity(id, 'Restored files; manual restarts are required to load them');
@@ -73,12 +75,21 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
const plan = parseBetaPlan(JSON.parse(stored.plan));
const beta = getBeta(id)!;
const servers = listBetaServers(id);
// Preserve the original coverage and evidence. A replacement with the same
// name cannot complete a test that began on a different server.
const missing = servers.filter(server => !isActiveBetaServer(id, server.server));
if (missing.length) {
betaState(id, false);
setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`);
return;
}
if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) {
betaState(id, false);
setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
return;
}
for (const server of servers) {
if (!isActiveBetaServer(id, server.server)) return;
const freshPlan = storedPlan(id)!;
if (!['active', 'passed'].includes(freshPlan.outcome)) return;
if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; }
@@ -108,6 +119,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
}
}
}
if (!isActiveBetaServer(id, server.server)) return;
if (server.stage === 'awaiting-empty' || server.stage === 'awaiting-health') {
const operation = requestRestart(server.server, { kind: 'beta', id: String(id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
@@ -129,6 +141,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
const run = currentRun(id, server.server);
if (!run?.observing_at) continue;
await observeRun(run, plan);
if (!isActiveBetaServer(id, server.server)) return;
if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; }
const metrics = readMetrics(currentRun(id, server.server)!);
const progress = evaluateRequirements(plan, metrics, run.phase);
@@ -143,6 +156,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
} else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing');
}
const updated = listBetaServers(id);
if (updated.some(server => !isActiveBetaServer(id, server.server))) return;
const queued = updated.filter(s => s.stage === 'queued');
if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) {
const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server);