josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve deleted server history

Browse Source 
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.
This commit is contained in:
josh
2026-09-27 21:48:55 -04:00
parent 53b29f726c
commit 981fb27c1c
61 changed files with 929 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/beta-plan-runner.ts
+19 -5
View File
@@ -1,5 +1,5 @@
import { db } from '../db/index.js';
import { betaOperationInFlight, deployBetaServers, getBeta, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { betaOperationInFlight, deployBetaServers, getBeta, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
@@ -7,9 +7,9 @@ import { emit, type BetaStage } from './bus.js';
import { requestRestart, releaseRestartRequest } from '../provisioning/index.js';
function stage(id: number, server: string, next: BetaStage, error: string | null = null): void {
db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ?')
const result = db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL')
.run(next, error, new Date().toISOString(), id, server);
emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error });
if (result.changes) emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error });
}
export function betaState(id: number, ready: boolean): void {
@@ -25,14 +25,16 @@ export function betaState(id: number, ready: boolean): void {
}
export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void> {
const servers = listBetaServers(id).filter(s => s.deployed_at);
const servers = listBetaServers(id).filter(s => s.deployed_at && isActiveBetaServer(id, s.server));
const result = await rollbackBeta(id);
if (plan.rollbackRestart === 'when-empty' && servers.length) {
const deploymentId = `dep-beta-rollback-${id}-${Date.now()}`;
db().transaction(() => {
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, 'beta-test-plan', ?)")
.run(deploymentId, new Date().toISOString(), id);
for (const server of servers) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server);
for (const server of servers) {
if (isActiveBetaServer(id, server.server)) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server);
}
})();
activity(id, 'Restored files; rolling restarts will wait for empty servers');
} else activity(id, 'Restored files; manual restarts are required to load them');
@@ -73,12 +75,21 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
const plan = parseBetaPlan(JSON.parse(stored.plan));
const beta = getBeta(id)!;
const servers = listBetaServers(id);
// Preserve the original coverage and evidence. A replacement with the same
// name cannot complete a test that began on a different server.
const missing = servers.filter(server => !isActiveBetaServer(id, server.server));
if (missing.length) {
betaState(id, false);
setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`);
return;
}
if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) {
betaState(id, false);
setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
return;
}
for (const server of servers) {
if (!isActiveBetaServer(id, server.server)) return;
const freshPlan = storedPlan(id)!;
if (!['active', 'passed'].includes(freshPlan.outcome)) return;
if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; }
@@ -108,6 +119,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
}
}
}
if (!isActiveBetaServer(id, server.server)) return;
if (server.stage === 'awaiting-empty' || server.stage === 'awaiting-health') {
const operation = requestRestart(server.server, { kind: 'beta', id: String(id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
@@ -129,6 +141,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
const run = currentRun(id, server.server);
if (!run?.observing_at) continue;
await observeRun(run, plan);
if (!isActiveBetaServer(id, server.server)) return;
if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; }
const metrics = readMetrics(currentRun(id, server.server)!);
const progress = evaluateRequirements(plan, metrics, run.phase);
@@ -143,6 +156,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
} else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing');
}
const updated = listBetaServers(id);
if (updated.some(server => !isActiveBetaServer(id, server.server))) return;
const queued = updated.filter(s => s.stage === 'queued');
if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) {
const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server);
src/lib/server/pipeline/beta-runner.ts
+7 -6
View File
@@ -25,7 +25,7 @@ const setStage = () =>
restarted_at = CASE WHEN ? = 'awaiting-health' OR ? = 'ready' THEN COALESCE(restarted_at, ?) ELSE restarted_at END,
healthy_at = CASE WHEN ? = 'awaiting-match' OR (? = 'ready' AND healthy_at IS NULL) THEN COALESCE(healthy_at, ?) ELSE healthy_at END,
match_seen_at = CASE WHEN ? = 'ready' AND match_seen_at IS NULL THEN ? ELSE match_seen_at END
WHERE beta_id = ? AND server = ?`
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
);
function transition(
@@ -35,7 +35,7 @@ function transition(
error: string | null = null
): void {
const t = nowIso();
setStage().run(
const result = setStage().run(
stage,
error,
t,
@@ -50,17 +50,17 @@ function transition(
betaId,
server
);
emit({ type: 'beta-server-stage', betaId, server, stage, error });
if (result.changes) emit({ type: 'beta-server-stage', betaId, server, stage, error });
}
// Bump beta.state to ready-promote when every one of its beta_servers is ready.
// A single failed row is enough to keep the beta in 'running' (operator retries).
function reconcileBetaState(betaId: number): void {
const rows = db()
.prepare(`SELECT stage FROM beta_servers WHERE beta_id = ?`)
.all(betaId) as Array<{ stage: BetaStage }>;
.prepare(`SELECT stage, server_deleted_at FROM beta_servers WHERE beta_id = ?`)
.all(betaId) as Array<{ stage: BetaStage; server_deleted_at: string | null }>;
if (rows.length === 0) return;
if (rows.every((r) => r.stage === 'ready')) {
if (rows.every((r) => r.stage === 'ready' && !r.server_deleted_at)) {
const upd = db()
.prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ? AND state = 'running'`)
.run(betaId);
@@ -98,6 +98,7 @@ async function runBetaTick(): Promise<{ processed: number }> {
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
WHERE b.state = 'running'
AND bs.server_deleted_at IS NULL
AND NOT EXISTS (SELECT 1 FROM beta_test_plans p WHERE p.beta_id = b.id)
AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')
ORDER BY b.id, bs.server`
src/lib/server/pipeline/sweep.ts
+3 -3
View File
@@ -19,7 +19,7 @@ export function sweep(): Promise<SweepResult> {
}
async function runSweep(): Promise<SweepResult> {
const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
WHERE ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
for (const row of pending) {
if (!row.files_changed) continue; // A failed file write is not a restart request.
@@ -40,10 +40,10 @@ async function runSweep(): Promise<SweepResult> {
emit({ type: 'server-state', deploymentId: row.deployment_id, server: row.server, state, players: operation.players, error });
}
db().prepare(`UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS (
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`).run(nowIso());
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`).run(nowIso());
db().prepare(`DELETE FROM deployments WHERE started_at < ?
AND NOT EXISTS (SELECT 1 FROM watchdog_actions WHERE campaign_id = deployments.id) AND NOT EXISTS (
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`)
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`)
.run(new Date(Date.now() - fleetSettings().deployKeepDays * 86_400_000).toISOString());
emit({ type: 'sweep-tick', frozen: false });
return result;