josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add configurable beta test plans and staged rollout

Browse Source
This commit is contained in:
josh
2026-09-17 21:35:28 -04:00
parent 5b5266da3a
commit a50eae9d21
26 changed files with 1518 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/beta-plan-runner.ts
+167
View File
@@ -0,0 +1,167 @@
import { db } from '../db/index.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { betaOperationInFlight, deployBetaServers, getBeta, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
import { emit, type BetaStage } from './bus.js';
import { restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
function stage(id: number, server: string, next: BetaStage, error: string | null = null): void {
db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ?')
.run(next, error, new Date().toISOString(), id, server);
emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error });
}
export function betaState(id: number, ready: boolean): void {
const state = ready ? 'ready-promote' : 'running';
const result = db().prepare("UPDATE betas SET state = ? WHERE id = ? AND state IN ('running','ready-promote') AND state <> ?").run(state, id, state);
if (result.changes) emit({ type: 'beta-state', betaId: id, state });
// A revoked pass must not leave this version labelled tested unless another beta still proves it.
db().prepare(`UPDATE script_versions SET state = ? WHERE id IN (SELECT script_version_id FROM beta_scripts WHERE beta_id = ?)
AND id NOT IN (SELECT current_version_id FROM scripts WHERE current_version_id IS NOT NULL)
AND NOT EXISTS (SELECT 1 FROM beta_scripts bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.script_version_id = script_versions.id AND b.id <> ? AND b.state IN ('ready-promote','promoted'))`)
.run(ready ? 'tested' : 'untested', id, id);
}
export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void> {
const servers = listBetaServers(id).filter(s => s.deployed_at);
const result = await rollbackBeta(id);
if (plan.rollbackRestart === 'when-empty' && servers.length) {
const deploymentId = `dep-beta-rollback-${id}-${Date.now()}`;
db().transaction(() => {
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, 'beta-test-plan', ?)")
.run(deploymentId, new Date().toISOString(), id);
for (const server of servers) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server);
})();
activity(id, 'Restored files; rolling restarts will wait for empty servers');
} else activity(id, 'Restored files; manual restarts are required to load them');
if (result.warnings.length) activity(id, result.warnings.join('; '));
}
async function fail(id: number, plan: BetaPlan, reason: string): Promise<void> {
betaState(id, false);
setOutcome(id, plan.failureAction === 'pause' ? 'paused' : 'failed', reason);
if (plan.failureAction === 'rollback') {
try { await rollbackTestPlan(id, plan); }
catch (error) { setOutcome(id, 'failed', `Automatic rollback failed: ${(error as Error).message}`); }
}
}
let inFlight: Promise<void> | null = null;
export function plannedBetaTick(): Promise<void> {
if (inFlight) return inFlight;
inFlight = tick().finally(() => { inFlight = null; });
return inFlight;
}
async function tick(): Promise<void> {
const plans = db().prepare(`SELECT p.* FROM beta_test_plans p JOIN betas b ON b.id = p.beta_id
WHERE b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed') ORDER BY p.beta_id`).all() as StoredPlan[];
for (const stored of plans) {
if (betaOperationInFlight(stored.beta_id)) continue;
try { await advancePlan(stored); }
catch (error) {
const reason = `Test runner: ${(error as Error).message}`;
try { await fail(stored.beta_id, parseBetaPlan(JSON.parse(stored.plan)), reason); }
catch { betaState(stored.beta_id, false); setOutcome(stored.beta_id, 'failed', reason); }
}
}
}
async function advancePlan(stored: StoredPlan): Promise<void> {
const id = stored.beta_id;
const plan = parseBetaPlan(JSON.parse(stored.plan));
const beta = getBeta(id)!;
const servers = listBetaServers(id);
if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) {
betaState(id, false);
setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
return;
}
if (!komodoConfigured() || !rconConfigured()) {
betaState(id, false);
setOutcome(id, 'active', 'Waiting for Komodo and RCON configuration');
return;
}
for (const server of servers) {
const freshPlan = storedPlan(id)!;
if (!['active', 'passed'].includes(freshPlan.outcome)) return;
if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; }
if (server.stage === 'deploying' || server.stage === 'restarting') {
stage(id, server.server, 'failed', `${server.stage === 'deploying' ? 'deploy: ' : ''}Operation interrupted; review the server before starting a fresh attempt`);
await fail(id, plan, `${server.server}: interrupted ${server.stage}`); return;
}
if (server.stage === 'queued') continue;
if (server.stage === 'awaiting-empty') {
const waitingRun = currentRun(id, server.server);
if (waitingRun?.observing_at) {
await observeRun(waitingRun, plan);
if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; }
const broken = evaluateRequirements(plan, readMetrics(currentRun(id, server.server)!), waitingRun.phase).find(p => p.failed);
if (broken) { await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
}
if (rolloutWaitReason(server.server)) continue;
if (await rconGetPlayerCount(server.server) !== 0) continue;
const oldRun = currentRun(id, server.server);
const phase = oldRun ? oldRun.phase + 1 : 0;
// Fail closed after interrupted restart requests; never silently reuse earlier evidence.
stage(id, server.server, 'restarting');
const run = await prepareRun(id, server.server, phase);
if (oldRun) {
const previous = readMetrics(oldRun);
const metrics = readMetrics(run);
metrics.outages = previous.outages; metrics.logErrors = previous.logErrors; metrics.restarts = previous.restarts;
saveMetrics(run, metrics);
}
const restart = await restartInRollout(server.server);
if (restart.kind === 'queued') {
db().prepare('DELETE FROM beta_test_runs WHERE id = ?').run(run.id);
if (oldRun) db().prepare('UPDATE beta_test_runs SET retired_at = NULL WHERE id = ?').run(oldRun.id);
stage(id, server.server, 'awaiting-empty', restart.message); continue;
}
if (restart.kind === 'failed') { stage(id, server.server, 'failed', restart.message); await fail(id, plan, restart.message); return; }
db().prepare('UPDATE beta_servers SET restarted_at = ?, healthy_at = NULL WHERE beta_id = ? AND server = ?')
.run(new Date().toISOString(), id, server.server);
stage(id, server.server, 'awaiting-health');
continue;
}
if (server.stage === 'awaiting-health') {
const health = rolloutHealth(server.server);
if (!health.ok) continue;
const now = new Date().toISOString();
db().prepare('UPDATE beta_servers SET healthy_at = ? WHERE beta_id = ? AND server = ?').run(now, id, server.server);
db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(now, id, server.server);
stage(id, server.server, 'observing');
}
const run = currentRun(id, server.server);
if (!run?.observing_at) continue;
await observeRun(run, plan);
if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; }
const metrics = readMetrics(currentRun(id, server.server)!);
const progress = evaluateRequirements(plan, metrics, run.phase);
const broken = progress.find(p => p.failed);
if (broken) { stage(id, server.server, 'failed', broken.detail); await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
const recoveryTarget = Math.max(0, ...plan.requirements.filter(r => r.kind === 'recovery').map(r => r.target));
if (run.phase < recoveryTarget && progress.filter(p => !plan.requirements.some(r => r.id === p.id && r.kind === 'recovery')).every(p => p.passed)) {
stage(id, server.server, 'awaiting-empty');
activity(id, `${server.server}: recovery cycle ${run.phase + 1}/${recoveryTarget} queued; gameplay requirements will be repeated`);
} else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing');
}
const updated = listBetaServers(id);
const queued = updated.filter(s => s.stage === 'queued');
if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) {
const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server);
activity(id, `Previous wave passed; deploying ${wave.join(', ')}`);
await deployBetaServers(id, wave);
betaState(id, false);
return;
}
const registry = db().prepare('SELECT server AS name, game, map FROM beta_test_coverage WHERE beta_id = ?').all(id) as Array<{ name: string; game: string | null; map: string | null }>;
// All waves must be activated, even when the final coverage rule is a quorum.
const ready = !queued.length && updated.every(s => !!currentRun(id, s.server)?.observing_at)
&& coveragePassed(plan, updated.map(s => ({ ...registry.find(r => r.name === s.server)!, passed: s.stage === 'ready' })));
betaState(id, ready);
setOutcome(id, ready ? 'passed' : 'active', null);
}
src/lib/server/pipeline/beta-runner.ts
+4 -1
View File
@@ -5,6 +5,7 @@ import { komodoConfigured, rconConfigured } from '../config.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { scanForMatchEnd } from '../fleet/match-log.js';
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
import { plannedBetaTick } from './beta-plan-runner.js';
interface BetaServerActiveRow {
beta_id: number;
@@ -93,8 +94,9 @@ export function betaTick(): Promise<{ processed: number }> {
}
async function runBetaTick(): Promise<{ processed: number }> {
if (integrationsOn()) await refreshRestartRollout();
await plannedBetaTick();
if (!integrationsOn()) return { processed: 0 };
await refreshRestartRollout();
const rows = db()
.prepare(
@@ -104,6 +106,7 @@ async function runBetaTick(): Promise<{ processed: number }> {
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
WHERE b.state = 'running'
AND NOT EXISTS (SELECT 1 FROM beta_test_plans p WHERE p.beta_id = b.id)
AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')
ORDER BY b.id, bs.server`
)
src/lib/server/pipeline/bus.ts
+3
View File
@@ -26,6 +26,9 @@ export interface ServerStatusEntry {
}
export type BetaStage =
| 'queued'
| 'observing'
| 'inconclusive'
| 'deploying'
| 'awaiting-empty'
| 'restarting'