josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: make provisioning own instance lifecycle operations

Browse Source
This commit is contained in:
josh
2026-09-27 14:45:00 -04:00
parent cfed584614
commit df265d1900
44 changed files with 1791 additions and 1609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/beta-plan-runner.ts
+28 -35
View File
@@ -1,12 +1,10 @@
import { db } from '../db/index.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { betaOperationInFlight, deployBetaServers, getBeta, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
import { emit, type BetaStage } from './bus.js';
import { restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
import { requestRestart, releaseRestartRequest } from '../provisioning/index.js';
function stage(id: number, server: string, next: BetaStage, error: string | null = null): void {
db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ?')
@@ -80,11 +78,6 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
return;
}
if (!komodoConfigured() || !rconConfigured()) {
betaState(id, false);
setOutcome(id, 'active', 'Waiting for Komodo and RCON configuration');
return;
}
for (const server of servers) {
const freshPlan = storedPlan(id)!;
if (!['active', 'passed'].includes(freshPlan.outcome)) return;
@@ -102,35 +95,33 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
const broken = evaluateRequirements(plan, readMetrics(currentRun(id, server.server)!), waitingRun.phase).find(p => p.failed);
if (broken) { await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
}
if (rolloutWaitReason(server.server)) continue;
if (await rconGetPlayerCount(server.server) !== 0) continue;
const oldRun = currentRun(id, server.server);
const phase = oldRun ? oldRun.phase + 1 : 0;
// Fail closed after interrupted restart requests; never silently reuse earlier evidence.
stage(id, server.server, 'restarting');
const run = await prepareRun(id, server.server, phase);
if (oldRun) {
const previous = readMetrics(oldRun);
const metrics = readMetrics(run);
metrics.outages = previous.outages;
saveMetrics(run, metrics);
// Prepare evidence once before handing off the restart. Empty-server
// checks and remote execution belong entirely to Provisioning.
let run = currentRun(id, server.server);
if (!run || run.observing_at) {
const oldRun = run;
run = await prepareRun(id, server.server, oldRun ? oldRun.phase + 1 : 0);
if (oldRun) {
const metrics = readMetrics(run);
metrics.outages = readMetrics(oldRun).outages;
saveMetrics(run, metrics);
}
}
const restart = await restartInRollout(server.server);
if (restart.kind === 'queued') {
db().prepare('DELETE FROM beta_test_runs WHERE id = ?').run(run.id);
if (oldRun) db().prepare('UPDATE beta_test_runs SET retired_at = NULL WHERE id = ?').run(oldRun.id);
stage(id, server.server, 'awaiting-empty', restart.message); continue;
}
if (restart.kind === 'failed') { stage(id, server.server, 'failed', restart.message); await fail(id, plan, restart.message); return; }
db().prepare('UPDATE beta_servers SET restarted_at = ?, healthy_at = NULL WHERE beta_id = ? AND server = ?')
.run(new Date().toISOString(), id, server.server);
stage(id, server.server, 'awaiting-health');
continue;
}
if (server.stage === 'awaiting-health') {
const health = rolloutHealth(server.server);
if (!health.ok) continue;
const now = new Date().toISOString();
if (server.stage === 'awaiting-empty' || server.stage === 'awaiting-health') {
const operation = requestRestart(server.server, { kind: 'beta', id: String(id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
const reason = operation.error ?? operation.stage;
stage(id, server.server, 'failed', reason); await fail(id, plan, reason); return;
}
if (!operation.submitted_at || operation.state === 'running') {
stage(id, server.server, 'awaiting-empty', operation.stage);
continue;
}
db().prepare('UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?')
.run(operation.submitted_at, id, server.server);
if (operation.state !== 'done') { stage(id, server.server, 'awaiting-health', operation.stage); continue; }
const now = operation.finished_at ?? new Date().toISOString();
db().prepare('UPDATE beta_servers SET healthy_at = ? WHERE beta_id = ? AND server = ?').run(now, id, server.server);
db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(now, id, server.server);
stage(id, server.server, 'observing');
@@ -145,6 +136,8 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
if (broken) { stage(id, server.server, 'failed', broken.detail); await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
const recoveryTarget = Math.max(0, ...plan.requirements.filter(r => r.kind === 'recovery').map(r => r.target));
if (run.phase < recoveryTarget && progress.filter(p => !plan.requirements.some(r => r.id === p.id && r.kind === 'recovery')).every(p => p.passed)) {
releaseRestartRequest(server.server, { kind: 'beta', id: String(id) });
db().prepare('UPDATE beta_servers SET restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND server = ?').run(id, server.server);
stage(id, server.server, 'awaiting-empty');
activity(id, `${server.server}: recovery cycle ${run.phase + 1}/${recoveryTarget} queued; gameplay requirements will be repeated`);
} else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing');