import { db } from '../db/index.js'; import { betaOperationInFlight, deployBetaServers, getBeta, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js'; import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js'; import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js'; import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; import { emit, type BetaStage } from './bus.js'; import { requestRestart, releaseRestartRequest } from '../provisioning/index.js'; function stage(id: number, server: string, next: BetaStage, error: string | null = null): void { const result = db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL') .run(next, error, new Date().toISOString(), id, server); if (result.changes) emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error }); } export function betaState(id: number, ready: boolean): void { const state = ready ? 'ready-promote' : 'running'; const result = db().prepare("UPDATE betas SET state = ? WHERE id = ? AND state IN ('running','ready-promote') AND state <> ?").run(state, id, state); if (result.changes) emit({ type: 'beta-state', betaId: id, state }); // A revoked pass must not leave this version labelled tested unless another beta still proves it. db().prepare(`UPDATE script_versions SET state = ? WHERE id IN (SELECT script_version_id FROM beta_scripts WHERE beta_id = ?) AND id NOT IN (SELECT current_version_id FROM scripts WHERE current_version_id IS NOT NULL) AND NOT EXISTS (SELECT 1 FROM beta_scripts bs JOIN betas b ON b.id = bs.beta_id WHERE bs.script_version_id = script_versions.id AND b.id <> ? AND b.state IN ('ready-promote','promoted'))`) .run(ready ? 'tested' : 'untested', id, id); } export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise { const result = await rollbackBeta(id, { restart: plan.rollbackRestart }); if (!result.deploymentId) activity(id, 'Cancelled queued changes; no deployed files need restoration'); else if (plan.rollbackRestart === 'when-empty') activity(id, 'Queued file restoration and rolling restarts; waiting for empty servers'); else activity(id, 'File restoration is held; resume the rollback campaign to apply it with a safe restart'); if (result.warnings.length) activity(id, result.warnings.join('; ')); } async function fail(id: number, plan: BetaPlan, reason: string): Promise { betaState(id, false); setOutcome(id, plan.failureAction === 'pause' ? 'paused' : 'failed', reason); if (plan.failureAction === 'rollback') { try { await rollbackTestPlan(id, plan); } catch (error) { setOutcome(id, 'failed', `Automatic rollback failed: ${(error as Error).message}`); } } } let inFlight: Promise | null = null; export function plannedBetaTick(): Promise { if (inFlight) return inFlight; inFlight = tick().finally(() => { inFlight = null; }); return inFlight; } async function tick(): Promise { const plans = db().prepare(`SELECT p.* FROM beta_test_plans p JOIN betas b ON b.id = p.beta_id WHERE b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed') ORDER BY p.beta_id`).all() as StoredPlan[]; for (const stored of plans) { if (betaOperationInFlight(stored.beta_id)) continue; try { await advancePlan(stored); } catch (error) { const reason = `Test runner: ${(error as Error).message}`; try { await fail(stored.beta_id, parseBetaPlan(JSON.parse(stored.plan)), reason); } catch { betaState(stored.beta_id, false); setOutcome(stored.beta_id, 'failed', reason); } } } } async function advancePlan(stored: StoredPlan): Promise { const id = stored.beta_id; const plan = parseBetaPlan(JSON.parse(stored.plan)); const beta = getBeta(id)!; const servers = listBetaServers(id); // Preserve the original coverage and evidence. A replacement with the same // name cannot complete a test that began on a different server. const missing = servers.filter(server => !isActiveBetaServer(id, server.server)); if (missing.length) { betaState(id, false); setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`); return; } if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) { betaState(id, false); setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence'); return; } for (const server of servers) { if (!isActiveBetaServer(id, server.server)) return; const freshPlan = storedPlan(id)!; if (!['active', 'passed'].includes(freshPlan.outcome)) return; if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; } if (server.stage === 'deploying' || server.stage === 'restarting') { stage(id, server.server, 'failed', `${server.stage === 'deploying' ? 'deploy: ' : ''}Operation interrupted; review the server before starting a fresh attempt`); await fail(id, plan, `${server.server}: interrupted ${server.stage}`); return; } if (server.stage === 'queued') continue; if (server.stage === 'awaiting-empty') { const waitingRun = currentRun(id, server.server); if (waitingRun?.observing_at) { await observeRun(waitingRun, plan); if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; } const broken = evaluateRequirements(plan, readMetrics(currentRun(id, server.server)!), waitingRun.phase).find(p => p.failed); if (broken) { await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; } } // Prepare evidence once before handing off the restart. Empty-server // checks and remote execution belong entirely to Provisioning. let run = currentRun(id, server.server); if (!run || run.observing_at) { const oldRun = run; run = await prepareRun(id, server.server, oldRun ? oldRun.phase + 1 : 0); if (oldRun) { const metrics = readMetrics(run); metrics.outages = readMetrics(oldRun).outages; saveMetrics(run, metrics); } } } if (!isActiveBetaServer(id, server.server)) return; if (server.stage === 'awaiting-empty' || server.stage === 'awaiting-health') { const operation = requestRestart(server.server, { kind: 'beta', id: String(id) }); if (['failed','interrupted','cancelled'].includes(operation.state)) { const reason = operation.error ?? operation.stage; stage(id, server.server, 'failed', reason); await fail(id, plan, reason); return; } if (!operation.submitted_at || operation.state === 'running') { stage(id, server.server, 'awaiting-empty', operation.stage); continue; } db().prepare('UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?') .run(operation.submitted_at, id, server.server); if (operation.state !== 'done') { stage(id, server.server, 'awaiting-health', operation.stage); continue; } const now = operation.finished_at ?? new Date().toISOString(); db().prepare('UPDATE beta_servers SET healthy_at = ? WHERE beta_id = ? AND server = ?').run(now, id, server.server); db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(now, id, server.server); stage(id, server.server, 'observing'); } const run = currentRun(id, server.server); if (!run?.observing_at) continue; await observeRun(run, plan); if (!isActiveBetaServer(id, server.server)) return; if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; } const metrics = readMetrics(currentRun(id, server.server)!); const progress = evaluateRequirements(plan, metrics, run.phase); const broken = progress.find(p => p.failed); if (broken) { stage(id, server.server, 'failed', broken.detail); await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; } const recoveryTarget = Math.max(0, ...plan.requirements.filter(r => r.kind === 'recovery').map(r => r.target)); if (run.phase < recoveryTarget && progress.filter(p => !plan.requirements.some(r => r.id === p.id && r.kind === 'recovery')).every(p => p.passed)) { releaseRestartRequest(server.server, { kind: 'beta', id: String(id) }); db().prepare('UPDATE beta_servers SET restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND server = ?').run(id, server.server); stage(id, server.server, 'awaiting-empty'); activity(id, `${server.server}: recovery cycle ${run.phase + 1}/${recoveryTarget} queued; gameplay requirements will be repeated`); } else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing'); } const updated = listBetaServers(id); if (updated.some(server => !isActiveBetaServer(id, server.server))) return; const queued = updated.filter(s => s.stage === 'queued'); if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) { const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server); activity(id, `Previous wave passed; deploying ${wave.join(', ')}`); await deployBetaServers(id, wave); betaState(id, false); return; } const registry = db().prepare('SELECT server AS name, game, map FROM beta_test_coverage WHERE beta_id = ?').all(id) as Array<{ name: string; game: string | null; map: string | null }>; // All waves must be activated, even when the final coverage rule is a quorum. const ready = !queued.length && updated.every(s => !!currentRun(id, s.server)?.observing_at) && coveragePassed(plan, updated.map(s => ({ ...registry.find(r => r.name === s.server)!, passed: s.stage === 'ready' }))); betaState(id, ready); setOutcome(id, ready ? 'passed' : 'active', null); }