electric-cherry/src/lib/server/pipeline/beta-plan-runner.ts

import { db } from '../db/index.js';
import { betaOperationInFlight, deployBetaServers, getBeta, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
import { emit, type BetaStage } from './bus.js';
import { requestRestart, releaseRestartRequest } from '../provisioning/index.js';

function stage(id: number, server: string, next: BetaStage, error: string | null = null): void {
    const result = db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL')
        .run(next, error, new Date().toISOString(), id, server);
    if (result.changes) emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error });
}

export function betaState(id: number, ready: boolean): void {
    const state = ready ? 'ready-promote' : 'running';
    const result = db().prepare("UPDATE betas SET state = ? WHERE id = ? AND state IN ('running','ready-promote') AND state <> ?").run(state, id, state);
    if (result.changes) emit({ type: 'beta-state', betaId: id, state });
    // A revoked pass must not leave this version labelled tested unless another beta still proves it.
    db().prepare(`UPDATE script_versions SET state = ? WHERE id IN (SELECT script_version_id FROM beta_scripts WHERE beta_id = ?)
        AND id NOT IN (SELECT current_version_id FROM scripts WHERE current_version_id IS NOT NULL)
        AND NOT EXISTS (SELECT 1 FROM beta_scripts bs JOIN betas b ON b.id = bs.beta_id
            WHERE bs.script_version_id = script_versions.id AND b.id <> ? AND b.state IN ('ready-promote','promoted'))`)
        .run(ready ? 'tested' : 'untested', id, id);
}

export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void> {
    const result = await rollbackBeta(id, { restart: plan.rollbackRestart });
    if (!result.deploymentId) activity(id, 'Cancelled queued changes; no deployed files need restoration');
    else if (plan.rollbackRestart === 'when-empty') activity(id, 'Queued file restoration and rolling restarts; waiting for empty servers');
    else activity(id, 'File restoration is held; resume the rollback campaign to apply it with a safe restart');
    if (result.warnings.length) activity(id, result.warnings.join('; '));
}

async function fail(id: number, plan: BetaPlan, reason: string): Promise<void> {
    betaState(id, false);
    setOutcome(id, plan.failureAction === 'pause' ? 'paused' : 'failed', reason);
    if (plan.failureAction === 'rollback') {
        try { await rollbackTestPlan(id, plan); }
        catch (error) { setOutcome(id, 'failed', `Automatic rollback failed: ${(error as Error).message}`); }
    }
}

let inFlight: Promise<void> | null = null;
export function plannedBetaTick(): Promise<void> {
    if (inFlight) return inFlight;
    inFlight = tick().finally(() => { inFlight = null; });
    return inFlight;
}
async function tick(): Promise<void> {
    const plans = db().prepare(`SELECT p.* FROM beta_test_plans p JOIN betas b ON b.id = p.beta_id
        WHERE b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed') ORDER BY p.beta_id`).all() as StoredPlan[];
    for (const stored of plans) {
        if (betaOperationInFlight(stored.beta_id)) continue;
        try { await advancePlan(stored); }
        catch (error) {
            const reason = `Test runner: ${(error as Error).message}`;
            try { await fail(stored.beta_id, parseBetaPlan(JSON.parse(stored.plan)), reason); }
            catch { betaState(stored.beta_id, false); setOutcome(stored.beta_id, 'failed', reason); }
        }
    }
}

async function advancePlan(stored: StoredPlan): Promise<void> {
    const id = stored.beta_id;
    const plan = parseBetaPlan(JSON.parse(stored.plan));
    const beta = getBeta(id)!;
    const servers = listBetaServers(id);
    // Preserve the original coverage and evidence. A replacement with the same
    // name cannot complete a test that began on a different server.
    const missing = servers.filter(server => !isActiveBetaServer(id, server.server));
    if (missing.length) {
        betaState(id, false);
        setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`);
        return;
    }
    if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) {
        betaState(id, false);
        setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
        return;
    }
    for (const server of servers) {
        if (!isActiveBetaServer(id, server.server)) return;
        const freshPlan = storedPlan(id)!;
        if (!['active', 'passed'].includes(freshPlan.outcome)) return;
        if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; }
        if (server.stage === 'deploying' || server.stage === 'restarting') {
            stage(id, server.server, 'failed', `${server.stage === 'deploying' ? 'deploy: ' : ''}Operation interrupted; review the server before starting a fresh attempt`);
            await fail(id, plan, `${server.server}: interrupted ${server.stage}`); return;
        }
        if (server.stage === 'queued') continue;
        if (server.stage === 'awaiting-empty') {
            const waitingRun = currentRun(id, server.server);
            if (waitingRun?.observing_at) {
                await observeRun(waitingRun, plan);
                if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; }
                const broken = evaluateRequirements(plan, readMetrics(currentRun(id, server.server)!), waitingRun.phase).find(p => p.failed);
                if (broken) { await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
            }
            // Prepare evidence once before handing off the restart. Empty-server
            // checks and remote execution belong entirely to Provisioning.
            let run = currentRun(id, server.server);
            if (!run || run.observing_at) {
                const oldRun = run;
                run = await prepareRun(id, server.server, oldRun ? oldRun.phase + 1 : 0);
                if (oldRun) {
                    const metrics = readMetrics(run);
                    metrics.outages = readMetrics(oldRun).outages;
                    saveMetrics(run, metrics);
                }
            }
        }
        if (!isActiveBetaServer(id, server.server)) return;
        if (server.stage === 'awaiting-empty' || server.stage === 'awaiting-health') {
            const operation = requestRestart(server.server, { kind: 'beta', id: String(id) });
            if (['failed','interrupted','cancelled'].includes(operation.state)) {
                const reason = operation.error ?? operation.stage;
                stage(id, server.server, 'failed', reason); await fail(id, plan, reason); return;
            }
            if (!operation.submitted_at || operation.state === 'running') {
                stage(id, server.server, 'awaiting-empty', operation.stage);
                continue;
            }
            db().prepare('UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?')
                .run(operation.submitted_at, id, server.server);
            if (operation.state !== 'done') { stage(id, server.server, 'awaiting-health', operation.stage); continue; }
            const now = operation.finished_at ?? new Date().toISOString();
            db().prepare('UPDATE beta_servers SET healthy_at = ? WHERE beta_id = ? AND server = ?').run(now, id, server.server);
            db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(now, id, server.server);
            stage(id, server.server, 'observing');
        }
        const run = currentRun(id, server.server);
        if (!run?.observing_at) continue;
        await observeRun(run, plan);
        if (!isActiveBetaServer(id, server.server)) return;
        if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; }
        const metrics = readMetrics(currentRun(id, server.server)!);
        const progress = evaluateRequirements(plan, metrics, run.phase);
        const broken = progress.find(p => p.failed);
        if (broken) { stage(id, server.server, 'failed', broken.detail); await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
        const recoveryTarget = Math.max(0, ...plan.requirements.filter(r => r.kind === 'recovery').map(r => r.target));
        if (run.phase < recoveryTarget && progress.filter(p => !plan.requirements.some(r => r.id === p.id && r.kind === 'recovery')).every(p => p.passed)) {
            releaseRestartRequest(server.server, { kind: 'beta', id: String(id) });
            db().prepare('UPDATE beta_servers SET restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND server = ?').run(id, server.server);
            stage(id, server.server, 'awaiting-empty');
            activity(id, `${server.server}: recovery cycle ${run.phase + 1}/${recoveryTarget} queued; gameplay requirements will be repeated`);
        } else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing');
    }
    const updated = listBetaServers(id);
    if (updated.some(server => !isActiveBetaServer(id, server.server))) return;
    const queued = updated.filter(s => s.stage === 'queued');
    if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) {
        const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server);
        activity(id, `Previous wave passed; deploying ${wave.join(', ')}`);
        await deployBetaServers(id, wave);
        betaState(id, false);
        return;
    }
    const registry = db().prepare('SELECT server AS name, game, map FROM beta_test_coverage WHERE beta_id = ?').all(id) as Array<{ name: string; game: string | null; map: string | null }>;
    // All waves must be activated, even when the final coverage rule is a quorum.
    const ready = !queued.length && updated.every(s => !!currentRun(id, s.server)?.observing_at)
        && coveragePassed(plan, updated.map(s => ({ ...registry.find(r => r.name === s.server)!, passed: s.stage === 'ready' })));
    betaState(id, ready);
    setOutcome(id, ready ? 'passed' : 'active', null);
}