electric-cherry/src/lib/server/pipeline/beta-runner.ts

import { db } from '../db/index.js';
import { emit } from './bus.js';
import type { BetaStage } from './bus.js';
import { requestRestart } from '../provisioning/index.js';
import { plannedBetaTick } from './beta-plan-runner.js';

interface BetaServerActiveRow {
    beta_id: number;
    server: string;
    stage: BetaStage;
    healthy_at: string | null;
    match_seen_at: string | null;
    wait_for_health: number;
    wait_for_match: number;
}

function nowIso(): string {
    return new Date().toISOString();
}

const setStage = () =>
    db().prepare(
        `UPDATE beta_servers
         SET stage = ?, error = ?, updated_at = ?,
             restarted_at = CASE WHEN ? = 'awaiting-health' OR ? = 'ready' THEN COALESCE(restarted_at, ?) ELSE restarted_at END,
             healthy_at   = CASE WHEN ? = 'awaiting-match' OR (? = 'ready' AND healthy_at IS NULL) THEN COALESCE(healthy_at, ?) ELSE healthy_at END,
             match_seen_at = CASE WHEN ? = 'ready' AND match_seen_at IS NULL THEN ? ELSE match_seen_at END
         WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
    );

function transition(
    betaId: number,
    server: string,
    stage: BetaStage,
    error: string | null = null
): void {
    const t = nowIso();
    const result = setStage().run(
        stage,
        error,
        t,
        stage,
        stage,
        t,
        stage,
        stage,
        t,
        stage,
        t,
        betaId,
        server
    );
    if (result.changes) emit({ type: 'beta-server-stage', betaId, server, stage, error });
}

// Bump beta.state to ready-promote when every one of its beta_servers is ready.
// A single failed row is enough to keep the beta in 'running' (operator retries).
function reconcileBetaState(betaId: number): void {
    const rows = db()
        .prepare(`SELECT stage, server_deleted_at FROM beta_servers WHERE beta_id = ?`)
        .all(betaId) as Array<{ stage: BetaStage; server_deleted_at: string | null }>;
    if (rows.length === 0) return;
    if (rows.every((r) => r.stage === 'ready' && !r.server_deleted_at)) {
        const upd = db()
            .prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ? AND state = 'running'`)
            .run(betaId);
        if (upd.changes > 0) {
            // Every version bundled into this beta just cleared its gate.
            // 'tested' survives promote (tested → promoted is derived from
            // scripts.current_version_id, not stored) and reject (rollbackBeta
            // demotes back to test-failed per-version).
            const versionRows = db()
                .prepare(`SELECT script_version_id FROM beta_scripts WHERE beta_id = ?`)
                .all(betaId) as Array<{ script_version_id: number }>;
            const markTested = db().prepare(`UPDATE script_versions SET state = 'tested' WHERE id = ?`);
            for (const v of versionRows) markTested.run(v.script_version_id);
        }
        emit({ type: 'beta-state', betaId, state: 'ready-promote' });
    }
}

let betaInFlight: Promise<{ processed: number }> | null = null;

export function betaTick(): Promise<{ processed: number }> {
    if (betaInFlight) return betaInFlight;
    betaInFlight = runBetaTick().finally(() => { betaInFlight = null; });
    return betaInFlight;
}

async function runBetaTick(): Promise<{ processed: number }> {
    await plannedBetaTick();

    const rows = db()
        .prepare(
            `SELECT bs.beta_id AS beta_id, bs.server AS server, bs.stage AS stage,
                    bs.healthy_at AS healthy_at, bs.match_seen_at AS match_seen_at,
                    b.wait_for_health AS wait_for_health, b.wait_for_match AS wait_for_match
             FROM beta_servers bs
             JOIN betas b ON b.id = bs.beta_id
             WHERE b.state = 'running'
               AND bs.server_deleted_at IS NULL
               AND NOT EXISTS (SELECT 1 FROM beta_test_plans p WHERE p.beta_id = b.id)
               AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')
             ORDER BY b.id, bs.server`
        )
        .all() as BetaServerActiveRow[];

    if (rows.length === 0) return { processed: 0 };

    const touchedBetas = new Set<number>();
    for (const row of rows) {
        try {
            await advance(row);
        } catch (err) {
            transition(row.beta_id, row.server, 'failed', `runner: ${(err as Error).message}`);
        }
        touchedBetas.add(row.beta_id);
    }
    for (const id of touchedBetas) reconcileBetaState(id);

    return { processed: rows.length };
}

async function advance(row: BetaServerActiveRow): Promise<void> {
    switch (row.stage) {
        case 'awaiting-empty':
        case 'restarting':
        case 'awaiting-health': {
            const operation = requestRestart(row.server, { kind: 'beta', id: String(row.beta_id) });
            if (['failed','interrupted','cancelled'].includes(operation.state)) {
                transition(row.beta_id, row.server, 'failed', operation.error ?? operation.stage);
                return;
            }
            if (!operation.submitted_at || operation.state === 'running') {
                transition(row.beta_id, row.server, 'awaiting-empty', operation.stage);
                return;
            }
            db().prepare('UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?')
                .run(operation.submitted_at, row.beta_id, row.server);
            if (row.wait_for_health && operation.state !== 'done') {
                transition(row.beta_id, row.server, 'awaiting-health', operation.stage);
                return;
            }
            const next = row.wait_for_match ? 'awaiting-match' : 'ready';
            transition(row.beta_id, row.server, next);
            return;
        }
        case 'awaiting-match': {
            if (!row.match_seen_at) return;
            transition(row.beta_id, row.server, 'ready');
            return;
        }
    }
}