Add configurable beta test plans and staged rollout

src/routes/api/betas/[id]/+server.ts

@@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit';
 import { db } from '$lib/server/db/index.js';
 import { getBeta, listBetaServers, listBetaScripts } from '$lib/server/fleet/beta.js';
 import { getScript, getVersion } from '$lib/server/fleet/scripts.js';
+import { testView } from '$lib/server/fleet/beta-tests.js';
 
 function parseId(raw: string): number | null {
     const n = Number.parseInt(raw, 10);
@@ -30,5 +31,5 @@ export async function GET({ params }: { params: { id: string } }) {
     const servers = listBetaServers(id);
 
     const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id);
-    return json({ beta, scripts, servers, events });
+    return json({ beta, scripts, servers, events, test: testView(id) });
 }

src/routes/api/betas/[id]/retry/+server.ts

@@ -1,6 +1,7 @@
 import { json } from '@sveltejs/kit';
 import { db } from '$lib/server/db/index.js';
 import { emit, type BetaStage } from '$lib/server/pipeline/bus.js';
+import { storedPlan } from '$lib/server/fleet/beta-tests.js';
 
 function parseId(raw: string): number | null {
     const n = Number.parseInt(raw, 10);
@@ -12,6 +13,7 @@ function parseId(raw: string): number | null {
 export async function POST({ params, request }: { params: { id: string }; request: Request }) {
     const betaId = parseId(params.id);
     if (betaId === null) return json({ ok: false, error: 'bad id' }, { status: 400 });
+    if (storedPlan(betaId)) return json({ ok: false, error: 'Use the test plan retry action to collect fresh evidence' }, { status: 409 });
     const body = (await request.json().catch(() => ({}))) as { server?: string };
     if (!body.server) return json({ ok: false, error: 'server is required' }, { status: 400 });
 

src/routes/api/betas/[id]/rollback/+server.ts

@@ -1,5 +1,8 @@
 import { json } from '@sveltejs/kit';
 import { rollbackBeta, BetaError } from '$lib/server/fleet/beta.js';
+import { plannedBetaTick, rollbackTestPlan } from '$lib/server/pipeline/beta-plan-runner.js';
+import { storedPlan, setOutcome } from '$lib/server/fleet/beta-tests.js';
+import { parseBetaPlan } from '$lib/beta-plan.js';
 
 function parseId(raw: string): number | null {
     const n = Number.parseInt(raw, 10);
@@ -10,6 +13,13 @@ export async function POST({ params }: { params: { id: string } }) {
     const id = parseId(params.id);
     if (id === null) return json({ ok: false, error: 'bad id' }, { status: 400 });
     try {
+        await plannedBetaTick();
+        const test = storedPlan(id);
+        if (test) {
+            setOutcome(id, 'paused', 'Rollback requested');
+            await rollbackTestPlan(id, parseBetaPlan(JSON.parse(test.plan)));
+            return json({ ok: true });
+        }
         const result = await rollbackBeta(id);
         return json({ ok: true, result });
     } catch (err) {

src/routes/api/betas/[id]/test/+server.ts

@@ -0,0 +1,55 @@
+import { json } from '@sveltejs/kit';
+import { db } from '$lib/server/db/index.js';
+import { getBeta } from '$lib/server/fleet/beta.js';
+import { activity, currentRun, evidence, readMetrics, saveMetrics, setOutcome, storedPlan } from '$lib/server/fleet/beta-tests.js';
+import { betaState, plannedBetaTick } from '$lib/server/pipeline/beta-plan-runner.js';
+import { parseBetaPlan } from '$lib/beta-plan.js';
+import { kick } from '$lib/server/pipeline/worker.js';
+
+export async function POST({ params, request }: { params: { id: string }; request: Request }) {
+    const id = Number(params.id);
+    if (!Number.isSafeInteger(id) || id < 1) return json({ error: 'Invalid beta ID' }, { status: 400 });
+    await plannedBetaTick();
+    const beta = getBeta(id);
+    const stored = storedPlan(id);
+    if (!beta || !stored) return json({ error: 'Test plan not found' }, { status: 404 });
+    if (!['running', 'ready-promote'].includes(beta.state)) return json({ error: 'Beta is closed' }, { status: 409 });
+    try {
+        const body = await request.json();
+        const plan = parseBetaPlan(JSON.parse(stored.plan));
+        if (body.action === 'sign-off') {
+            if (!['active', 'passed'].includes(stored.outcome)) throw new Error('Resume or retry the test before signing off');
+            const requirement = plan.requirements.find(r => r.id === body.requirementId && r.kind === 'manual');
+            const run = typeof body.server === 'string' ? currentRun(id, body.server) : undefined;
+            if (!requirement || !run?.observing_at) throw new Error('Checklist item or active server run not found');
+            if (typeof body.note !== 'string' || !body.note.trim() || body.note.length > 1000) throw new Error('Add a tester name and observation (up to 1000 characters)');
+            const metrics = readMetrics(run);
+            metrics.manual[requirement.id] = body.note.trim();
+            saveMetrics(run, metrics);
+            evidence(run, `manual:${requirement.id}:${Date.now()}`, 'manual-sign-off', { requirement: requirement.label, note: body.note.trim() });
+            activity(id, `${body.server}: ${requirement.label} signed off: ${body.note.trim()}`);
+        } else if (body.action === 'pause') {
+            setOutcome(id, 'paused', 'Paused by operator; deployment and evidence evaluation held');
+            betaState(id, false);
+        } else if (body.action === 'resume') {
+            if (stored.outcome !== 'paused') throw new Error('Only paused plans can resume; failed or expired plans need a fresh attempt');
+            setOutcome(id, 'active', null);
+        } else if (body.action === 'retry') {
+            // Fresh attempts keep their predecessors as audit history. Preserve the original deadline:
+            // operators open another beta if the overall testing window has expired.
+            if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline');
+            if (db().prepare("SELECT 1 FROM beta_servers WHERE beta_id = ? AND error LIKE 'deploy:%'").get(id)) throw new Error('Deployment was incomplete. Roll back before opening a fresh beta');
+            db().transaction(() => {
+                db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND retired_at IS NULL').run(new Date().toISOString(), id);
+                db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND deployed_at IS NOT NULL AND error NOT LIKE 'deploy:%'").run(id);
+                // SQL NULL does not satisfy NOT LIKE.
+                db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND deployed_at IS NOT NULL AND error IS NULL").run(id);
+                betaState(id, false);
+                setOutcome(id, 'active', null);
+                activity(id, 'Fresh test attempt requested; deployed servers will restart when empty');
+            })();
+        } else throw new Error('Unknown test action');
+        void kick();
+        return json({ ok: true });
+    } catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); }
+}