Add configurable beta test plans and staged rollout

src/routes/api/betas/+server.ts

@@ -8,6 +8,8 @@ import {
 } from '$lib/server/fleet/beta.js';
 import { getScript, getVersion } from '$lib/server/fleet/scripts.js';
 import { kick } from '$lib/server/pipeline/worker.js';
+import { parseBetaPlan, type BetaPlan } from '$lib/beta-plan.js';
+import { storedPlan } from '$lib/server/fleet/beta-tests.js';
 
 export async function GET({ url }: { url: URL }) {
     const stateFilter = url.searchParams.get('state') as
@@ -32,6 +34,7 @@ export async function GET({ url }: { url: URL }) {
         });
         return {
             ...b,
+            testOutcome: storedPlan(b.id)?.outcome ?? null,
             scripts,
             servers: listBetaServers(b.id)
         };
@@ -51,7 +54,9 @@ export async function POST({ request }: { request: Request }) {
         waitForHealth?: boolean;
         waitForMatch?: boolean;
         notes?: string;
+        testPlan?: BetaPlan;
     };
+    if (!body || typeof body !== 'object' || Array.isArray(body)) return json({ ok: false, error: 'Expected a JSON object' }, { status: 400 });
 
     // Normalize into the openBeta shape. Both fields feed the same list.
     const rawEntries: Array<{ scriptId?: number; scriptVersionId?: number }> = [];
@@ -82,7 +87,11 @@ export async function POST({ request }: { request: Request }) {
     }
 
     try {
+        let testPlan: BetaPlan | undefined;
+        try { if (body.testPlan !== undefined) testPlan = parseBetaPlan(body.testPlan); }
+        catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); }
         const result = await openBeta({
+            testPlan,
             scripts: normalized,
             servers: body.servers,
             waitForHealth: body.waitForHealth ?? true,

src/routes/api/betas/[id]/+server.ts

@@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit';
 import { db } from '$lib/server/db/index.js';
 import { getBeta, listBetaServers, listBetaScripts } from '$lib/server/fleet/beta.js';
 import { getScript, getVersion } from '$lib/server/fleet/scripts.js';
+import { testView } from '$lib/server/fleet/beta-tests.js';
 
 function parseId(raw: string): number | null {
     const n = Number.parseInt(raw, 10);
@@ -30,5 +31,5 @@ export async function GET({ params }: { params: { id: string } }) {
     const servers = listBetaServers(id);
 
     const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id);
-    return json({ beta, scripts, servers, events });
+    return json({ beta, scripts, servers, events, test: testView(id) });
 }

src/routes/api/betas/[id]/retry/+server.ts

@@ -1,6 +1,7 @@
 import { json } from '@sveltejs/kit';
 import { db } from '$lib/server/db/index.js';
 import { emit, type BetaStage } from '$lib/server/pipeline/bus.js';
+import { storedPlan } from '$lib/server/fleet/beta-tests.js';
 
 function parseId(raw: string): number | null {
     const n = Number.parseInt(raw, 10);
@@ -12,6 +13,7 @@ function parseId(raw: string): number | null {
 export async function POST({ params, request }: { params: { id: string }; request: Request }) {
     const betaId = parseId(params.id);
     if (betaId === null) return json({ ok: false, error: 'bad id' }, { status: 400 });
+    if (storedPlan(betaId)) return json({ ok: false, error: 'Use the test plan retry action to collect fresh evidence' }, { status: 409 });
     const body = (await request.json().catch(() => ({}))) as { server?: string };
     if (!body.server) return json({ ok: false, error: 'server is required' }, { status: 400 });
 

src/routes/api/betas/[id]/rollback/+server.ts

@@ -1,5 +1,8 @@
 import { json } from '@sveltejs/kit';
 import { rollbackBeta, BetaError } from '$lib/server/fleet/beta.js';
+import { plannedBetaTick, rollbackTestPlan } from '$lib/server/pipeline/beta-plan-runner.js';
+import { storedPlan, setOutcome } from '$lib/server/fleet/beta-tests.js';
+import { parseBetaPlan } from '$lib/beta-plan.js';
 
 function parseId(raw: string): number | null {
     const n = Number.parseInt(raw, 10);
@@ -10,6 +13,13 @@ export async function POST({ params }: { params: { id: string } }) {
     const id = parseId(params.id);
     if (id === null) return json({ ok: false, error: 'bad id' }, { status: 400 });
     try {
+        await plannedBetaTick();
+        const test = storedPlan(id);
+        if (test) {
+            setOutcome(id, 'paused', 'Rollback requested');
+            await rollbackTestPlan(id, parseBetaPlan(JSON.parse(test.plan)));
+            return json({ ok: true });
+        }
         const result = await rollbackBeta(id);
         return json({ ok: true, result });
     } catch (err) {

src/routes/api/betas/[id]/test/+server.ts

@@ -0,0 +1,55 @@
+import { json } from '@sveltejs/kit';
+import { db } from '$lib/server/db/index.js';
+import { getBeta } from '$lib/server/fleet/beta.js';
+import { activity, currentRun, evidence, readMetrics, saveMetrics, setOutcome, storedPlan } from '$lib/server/fleet/beta-tests.js';
+import { betaState, plannedBetaTick } from '$lib/server/pipeline/beta-plan-runner.js';
+import { parseBetaPlan } from '$lib/beta-plan.js';
+import { kick } from '$lib/server/pipeline/worker.js';
+
+export async function POST({ params, request }: { params: { id: string }; request: Request }) {
+    const id = Number(params.id);
+    if (!Number.isSafeInteger(id) || id < 1) return json({ error: 'Invalid beta ID' }, { status: 400 });
+    await plannedBetaTick();
+    const beta = getBeta(id);
+    const stored = storedPlan(id);
+    if (!beta || !stored) return json({ error: 'Test plan not found' }, { status: 404 });
+    if (!['running', 'ready-promote'].includes(beta.state)) return json({ error: 'Beta is closed' }, { status: 409 });
+    try {
+        const body = await request.json();
+        const plan = parseBetaPlan(JSON.parse(stored.plan));
+        if (body.action === 'sign-off') {
+            if (!['active', 'passed'].includes(stored.outcome)) throw new Error('Resume or retry the test before signing off');
+            const requirement = plan.requirements.find(r => r.id === body.requirementId && r.kind === 'manual');
+            const run = typeof body.server === 'string' ? currentRun(id, body.server) : undefined;
+            if (!requirement || !run?.observing_at) throw new Error('Checklist item or active server run not found');
+            if (typeof body.note !== 'string' || !body.note.trim() || body.note.length > 1000) throw new Error('Add a tester name and observation (up to 1000 characters)');
+            const metrics = readMetrics(run);
+            metrics.manual[requirement.id] = body.note.trim();
+            saveMetrics(run, metrics);
+            evidence(run, `manual:${requirement.id}:${Date.now()}`, 'manual-sign-off', { requirement: requirement.label, note: body.note.trim() });
+            activity(id, `${body.server}: ${requirement.label} signed off: ${body.note.trim()}`);
+        } else if (body.action === 'pause') {
+            setOutcome(id, 'paused', 'Paused by operator; deployment and evidence evaluation held');
+            betaState(id, false);
+        } else if (body.action === 'resume') {
+            if (stored.outcome !== 'paused') throw new Error('Only paused plans can resume; failed or expired plans need a fresh attempt');
+            setOutcome(id, 'active', null);
+        } else if (body.action === 'retry') {
+            // Fresh attempts keep their predecessors as audit history. Preserve the original deadline:
+            // operators open another beta if the overall testing window has expired.
+            if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline');
+            if (db().prepare("SELECT 1 FROM beta_servers WHERE beta_id = ? AND error LIKE 'deploy:%'").get(id)) throw new Error('Deployment was incomplete. Roll back before opening a fresh beta');
+            db().transaction(() => {
+                db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND retired_at IS NULL').run(new Date().toISOString(), id);
+                db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND deployed_at IS NOT NULL AND error NOT LIKE 'deploy:%'").run(id);
+                // SQL NULL does not satisfy NOT LIKE.
+                db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND deployed_at IS NOT NULL AND error IS NULL").run(id);
+                betaState(id, false);
+                setOutcome(id, 'active', null);
+                activity(id, 'Fresh test attempt requested; deployed servers will restart when empty');
+            })();
+        } else throw new Error('Unknown test action');
+        void kick();
+        return json({ ok: true });
+    } catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); }
+}

src/routes/api/betas/presets/+server.ts

@@ -0,0 +1,18 @@
+import { json } from '@sveltejs/kit';
+import { db } from '$lib/server/db/index.js';
+import { builtInPlans, parseBetaPlan } from '$lib/beta-plan.js';
+export function GET() {
+    const saved = db().prepare('SELECT name, plan FROM beta_test_presets ORDER BY name').all() as Array<{ name: string; plan: string }>;
+    return json({ presets: [...builtInPlans, ...saved.map(p => ({ name: p.name, plan: JSON.parse(p.plan) }))] });
+}
+export async function POST({ request }: { request: Request }) {
+    try {
+        const body = await request.json();
+        if (typeof body.name !== 'string' || !body.name.trim() || body.name.length > 80) throw new Error('Preset name must be 1–80 characters');
+        if (builtInPlans.some(p => p.name === body.name.trim())) throw new Error('Choose a name other than a built-in preset');
+        const plan = parseBetaPlan(body.plan);
+        db().prepare('INSERT INTO beta_test_presets (name, plan) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET plan = excluded.plan')
+            .run(body.name.trim(), JSON.stringify(plan));
+        return json({ ok: true });
+    } catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); }
+}

src/routes/api/telemetry/+server.ts

@@ -2,6 +2,19 @@ import { json } from '@sveltejs/kit';
 import { getServer, listServers } from '$lib/server/fleet/servers.js';
 import { saveTelemetry } from '$lib/server/fleet/telemetry.js';
 import { emit } from '$lib/server/pipeline/bus.js';
+import { db } from '$lib/server/db/index.js';
+import { acceptTestSignal, parseTestSignal } from '$lib/server/fleet/beta-tests.js';
+
+// Reporters fetch this once at process startup and keep the token for that boot.
+export async function GET({ url }: { url: URL }) {
+    const server = url.searchParams.get('server');
+    if (!server || !getServer(server)) return json({ error: 'Unknown server' }, { status: 404 });
+    const runs = db().prepare(`SELECT r.id AS runId, r.beta_id AS betaId FROM beta_test_runs r
+        JOIN betas b ON b.id = r.beta_id JOIN beta_test_plans p ON p.beta_id = b.id
+        WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote')
+        AND p.outcome IN ('active','passed')`).all(server);
+    return json({ runs }, { headers: { 'Cache-Control': 'no-store' } });
+}
 
 export async function POST({ request }: { request: Request }) {
     console.log('[telemetry] received POST /api/telemetry');
@@ -30,7 +43,7 @@ export async function POST({ request }: { request: Request }) {
         return reject(400, 'Payload must be a JSON object');
     }
     const issues: string[] = [];
-    if (body.event !== 'round_change') issues.push('event must equal "round_change"');
+    if (body.runId === undefined && body.event !== 'round_change') issues.push('Session telemetry requires runId');
     if (typeof body.server !== 'string' || !body.server.length || body.server.length > 255) {
         issues.push('server must be a registered server name (1–255 characters)');
     }
@@ -56,6 +69,14 @@ export async function POST({ request }: { request: Request }) {
             receivedServer: body.server, destroyedAt: server.destroyed_at
         });
     }
+    let test: { accepted: boolean; reason?: string } | undefined;
+    if (body.runId !== undefined) {
+        try {
+            const signal = parseTestSignal(body);
+            test = acceptTestSignal(body.server, body.runId, body.eventId, signal);
+        } catch (error) { return reject(400, (error as Error).message); }
+        if (body.event !== 'round_change') return json({ ok: true, test });
+    }
     const telemetry = {
         server: body.server as string, round: body.round as number,
         players: body.players as number, receivedAt: new Date().toISOString()
@@ -63,5 +84,5 @@ export async function POST({ request }: { request: Request }) {
     saveTelemetry(telemetry);
     console.log(`[telemetry] accepted round_change ${JSON.stringify(telemetry)}`);
     emit({ type: 'server-telemetry', entries: [telemetry] });
-    return json({ ok: true, telemetry });
+    return json({ ok: true, telemetry, test });
 }

src/routes/betas/+page.svelte

@@ -1,4 +1,6 @@
 <script lang="ts">
+    import BetaPlanBuilder from "$ui/components/BetaPlanBuilder.svelte";
+    import { defaultPlan, parseBetaPlan } from "$lib/beta-plan.js";
     import { onMount, onDestroy } from 'svelte';
     import { goto } from '$app/navigation';
     import Card from '$ui/components/primitives/Card.svelte';
@@ -12,6 +14,7 @@
     interface BetaListRow {
         id: number;
         state: 'running' | 'ready-promote' | 'promoted' | 'rolled-back';
+        testOutcome: string | null;
         wait_for_health: number;
         wait_for_match: number;
         opened_at: string;
@@ -45,6 +48,7 @@
     let nextKey = 0;
     let picks = $state<Pick[]>([{ key: nextKey++, scriptId: null, versionId: null }]);
     let pickedServers = $state<Set<string>>(new Set());
+    let testPlan = $state(defaultPlan());
     let waitForHealth = $state(true);
     let waitForMatch = $state(false);
     let openingBeta = $state(false);
@@ -175,6 +179,7 @@
                 body: JSON.stringify({
                     scripts: picks.map((p) => ({ scriptId: p.scriptId!, scriptVersionId: p.versionId! })),
                     servers: [...pickedServers],
+                    testPlan: parseBetaPlan(testPlan),
                     waitForHealth,
                     waitForMatch
                 })
@@ -185,7 +190,7 @@
                 return;
             }
             if (body.result) await goto(`/betas/${body.result.betaId}`);
-        } finally {
+        } catch (error) { flash = (error as Error).message; } finally {
             openingBeta = false;
         }
     }
@@ -298,18 +303,7 @@
                         </div>
                     </Field>
 
-                    <Field label="Options">
-                        <div class="opts">
-                            <label class="chk">
-                                <input type="checkbox" bind:checked={waitForHealth} />
-                                <span>Wait for server start healthy</span>
-                            </label>
-                            <label class="chk">
-                                <input type="checkbox" bind:checked={waitForMatch} />
-                                <span>Wait for server match</span>
-                            </label>
-                        </div>
-                    </Field>
+                    <BetaPlanBuilder bind:plan={testPlan} />
 
                     <div class="form-actions">
                         <Button
@@ -341,7 +335,7 @@
                                 <a class="beta-row" href="/betas/{b.id}">
                                     <span class="beta-id">#{b.id}</span>
                                     <span class="beta-script">{summarizeScripts(b)}</span>
-                                    <Badge variant={betaStateVariant(b.state)} size="sm" uppercase>{b.state}</Badge>
+                                    <Badge variant={betaStateVariant(b.state)} size="sm" uppercase>{b.testOutcome && b.state === 'running' ? b.testOutcome : b.state}</Badge>
                                     <span class="beta-servers">{b.servers.length} server{b.servers.length === 1 ? '' : 's'}</span>
                                 </a>
                             {/each}
@@ -455,7 +449,6 @@
     }
     .chk input { accent-color: var(--accent-primary); cursor: pointer; }
 
-    .opts { display: flex; flex-wrap: wrap; gap: var(--space-6); }
 
     .form-actions { display: flex; align-items: center; gap: var(--space-4); }
     .flash { color: var(--state-danger); font-family: var(--font-mono); font-size: var(--text-xs); }

src/routes/betas/[id]/+page.svelte

@@ -1,4 +1,6 @@
 <script lang="ts">
+    import BetaTestProgress from "$ui/components/BetaTestProgress.svelte";
+    import type { BetaTestView } from "$lib/beta-plan.js";
     import { onMount, onDestroy } from 'svelte';
     import { page } from '$app/stores';
     import Card from '$ui/components/primitives/Card.svelte';
@@ -30,6 +32,7 @@
         version: { id: number; version: number; sha256: string } | null;
     }
 
+    let test = $state<BetaTestView | null>(null);
     let beta = $state<BetaRow | null>(null);
     let scripts = $state<BetaScript[]>([]);
     let servers = $state<BetaServerRow[]>([]);
@@ -50,7 +53,7 @@
             if (!res.ok) throw new Error(res.status === 404 ? 'Beta not found.' : 'Could not refresh beta.');
             const body = await res.json();
             if (current !== request || requestedId !== id) return;
-            beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events;
+            test = body.test; beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events;
             loadError = '';
         } catch (err) {
             if (current === request && requestedId === id) loadError = (err as Error).message;
@@ -79,7 +82,8 @@
         const prompt = isReject
             ? `Reject beta #${id}? This restores original files on every beta server and marks the version as test-failed.`
             : `Rollback beta #${id}? This restores original files on every beta server.`;
-        if (!confirm(prompt)) return;
+        const restart = test ? test.plan.rollbackRestart === 'when-empty' ? ' Restarts will roll out when servers are empty.' : ' Restart servers manually to load the restored files.' : '';
+        if (!confirm(prompt + restart)) return;
         acting = 'rollback'; flash = null;
         try {
             const res = await fetch(`/api/betas/${id}/rollback`, { method: 'POST' });
@@ -143,7 +147,7 @@
             </div>
         </Card>
 
-        <OperationHistory steps={betaFlow(b, servers)} {events} active={b.state === 'running' || b.state === 'ready-promote'} />
+        <OperationHistory steps={betaFlow({ ...b, testPlan: !!test }, servers)} {events} active={b.state === 'running' || b.state === 'ready-promote'} />
 
         <!-- Artifacts -->
         <Card padding="lg">
@@ -168,6 +172,9 @@
             {/if}
         </Card>
 
+        {#if test}
+            <Card padding="lg"><BetaTestProgress {test} betaId={id} closed={b.state === "promoted" || b.state === "rolled-back"} onUpdate={load} /></Card>
+        {:else}
         <!-- Options -->
         <Card padding="lg">
             <div class="panel-head"><h2>Options</h2></div>
@@ -183,6 +190,8 @@
             </div>
         </Card>
 
+        {/if}
+
         <!-- Server progress -->
         <Card padding="lg">
             <div class="panel-head">
@@ -206,7 +215,7 @@
                         {#if bs.error}
                             <span class="err">{bs.error}</span>
                         {/if}
-                        {#if bs.stage === 'failed'}
+                        {#if bs.stage === 'failed' && !test}
                             <Button
                                 variant="secondary"
                                 size="sm"
@@ -231,7 +240,7 @@
                     onclick={promote}
                     disabled={b.state !== 'ready-promote' || acting !== null}
                     loading={acting === 'promote'}
-                    title={b.state !== 'ready-promote' ? 'All servers must reach ready first' : ''}
+                    title={b.state !== 'ready-promote' ? 'The test plan must pass before promotion' : ''}
                 >
                     Promote
                 </Button>