Improve operational UX, recovery flows, and action safety

Implement the approved UX review across Fleet, Betas, Library, deployments, and Settings. Add reviewed promotion and reconciliation plans, atomic queued-removal cancellation, recoverable beta drafts and launch inputs, accessible controls, and mobile improvements.

Validated with 473 tests, a clean typecheck, production build, and isolated desktop/mobile browser walkthroughs. Include the review and implementation coverage notes.

src/routes/campaigns/+page.svelte

@@ -1,5 +1,9 @@
 <script lang="ts">
     import { onMount } from 'svelte';
+    import { page } from '$app/stores';
+    import { goto } from '$app/navigation';
+    import Dialog from '$ui/components/primitives/Dialog.svelte';
+    import type { DriftPlan, DriftResult } from '$lib/drift-plan.js';
     import { pollVisible } from '$ui/poll.js';
     import { confirmAction } from '$ui/confirm.js';
     import { serverTitle } from '$ui/server-labels.js';
@@ -51,7 +55,7 @@
         driftKind: 'untracked' | 'missing' | 'replace';
     }
     interface DriftServer { server: string; files: DriftFile[] }
-    interface DriftPreview { totalDrift: number; servers: DriftServer[] }
+    interface DriftPreview { totalDrift: number; servers: DriftServer[]; scannedAt: string }
 
     let loadError = $state('');
     let limit = $state(25);
@@ -64,10 +68,27 @@
     let running = $state(false);
     let rollingBack = $state<string | null>(null);
     let flash = $state<string | null>(null);
+    let flashError = $state(false);
+    let flashId = $state('');
+    let selected = $state<string[]>([]);
+    let reviewOpen = $state(false);
+    let plan = $state<DriftPlan | null>(null);
+    let planError = $state('');
+    let reviewing = $state(false);
+    let reviewAll = $state(false);
+    let result = $state<DriftResult | null>(null);
+    const skipped = $derived(result?.perServer.filter(server => server.skipped.length) ?? []);
     let driftError = $state('');
     // Drift panel opens automatically when there's a problem; collapses when clean.
     let driftOpen = $state(false);
     let driftFilter = $state('');
+    $effect(() => { driftFilter = $page.url.searchParams.get('server') ?? ''; limit = Math.max(25, Number($page.url.searchParams.get('limit')) || 25); });
+    function saveContext() {
+        const url = new URL($page.url);
+        if (driftFilter) url.searchParams.set('server', driftFilter); else url.searchParams.delete('server');
+        if (limit > 25) url.searchParams.set('limit', String(limit)); else url.searchParams.delete('limit');
+        void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
+    }
 
     async function loadCampaigns(): Promise<void> {
         try {
@@ -84,6 +105,7 @@
             const res = await fetch('/api/drift');
             if (!res.ok) throw new Error('Could not scan drift.');
             drift = (await res.json()) as DriftPreview;
+            selected = selected.filter(name => drift!.servers.some(server => server.server === name));
             driftError = '';
         } catch (error) { driftError = (error as Error).message;
         } finally {
@@ -109,33 +131,31 @@
         }
     });
 
+    async function reviewDrift(all: boolean, targets = selected): Promise<void> {
+        reviewOpen = true; reviewing = true; plan = null; planError = ''; reviewAll = all;
+        try {
+            const response = await fetch('/api/drift', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ dryRun: true, servers: all ? undefined : targets }) });
+            const body = await response.json();
+            if (!response.ok) throw new Error(body.error ?? 'Could not prepare the reconciliation plan.');
+            plan = body.plan;
+        } catch (error) { planError = (error as Error).message; }
+        finally { reviewing = false; }
+    }
+
     async function runDrift(): Promise<void> {
-        if (!drift || drift.totalDrift === 0) return;
-        const parts: string[] = [];
-        if (driftCounts.untracked > 0) parts.push(`delete ${driftCounts.untracked} untracked file(s)`);
-        if (driftCounts.missing > 0) parts.push(`restore ${driftCounts.missing} missing artifact(s)`);
-        if (driftCounts.replace > 0) parts.push(`replace ${driftCounts.replace} existing artifact(s)`);
-        if (!await confirmAction({ title:'Reconcile drift?', message:`This will ${parts.join(' and ')} across ${drift.servers.length} servers.`, confirmLabel:'Reconcile', danger:true })) return;
-        running = true; flash = null;
+        if (!plan || running) return;
+        running = true; planError = ''; result = null;
         try {
             const res = await fetch('/api/drift', {
                 method: 'POST',
                 headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({})
+                body: JSON.stringify({ servers: reviewAll ? undefined : plan.servers, expectedFingerprint: plan.fingerprint })
             });
-            const body = (await res.json()) as {
-                ok: boolean; error?: string;
-                result?: { deploymentId: string | null; totalRemoved: number; totalRestored?: number; totalServers: number };
-            };
-            if (!body.ok) { flash = body.error ?? 'drift run failed'; return; }
-            const removed = body.result?.totalRemoved ?? 0;
-            const restored = body.result?.totalRestored ?? 0;
-            const parts2: string[] = [];
-            if (removed > 0) parts2.push(`removed ${removed}`);
-            if (restored > 0) parts2.push(`installed ${restored}`);
-            flash = `${parts2.join(', ') || 'no'} file(s) across ${body.result?.totalServers ?? 0} server(s)`;
+            const body = await res.json();
+            if (!res.ok || !body.ok) { if (body.plan) plan = body.plan; throw new Error(body.error ?? 'Reconciliation failed.'); }
+            result = body.result; reviewOpen = false;
             await Promise.all([loadCampaigns(), scanDrift()]);
-        } catch (error) { flash = (error as Error).message; } finally {
+        } catch (error) { planError = (error as Error).message; } finally {
             running = false;
         }
     }
@@ -174,14 +194,14 @@
         const summary = scriptSummary(d) || 'this promote';
         if (!await confirmAction({ title:'Roll back promotion?', message:`Restore the previous version of ${summary} on ${d.servers.length} servers.`, confirmLabel:'Roll back', danger:true })) return;
         rollingBack = d.id;
-        flash = null;
+        flash = null; flashId = d.id; flashError = false;
         try {
             const res = await fetch(`/api/campaigns/${d.id}/rollback`, { method: 'POST' });
             const body = (await res.json()) as { ok: boolean; error?: string };
-            if (!body.ok) { flash = body.error ?? 'rollback failed'; return; }
+            if (!res.ok || !body.ok) { flashError = true; flash = body.error ?? 'Rollback failed'; return; }
             flash = 'Rollback started';
             await loadCampaigns();
-        } catch (error) { flash = (error as Error).message; } finally {
+        } catch (error) { flashError = true; flash = (error as Error).message; } finally {
             rollingBack = null;
         }
     }
@@ -205,6 +225,8 @@
     );
 </script>
 
+<svelte:head><title>Rollouts | Electric Cherry</title></svelte:head>
+
 <div class="page">
     <SectionHeader
         title="Rollouts"
@@ -249,28 +271,32 @@
                     <Button
                         variant="danger"
                         size="sm"
-                        onclick={runDrift}
-                        disabled={running || scanning || !drift || drift.totalDrift === 0}
+                        onclick={() => reviewDrift(false)}
+                        disabled={running || scanning || reviewing || !selected.length}
                         loading={running}
                     >
-                        Reconcile
+                        Review {selected.length} selected
                     </Button>
+                    <Button size="sm" variant="ghost" onclick={() => reviewDrift(true)} disabled={running || scanning || reviewing || !drift?.totalDrift}>Review entire fleet</Button>
                 </div>
             </div>
 
             {#if driftOpen}
                 <div class="drift-body" id="drift-body">
+                    {#if drift}<p class="scan-context">Scanned {new Date(drift.scannedAt).toLocaleString()}. Selection controls the action scope; search only changes the visible list.</p>{/if}
                     {#if drift && drift.totalDrift > 0}
                         <input
                             class="drift-filter" aria-label="Filter drift by server"
                             type="search"
                             placeholder="Filter by server name…"
                             bind:value={driftFilter}
+                            oninput={event => { driftFilter = event.currentTarget.value; saveContext(); }}
                         />
+                        <div class="selection-actions"><Button size="sm" onclick={() => selected = [...new Set([...selected, ...filteredDriftServers.map(server => server.server)])]}>Select visible</Button><Button size="sm" variant="ghost" disabled={!selected.length} onclick={() => selected = []}>Clear selection</Button><span>{selected.length} selected</span></div>
                         {#each filteredDriftServers.slice(0,driftLimit) as s (s.server)}
                             <div class="drift-server">
                                 <div class="drift-server-head">
-                                    <span class="server-name"><ServerIdentity name={s.server} /></span>
+                                    <label class="server-name"><input type="checkbox" checked={selected.includes(s.server)} onchange={event => selected = event.currentTarget.checked ? [...selected, s.server] : selected.filter(name => name !== s.server)} /> <ServerIdentity name={s.server} /></label>
                                     <span class="server-count">{s.files.length} entr{s.files.length === 1 ? 'y' : 'ies'}</span>
                                 </div>
                                 <div class="drift-files">
@@ -296,9 +322,13 @@
                 </div>
             {/if}
 
-            {#if flash}
-                <div class="flash">{flash}</div>
-            {/if}
+            {#if result}<div class="result" class:result-error={skipped.length > 0} role={skipped.length ? 'alert' : 'status'}>
+                <strong>{result.totalServers} server{result.totalServers === 1 ? '' : 's'} updated{skipped.length ? `, ${skipped.length} with skipped files` : ''}</strong>
+                <p>{result.totalRemoved} files removed; {result.totalRestored} installed. {result.totalServers ? 'Affected servers restart when empty, then health is verified.' : skipped.length ? 'No changes could be applied. Review the reasons below.' : 'The reviewed servers already match the intended files.'}</p>
+                {#if result.deploymentId}<a href="/campaigns/{result.deploymentId}">View rollout →</a>{/if}
+                {#each skipped as server}<div><a href="/servers/{encodeURIComponent(server.server)}?tab=diagnostics"><ServerIdentity name={server.server} /></a><ul>{#each server.skipped as reason}<li>{reason}</li>{/each}</ul></div>{/each}
+                {#if skipped.length}<Button onclick={() => reviewDrift(false, skipped.map(server => server.server))}>Review unresolved servers</Button>{/if}
+            </div>{/if}
         </Card>
 
         <!-- Active pipelines -->
@@ -345,12 +375,25 @@
                 {/each}
             {/if}
         </section>
-        <ListFooter total={history.length} bind:limit label="rollouts" />
+        <ListFooter total={history.length} bind:limit onchange={saveContext} label="rollouts" />
     {/if}
 </div>
 
+<Dialog bind:open={reviewOpen} title="Review reconciliation" busy={running || reviewing} wide>
+    {#if planError}<p class="ui-error" role="alert">{planError}</p>{/if}
+    {#if reviewing}<p role="status">Comparing current files with their intended versions…</p>
+    {:else if plan}
+        <p><strong>{reviewAll ? 'Entire fleet' : 'Selected servers'}: {plan.servers.length} server{plan.servers.length === 1 ? '' : 's'}, {plan.files.length} file change{plan.files.length === 1 ? '' : 's'}.</strong></p>
+        <p>Files change immediately. Each affected server restarts after a fresh check confirms it is empty, then waits for health verification. Untracked files will be removed.</p>
+        <p class="scan-context">Reviewed {new Date(plan.scannedAt).toLocaleString()}. Changes to this plan require another review.</p>
+        <div class="plan-files">{#each plan.servers as server}{@const files = plan.files.filter(file => file.server === server)}{#if files.length}<section><h3><ServerIdentity name={server} /></h3>{#each files as file}<div class="plan-file"><strong>{file.driftKind === 'untracked' ? 'Remove' : file.driftKind === 'missing' ? 'Restore' : 'Replace'} <code>{file.relPath}</code></strong><span>{file.beforeSize === null ? 'Absent' : `${file.beforeSize} bytes`} → {file.afterSize === null ? 'Removed' : `${file.afterSize} bytes`}</span><details><summary>File fingerprints</summary><p>Before: {file.beforeSha256 ?? 'Absent'}<br />After: {file.afterSha256 ?? 'Removed'}</p></details></div>{/each}</section>{/if}{/each}</div>
+    {/if}
+    {#snippet actions()}<Button variant="ghost" disabled={running || reviewing} onclick={() => reviewOpen = false}>Cancel</Button><Button variant="danger" disabled={running || reviewing || !plan?.files.length} loading={running} onclick={runDrift}>Apply {plan?.files.length ?? 0} reviewed changes</Button>{/snippet}
+</Dialog>
+
 {#snippet deployRow(d: Deployment)}
     <article class="deploy-row">
+        {#if flash && flashId === d.id}<div class="result" class:result-error={flashError} role={flashError ? 'alert' : 'status'}>{flash}</div>{/if}
         <div class="row-head">
             <div class="row-head-l">
                 <a class="action" href="/campaigns/{d.id}">{d.action} rollout <span class="sr-only">{d.id}</span></a>
@@ -384,7 +427,7 @@
                     title="{s.server}: {s.restart_state}{s.error ? ' — ' + s.error : ''}"
                 >
                     <span class="dot"></span>
-                    <span class="node-label">{serverTitle(s.server)}{s.players !== null && s.players > 0 ? ` (${s.players}p)` : ''}</span>
+                    <a class="node-label" href="/servers/{encodeURIComponent(s.server)}?tab=diagnostics">{serverTitle(s.server)}{s.players !== null && s.players > 0 ? ` (${s.players} players at last check)` : ''}</a><StatusBadge state={s.restart_state} />
                 </span>
             {/each}
         </div>
@@ -403,6 +446,16 @@
 {/snippet}
 
 <style>
+    .scan-context { color: var(--text-secondary); font-size: 13px; line-height: 1.6; }
+    .selection-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; margin-block: 12px; }
+    .result { padding: 16px; margin-top: 16px; border: 1px solid var(--state-online); border-radius: var(--radius-md); line-height: 1.6; }
+    .result-error { border-color: var(--state-warning); }
+    .result p { margin-block: 8px; }
+    .result a { color: var(--accent-primary-hover); }
+    .plan-files { display: grid; gap: 20px; }
+    .plan-file { display: grid; gap: 8px; padding-block: 12px; border-bottom: 1px solid var(--border-subtle); overflow-wrap: anywhere; }
+    .plan-file span, .plan-file details { font-size: 13px; color: var(--text-secondary); }
+    .server-name { display: flex; align-items: center; gap: 10px; }
     .page {
         display: flex;
         flex-direction: column;
@@ -505,15 +558,6 @@
         gap: var(--space-1);
     }
 
-    .flash {
-        margin-top: var(--space-3);
-        padding: var(--space-3) var(--space-4);
-        background: rgba(124, 212, 255, 0.06);
-        border: 1px solid rgba(124, 212, 255, 0.22);
-        border-radius: var(--radius-md);
-        color: var(--state-online);
-        font-size: var(--text-sm);
-    }
 
     /* Sections */
     .section {