feat: add Watchdog recovery and drift rebuild workflows

This commit is contained in:
2026-09-27 15:13:38 -04:00
parent a7044f21a1
commit b9f5902945
36 changed files with 2131 additions and 424 deletions
+26 -312
View File
@@ -3,12 +3,9 @@
import { onMount } from 'svelte';
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import Dialog from '$ui/components/primitives/Dialog.svelte';
import type { DriftPlan, DriftResult } from '$lib/drift-plan.js';
import { pollVisible } from '$ui/poll.js';
import { confirmAction } from '$ui/confirm.js';
import { serverTitle } from '$ui/server-labels.js';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
import Card from '$ui/components/primitives/Card.svelte';
@@ -48,45 +45,23 @@
prior_version: number | null;
scripts: DepScript[];
}
interface DriftFile {
relPath: string;
size: number;
sha256: string;
kind: 'script' | 'config' | 'plugin' | 'baseline';
driftKind: 'untracked' | 'missing' | 'replace';
}
interface DriftServer { server: string; files: DriftFile[] }
interface DriftPreview { totalDrift: number; servers: DriftServer[]; scannedAt: string }
let loadError = $state('');
let limit = $state(25);
let driftLimit = $state(25);
$effect(() => { driftFilter; driftLimit = 25; });
let deployments = $state<Deployment[]>([]);
let loading = $state(true);
let drift = $state<DriftPreview | null>(null);
let scanning = $state(false);
let running = $state(false);
let rollingBack = $state<string | null>(null);
let flash = $state<string | null>(null);
let flashError = $state(false);
let flashId = $state('');
let selected = $state<string[]>([]);
let reviewOpen = $state(false);
let plan = $state<DriftPlan | null>(null);
let planError = $state('');
let reviewing = $state(false);
let reviewAll = $state(false);
let result = $state<DriftResult | null>(null);
const skipped = $derived(result?.perServer.filter(server => server.skipped.length) ?? []);
let driftError = $state('');
// Drift panel opens automatically when there's a problem; collapses when clean.
let driftOpen = $state(false);
let driftFilter = $state('');
$effect(() => { driftFilter = $page.url.searchParams.get('server') ?? ''; limit = Math.max(25, Number($page.url.searchParams.get('limit')) || 25); });
function saveContext() {
let serverFilter = $state('');
$effect(() => {
serverFilter = $page.url.searchParams.get('server') ?? '';
limit = Math.max(25, Number($page.url.searchParams.get('limit')) || 25);
});
function saveContext(reset = false) {
if (reset) limit = 25;
const url = new URL($page.url);
if (driftFilter) url.searchParams.set('server', driftFilter); else url.searchParams.delete('server');
if (serverFilter) url.searchParams.set('server', serverFilter); else url.searchParams.delete('server');
if (limit > 25) url.searchParams.set('limit', String(limit)); else url.searchParams.delete('limit');
void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
}
@@ -99,69 +74,7 @@
} catch (error) { loadError = (error as Error).message; }
finally { loading = false; }
}
async function scanDrift(): Promise<void> {
if (scanning) return;
scanning = true;
try {
const res = await fetch('/api/drift');
if (!res.ok) throw new Error('Could not scan drift.');
drift = (await res.json()) as DriftPreview;
selected = selected.filter(name => drift!.servers.some(server => server.server === name));
driftError = '';
} catch (error) { driftError = (error as Error).message;
} finally {
scanning = false;
}
}
let driftCounts = $derived.by(() => {
if (!drift) return { untracked: 0, missing: 0, replace: 0 };
let u = 0, m = 0, r = 0;
for (const s of drift.servers) for (const f of s.files) {
if (f.driftKind === 'untracked') u++; else if (f.driftKind === 'replace') r++; else m++;
}
return { untracked: u, missing: m, replace: r };
});
// Auto-open on first scan if there's drift; leave user's toggle alone after.
let firstScan = $state(true);
$effect(() => {
if (drift && firstScan) {
firstScan = false;
if (drift.totalDrift > 0) driftOpen = true;
}
});
async function reviewDrift(all: boolean, targets = selected): Promise<void> {
reviewOpen = true; reviewing = true; plan = null; planError = ''; reviewAll = all;
try {
const response = await fetch('/api/drift', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ dryRun: true, servers: all ? undefined : targets }) });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not prepare the reconciliation plan.');
plan = body.plan;
} catch (error) { planError = (error as Error).message; }
finally { reviewing = false; }
}
async function runDrift(): Promise<void> {
if (!plan || running) return;
running = true; planError = ''; result = null;
try {
const res = await fetch('/api/drift', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ servers: reviewAll ? undefined : plan.servers, expectedFingerprint: plan.fingerprint })
});
const body = await res.json();
if (!res.ok || !body.ok) { if (body.plan) plan = body.plan; throw new Error(body.error ?? 'Reconciliation failed.'); }
result = body.result; reviewOpen = false;
await Promise.all([loadCampaigns(), scanDrift()]);
} catch (error) { planError = (error as Error).message; } finally {
running = false;
}
}
onMount(() => { void scanDrift(); return pollVisible(loadCampaigns, 5000); });
onMount(() => pollVisible(loadCampaigns, 5000));
function isActive(d: Deployment): boolean {
return d.servers.some((s) => s.restart_state === 'pending' || s.restart_state === 'failed');
@@ -174,8 +87,11 @@
return `${Math.floor(d / 86_400_000)}d ago`;
}
let active = $derived(deployments.filter(isActive));
let history = $derived(deployments.filter((d) => !isActive(d)));
const matching = $derived(deployments.filter(d => !serverFilter || d.servers.some(server =>
(server.server + ' ' + serverTitle(server.server)).toLowerCase().includes(serverFilter.toLowerCase())
)));
let active = $derived(matching.filter(isActive));
let history = $derived(matching.filter((d) => !isActive(d)));
function scriptSummary(d: Deployment): string {
if (d.scripts && d.scripts.length > 0) {
@@ -218,12 +134,6 @@
return Array.from(groups.entries());
});
// Drift filtering — spare the operator's eyes on huge fleets.
let filteredDriftServers = $derived(
drift?.servers.filter((s) =>
!driftFilter || (s.server + ' ' + serverTitle(s.server)).toLowerCase().includes(driftFilter.toLowerCase())
) ?? []
);
</script>
<svelte:head><title>Campaigns | Electric Cherry</title></svelte:head>
@@ -231,106 +141,20 @@
<div class="page">
<SectionHeader
title="Campaigns"
subtitle="Track file campaigns, reconcile drift, and review rollbacks."
subtitle="Track file campaigns, server activation, and rollbacks."
/>
{#if loadError}<div class="ui-error" role="alert">{loadError}<Button size="sm" onclick={loadCampaigns}>Retry</Button></div>{/if}
{#if loading}
<div class="loading">Loading…</div>
{:else}
<!-- Drift summary — collapsed by default when there's no problem. -->
<Card padding="lg">
{#if driftError}<div class="ui-error" role="alert">{driftError}<Button size="sm" onclick={scanDrift} disabled={scanning}>Retry scan</Button></div>{/if}
<div class="drift-summary">
<button class="drift-summary-l" aria-expanded={driftOpen} aria-controls="drift-body" onclick={() => driftOpen = !driftOpen}>
<span class="chev" class:open={driftOpen} aria-hidden="true">›</span>
<span class="drift-title">Drift</span>
{#if drift === null}
<Badge variant="neutral" size="sm">Not scanned</Badge>
{:else if drift.totalDrift === 0}
<Badge variant="online" size="sm">In sync</Badge>
{:else}
{#if driftCounts.untracked > 0}
<Badge variant="danger" size="sm">{driftCounts.untracked} untracked</Badge>
{/if}
{#if driftCounts.missing > 0}
<Badge variant="warning" size="sm">{driftCounts.missing} missing</Badge>
{/if}
{#if driftCounts.replace > 0}
<Badge variant="warning" size="sm">{driftCounts.replace} to replace</Badge>
{/if}
<span class="drift-servers">
across {drift.servers.length} server{drift.servers.length === 1 ? '' : 's'}
</span>
{/if}
</button>
<div class="drift-summary-r">
<Button variant="ghost" size="sm" onclick={scanDrift} disabled={scanning || running} loading={scanning}>
Rescan
</Button>
<Button
variant="danger"
size="sm"
onclick={() => reviewDrift(false)}
disabled={running || scanning || reviewing || !selected.length}
loading={running}
>
Review {selected.length} selected
</Button>
<Button size="sm" variant="ghost" onclick={() => reviewDrift(true)} disabled={running || scanning || reviewing || !drift?.totalDrift}>Review entire fleet</Button>
</div>
<div class="watchdog-link">
<div><h2>Automatic drift repair</h2><p>Watchdog detects drift and rebuilds affected servers with the promoted setup. Repair campaigns remain in this history.</p></div>
<Button href={serverFilter ? `/watchdog?kind=drift&server=${encodeURIComponent(serverFilter)}` : '/watchdog?kind=drift'}>Open Watchdog</Button>
</div>
{#if driftOpen}
<div class="drift-body" id="drift-body">
{#if drift}<p class="scan-context">Scanned {$timeFormat.dateTime(drift.scannedAt)}. Selection controls the action scope; search only changes the visible list.</p>{/if}
{#if drift && drift.totalDrift > 0}
<input
class="drift-filter" aria-label="Filter drift by server"
type="search"
placeholder="Filter by server name…"
bind:value={driftFilter}
oninput={event => { driftFilter = event.currentTarget.value; saveContext(); }}
/>
<div class="selection-actions"><Button size="sm" onclick={() => selected = [...new Set([...selected, ...filteredDriftServers.map(server => server.server)])]}>Select visible</Button><Button size="sm" variant="ghost" disabled={!selected.length} onclick={() => selected = []}>Clear selection</Button><span>{selected.length} selected</span></div>
{#each filteredDriftServers.slice(0,driftLimit) as s (s.server)}
<div class="drift-server">
<div class="drift-server-head">
<label class="server-name"><input type="checkbox" checked={selected.includes(s.server)} onchange={event => selected = event.currentTarget.checked ? [...selected, s.server] : selected.filter(name => name !== s.server)} /> <ServerIdentity name={s.server} /></label>
<span class="server-count">{s.files.length} entr{s.files.length === 1 ? 'y' : 'ies'}</span>
</div>
<div class="drift-files">
{#each s.files as f (f.driftKind + ':' + f.kind + ':' + f.relPath)}
<Badge
variant={f.driftKind === 'untracked' ? 'danger' : 'warning'}
size="sm"
mono
title="[{f.kind}] {f.driftKind} · {f.size}B · {f.sha256.slice(0, 12)}"
>
[{f.driftKind === 'missing' ? '+' : f.driftKind === 'replace' ? '↻' : '−'} {f.kind}] {f.relPath}
</Badge>
{/each}
</div>
</div>
{/each}
<ListFooter total={filteredDriftServers.length} bind:limit={driftLimit} label="servers" />
{:else if drift?.totalDrift === 0}
<EmptyState title="Inventory matches disk" description="Nothing to reconcile." />
{:else}
<EmptyState title="Scan hasn't run yet" description="Click Rescan to look for drift." />
{/if}
</div>
{/if}
{#if result}<div class="result" class:result-error={skipped.length > 0} role={skipped.length ? 'alert' : 'status'}>
<strong>{result.totalServers} server{result.totalServers === 1 ? '' : 's'} updated{skipped.length ? `, ${skipped.length} with skipped files` : ''}</strong>
<p>{result.totalRemoved} files removed; {result.totalRestored} installed. {result.totalServers ? 'Affected servers restart when empty, then health is verified.' : skipped.length ? 'No changes could be applied. Review the reasons below.' : 'The reviewed servers already match the intended files.'}</p>
{#if result.deploymentId}<a href="/campaigns/{result.deploymentId}">View campaign →</a>{/if}
{#each skipped as server}<div><a href="/servers/{encodeURIComponent(server.server)}?tab=diagnostics"><ServerIdentity name={server.server} /></a><ul>{#each server.skipped as reason}<li>{reason}</li>{/each}</ul></div>{/each}
{#if skipped.length}<Button onclick={() => reviewDrift(false, skipped.map(server => server.server))}>Review unresolved servers</Button>{/if}
</div>{/if}
</Card>
<label class="server-filter">Server <input type="search" bind:value={serverFilter} oninput={event => { serverFilter = event.currentTarget.value; saveContext(true); }} placeholder="Filter campaigns by server name" /></label>
<!-- Active pipelines -->
<section class="section">
@@ -376,22 +200,10 @@
{/each}
{/if}
</section>
<ListFooter total={history.length} bind:limit onchange={saveContext} label="campaigns" />
<ListFooter total={history.length} bind:limit onchange={() => saveContext()} label="campaigns" />
{/if}
</div>
<Dialog bind:open={reviewOpen} title="Review reconciliation" busy={running || reviewing} wide>
{#if planError}<p class="ui-error" role="alert">{planError}</p>{/if}
{#if reviewing}<p role="status">Comparing current files with their intended versions…</p>
{:else if plan}
<p><strong>{reviewAll ? 'Entire fleet' : 'Selected servers'}: {plan.servers.length} server{plan.servers.length === 1 ? '' : 's'}, {plan.files.length} file change{plan.files.length === 1 ? '' : 's'}.</strong></p>
<p>Files change immediately. Each affected server restarts after a fresh check confirms it is empty, then waits for health verification. Untracked files will be removed.</p>
<p class="scan-context">Reviewed {$timeFormat.dateTime(plan.scannedAt)}. Changes to this plan require another review.</p>
<div class="plan-files">{#each plan.servers as server}{@const files = plan.files.filter(file => file.server === server)}{#if files.length}<section><h3><ServerIdentity name={server} /></h3>{#each files as file}<div class="plan-file"><strong>{file.driftKind === 'untracked' ? 'Remove' : file.driftKind === 'missing' ? 'Restore' : 'Replace'} <code>{file.relPath}</code></strong><span>{file.beforeSize === null ? 'Absent' : `${file.beforeSize} bytes`} → {file.afterSize === null ? 'Removed' : `${file.afterSize} bytes`}</span><details><summary>File fingerprints</summary><p>Before: {file.beforeSha256 ?? 'Absent'}<br />After: {file.afterSha256 ?? 'Removed'}</p></details></div>{/each}</section>{/if}{/each}</div>
{/if}
{#snippet actions()}<Button variant="ghost" disabled={running || reviewing} onclick={() => reviewOpen = false}>Cancel</Button><Button variant="danger" disabled={running || reviewing || !plan?.files.length} loading={running} onclick={runDrift}>Apply {plan?.files.length ?? 0} reviewed changes</Button>{/snippet}
</Dialog>
{#snippet deployRow(d: Deployment)}
<article class="deploy-row">
{#if flash && flashId === d.id}<div class="result" class:result-error={flashError} role={flashError ? 'alert' : 'status'}>{flash}</div>{/if}
@@ -447,16 +259,13 @@
{/snippet}
<style>
.scan-context { color: var(--text-secondary); font-size: 13px; line-height: 1.6; }
.selection-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; margin-block: 12px; }
.result { padding: 16px; margin-top: 16px; border: 1px solid var(--state-online); border-radius: var(--radius-md); line-height: 1.6; }
.watchdog-link { display: flex; align-items: center; justify-content: space-between; gap: 24px; flex-wrap: wrap; }
.watchdog-link h2 { margin: 0 0 8px; font-size: var(--text-md); }
.watchdog-link p { margin: 0; max-width: 680px; font-size: var(--text-sm); color: var(--text-secondary); line-height: 1.6; }
.server-filter { display: grid; gap: 8px; max-width: 400px; font-size: var(--text-sm); color: var(--text-secondary); }
.server-filter input { padding: 10px 12px; background: var(--surface-sunken); color: var(--text-primary); border: 1px solid var(--border-control); border-radius: var(--radius-sm); font: inherit; }
.result { padding: 16px; border: 1px solid var(--state-online); border-radius: var(--radius-md); line-height: 1.6; }
.result-error { border-color: var(--state-warning); }
.result p { margin-block: 8px; }
.result a { color: var(--accent-primary-hover); }
.plan-files { display: grid; gap: 20px; }
.plan-file { display: grid; gap: 8px; padding-block: 12px; border-bottom: 1px solid var(--border-subtle); overflow-wrap: anywhere; }
.plan-file span, .plan-file details { font-size: 13px; color: var(--text-secondary); }
.server-name { display: flex; align-items: center; gap: 10px; }
.page {
display: flex;
flex-direction: column;
@@ -468,98 +277,6 @@
color: var(--text-tertiary);
}
/* Drift summary bar */
.drift-summary {
display: flex;
align-items: center;
justify-content: space-between;
gap: var(--space-3);
width: 100%;
padding: 0;
background: transparent;
border: none;
cursor: pointer;
color: inherit;
text-align: left;
font: inherit;
}
.drift-summary:focus-visible { outline: none; box-shadow: var(--focus-ring); border-radius: var(--radius-md); }
.drift-summary-l {
display: flex;
align-items: center;
gap: var(--space-3);
min-width: 0;
flex-wrap: wrap;
}
.drift-summary-r {
display: flex;
align-items: center;
gap: var(--space-2);
flex-shrink: 0;
}
.drift-title {
font-size: var(--text-md);
font-weight: var(--weight-semibold);
color: var(--text-primary);
}
.drift-servers {
font-size: var(--text-sm);
color: var(--text-tertiary);
}
.chev {
color: var(--text-tertiary);
font-size: var(--text-lg);
line-height: 1;
transition: transform var(--dur-base) var(--ease-standard);
display: inline-block;
}
.chev.open { transform: rotate(90deg); }
.drift-body {
margin-top: var(--space-5);
padding-top: var(--space-5);
border-top: 1px solid var(--border-subtle);
}
.drift-filter {
width: 100%;
padding: var(--space-3) var(--space-4);
margin-bottom: var(--space-4);
background: var(--surface-sunken);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-md);
color: var(--text-primary);
font-family: var(--font-body);
font-size: var(--text-sm);
transition: border-color var(--dur-fast) var(--ease-standard);
}
.drift-filter:focus {
outline: none;
border-color: var(--accent-primary);
}
.drift-server {
padding: var(--space-3) var(--space-4);
margin: var(--space-2) 0;
background: var(--surface-sunken);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-md);
}
.drift-server-head {
display: flex;
justify-content: space-between;
align-items: baseline;
margin-bottom: var(--space-2);
font-family: var(--font-mono);
font-size: var(--text-sm);
}
.server-name { color: var(--text-primary); }
.server-count { color: var(--text-tertiary); font-size: var(--text-xs); }
.drift-files {
display: flex;
flex-wrap: wrap;
gap: var(--space-1);
}
/* Sections */
.section {
display: flex;
@@ -709,10 +426,7 @@
@media (max-width: 640px) {
.row-head, .row-foot { flex-direction: column; align-items: flex-start; }
.drift-summary { flex-direction: column; align-items: flex-start; }
.drift-summary-r { width: 100%; justify-content: flex-end; }
}
.drift-summary-l { padding:0; background:transparent; border:0; color:inherit; text-align:left; cursor:pointer; }
.state-count { display:inline-flex; align-items:center; gap:5px; }
.action { color:var(--text-primary); text-decoration:underline; text-underline-offset:4px; }
</style>
+5 -1
View File
@@ -35,6 +35,7 @@
servers: ServerRow[];
files: FileRow[];
events: ActivityEvent[];
watchdogIncidentId?: string | null;
}
let detail = $state<Detail | null>(null);
@@ -121,7 +122,9 @@
{/snippet}
</SectionHeader>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}</h2><p>Files are staged before activation. Servers restart when empty, then their health is verified.</p>{#if unresolved}<p>Review failed or waiting servers below. Diagnose connectivity or inspect startup logs from the affected row.</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}</h2><p>{detail.watchdogIncidentId ? 'Watchdog repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}</p>{#if unresolved}<p>Review failed or waiting servers below. Diagnose connectivity or inspect startup logs from the affected row.</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
{#if detail.watchdogIncidentId}<p class="watchdog-source">Started by Watchdog · <a href={`/watchdog/${encodeURIComponent(detail.watchdogIncidentId)}`}>View incident and recovery history</a></p>{/if}
<Card padding="lg">
<div class="meta-grid">
@@ -204,6 +207,7 @@
{/if}
<style>
.watchdog-source { margin: 0; color: var(--text-secondary); font-size: var(--text-sm); }.watchdog-source a { color: var(--accent-primary-hover); }
.outcome { padding: 20px; border: 1px solid var(--border-strong); border-radius: var(--radius-md); background: var(--surface); }.outcome h2 { font-size: 20px; margin: 0; }.outcome p { color: var(--text-secondary); line-height: 1.6; margin: 10px 0 0; }.srv-r a { color: var(--accent-primary-hover); font-size: 13px; }
.page {
display: flex;