josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Watchdog recovery and drift rebuild workflows

Browse Source
This commit is contained in:
josh
2026-09-27 15:13:38 -04:00
parent a7044f21a1
commit b9f5902945
36 changed files with 2131 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/routes/+layout.svelte
+4 -4
View File
@@ -52,7 +52,7 @@
// Two-key nav shortcuts (Gmail-style): `g f` fleet, `g d` provisioning,
// `g b` betas, `g s` scripts. Ignored while typing in inputs.
const shortcuts: Record<string, string> = {
f: '/', d: '/provisioning', a: '/campaigns', b: '/betas',
f: '/', d: '/provisioning', a: '/campaigns', w: '/watchdog', b: '/betas',
k: '/server-keys', s: '/scripts', c: '/configs', p: '/plugins', l: '/leaderboard', h: '/settings', m: '/metrics'
};
let armed = false;
@@ -99,7 +99,7 @@
const tabs = [
{ href: '/', label: 'Fleet', match: (p: string) => p === '/' || p.startsWith('/servers/') },
{ href: '/scripts', label: 'Library', match: (p: string) => /^\/(scripts|configs|plugins)(\/|$)/.test(p) },
{ href: '/provisioning', label: 'Operations', match: (p: string) => /^\/(provisioning|deployments|collections|campaigns|server-keys)(\/|$)/.test(p) },
{ href: '/provisioning', label: 'Operations', match: (p: string) => /^\/(provisioning|deployments|collections|campaigns|watchdog|server-keys)(\/|$)/.test(p) },
{ href: '/betas', label: 'Betas', match: (p: string) => p.startsWith('/betas') },
{ href: '/settings', label: 'Settings', match: (p: string) => p.startsWith('/settings') },
{ href: '/leaderboard', label: 'Leaderboard', match: (p: string) => p.startsWith('/leaderboard') },
@@ -109,7 +109,7 @@
const secondary = $derived(tabs[1].match($page.url.pathname)
? [{ href: '/scripts', label: 'Scripts' }, { href: '/configs', label: 'Configs' }, { href: '/plugins', label: 'Plugins' }]
: tabs[2].match($page.url.pathname)
? [{ href: '/provisioning', label: 'Provisioning' }, { href: '/collections', label: 'Collections' }, { href: '/campaigns', label: 'Campaigns' }, { href: '/server-keys', label: 'Keys' }]
? [{ href: '/provisioning', label: 'Provisioning' }, { href: '/collections', label: 'Collections' }, { href: '/campaigns', label: 'Campaigns' }, { href: '/watchdog', label: 'Watchdog' }, { href: '/server-keys', label: 'Keys' }]
: tabs[4].match($page.url.pathname) ? [{ href: '/settings', label: 'Preferences' }] : []);
</script>
@@ -149,7 +149,7 @@
<Confirmation />
<NotificationHistory bind:open={notificationsOpen} />
<Dialog bind:open={helpOpen} title="Keyboard shortcuts" description="Press g, then the destination key. Shortcuts are inactive while typing or using a dialog.">
<dl class="shortcuts">{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Provisioning'],['g a','Campaigns'],['g b','Betas'],['g h','Settings'],['g k','Keys'],['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}<div><dt><kbd>{keys}</kbd></dt><dd>{label}</dd></div>{/each}</dl>
<dl class="shortcuts">{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Provisioning'],['g a','Campaigns'],['g w','Watchdog'],['g b','Betas'],['g h','Settings'],['g k','Keys'],['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}<div><dt><kbd>{keys}</kbd></dt><dd>{label}</dd></div>{/each}</dl>
</Dialog>
</div>
src/routes/api/campaigns/[id]/+server.ts
+2 -1
View File
@@ -39,5 +39,6 @@ export async function GET({ params }: { params: { id: string } }) {
.all(params.id);
const events = db().prepare('SELECT id, created_at, message FROM campaign_events WHERE deployment_id = ? ORDER BY id').all(params.id);
return json({ deployment: dep, servers, files, scripts, events });
const watchdog = db().prepare('SELECT incident_id FROM watchdog_actions WHERE campaign_id = ? LIMIT 1').get(params.id) as { incident_id: string } | undefined;
return json({ deployment: dep, servers, files, scripts, events, watchdogIncidentId: watchdog?.incident_id ?? null });
}
src/routes/api/drift/+server.ts
+3 -8
View File
@@ -1,6 +1,5 @@
import { json } from '@sveltejs/kit';
import { DriftPlanChanged, previewDrift, runDriftDeployment } from '$lib/server/fleet/drift.js';
import { kick } from '$lib/server/pipeline/worker.js';
import { previewDrift } from '$lib/server/fleet/drift.js';
// GET returns the current drift plan for the whole fleet without touching disk.
export async function GET() {
@@ -27,7 +26,7 @@ export async function GET() {
});
}
// POST removes drift and creates a drift deployment. Body: { servers?: string[], dryRun?: boolean }
// Legacy previews remain readable. Watchdog owns all new reconciliation requests.
export async function POST({ request }: { request: Request }) {
const body = await request.json().catch(() => null);
if (!body || (body.servers !== undefined && (!Array.isArray(body.servers) || !body.servers.every((server: unknown) => typeof server === 'string')))) {
@@ -35,12 +34,8 @@ export async function POST({ request }: { request: Request }) {
}
try {
if (body.dryRun === true) return json({ ok: true, plan: await previewDrift(body.servers) });
if (typeof body.expectedFingerprint !== 'string' || !body.expectedFingerprint) return json({ ok: false, error: 'Review a reconciliation plan before applying changes.' }, { status: 400 });
const result = await runDriftDeployment({ servers: body.servers, expectedFingerprint: body.expectedFingerprint, triggeredBy: 'ui' });
if (result.deploymentId) void kick();
return json({ ok: true, result });
return json({ ok: false, error: 'Drift reconciliation has moved to Watchdog. Scan there to queue a rebuild and campaign.', watchdogUrl: '/watchdog' }, { status: 410 });
} catch (error) {
if (error instanceof DriftPlanChanged) return json({ ok: false, error: error.message, plan: error.plan }, { status: 409 });
return json({ ok: false, error: (error as Error).message }, { status: 400 });
}
}
src/routes/api/watchdog/+server.ts
+18
View File
@@ -0,0 +1,18 @@
import { json } from '@sveltejs/kit';
import { watchdogOverview, watchdogTick, setWatchdogEnabled } from '$lib/server/watchdog/index.js';
import { kick } from '$lib/server/pipeline/worker.js';
export function GET() { return json(watchdogOverview()); }
export async function POST({ request }: { request: Request }) {
const body = await request.json().catch(() => null);
if (!body || typeof body !== 'object' || Array.isArray(body)
|| (body.enabled !== undefined && typeof body.enabled !== 'boolean')
|| (body.action !== 'scan' && typeof body.enabled !== 'boolean')
|| (body.action !== undefined && body.action !== 'scan')) {
return json({ error: 'Provide action: scan or an enabled boolean.' }, { status: 400 });
}
if (typeof body.enabled === 'boolean') setWatchdogEnabled(body.enabled);
if (body.action === 'scan') await watchdogTick(true);
void kick().catch(err => console.error('[watchdog] worker kick failed:', err));
return json(watchdogOverview());
}
src/routes/api/watchdog/[id]/+server.ts
+16
View File
@@ -0,0 +1,16 @@
import { json } from '@sveltejs/kit';
import { watchdogDetail, updateWatchdogIncident } from '$lib/server/watchdog/index.js';
import { kick } from '$lib/server/pipeline/worker.js';
export function GET({ params }: { params: { id: string } }) {
const detail = watchdogDetail(params.id);
return detail ? json(detail) : json({ error: 'Watchdog incident not found.' }, { status: 404 });
}
export async function POST({ params, request }: { params: { id: string }; request: Request }) {
const body = await request.json().catch(() => null);
if (!body || !['retry','dismiss'].includes(body.action)) return json({ error: 'Choose retry or dismiss.' }, { status: 400 });
const result = updateWatchdogIncident(params.id, body.action);
if (!result.ok) return json({ error: result.error }, { status: result.status });
void kick().catch(err => console.error('[watchdog] worker kick failed:', err));
return json(watchdogDetail(params.id));
}
src/routes/campaigns/+page.svelte
+26 -312
View File
@@ -3,12 +3,9 @@
import { onMount } from 'svelte';
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import Dialog from '$ui/components/primitives/Dialog.svelte';
import type { DriftPlan, DriftResult } from '$lib/drift-plan.js';
import { pollVisible } from '$ui/poll.js';
import { confirmAction } from '$ui/confirm.js';
import { serverTitle } from '$ui/server-labels.js';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
import Card from '$ui/components/primitives/Card.svelte';
@@ -48,45 +45,23 @@
prior_version: number | null;
scripts: DepScript[];
}
interface DriftFile {
relPath: string;
size: number;
sha256: string;
kind: 'script' | 'config' | 'plugin' | 'baseline';
driftKind: 'untracked' | 'missing' | 'replace';
}
interface DriftServer { server: string; files: DriftFile[] }
interface DriftPreview { totalDrift: number; servers: DriftServer[]; scannedAt: string }
let loadError = $state('');
let limit = $state(25);
let driftLimit = $state(25);
$effect(() => { driftFilter; driftLimit = 25; });
let deployments = $state<Deployment[]>([]);
let loading = $state(true);
let drift = $state<DriftPreview | null>(null);
let scanning = $state(false);
let running = $state(false);
let rollingBack = $state<string | null>(null);
let flash = $state<string | null>(null);
let flashError = $state(false);
let flashId = $state('');
let selected = $state<string[]>([]);
let reviewOpen = $state(false);
let plan = $state<DriftPlan | null>(null);
let planError = $state('');
let reviewing = $state(false);
let reviewAll = $state(false);
let result = $state<DriftResult | null>(null);
const skipped = $derived(result?.perServer.filter(server => server.skipped.length) ?? []);
let driftError = $state('');
// Drift panel opens automatically when there's a problem; collapses when clean.
let driftOpen = $state(false);
let driftFilter = $state('');
$effect(() => { driftFilter = $page.url.searchParams.get('server') ?? ''; limit = Math.max(25, Number($page.url.searchParams.get('limit')) || 25); });
function saveContext() {
let serverFilter = $state('');
$effect(() => {
serverFilter = $page.url.searchParams.get('server') ?? '';
limit = Math.max(25, Number($page.url.searchParams.get('limit')) || 25);
});
function saveContext(reset = false) {
if (reset) limit = 25;
const url = new URL($page.url);
if (driftFilter) url.searchParams.set('server', driftFilter); else url.searchParams.delete('server');
if (serverFilter) url.searchParams.set('server', serverFilter); else url.searchParams.delete('server');
if (limit > 25) url.searchParams.set('limit', String(limit)); else url.searchParams.delete('limit');
void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
}
@@ -99,69 +74,7 @@
} catch (error) { loadError = (error as Error).message; }
finally { loading = false; }
}
async function scanDrift(): Promise<void> {
if (scanning) return;
scanning = true;
try {
const res = await fetch('/api/drift');
if (!res.ok) throw new Error('Could not scan drift.');
drift = (await res.json()) as DriftPreview;
selected = selected.filter(name => drift!.servers.some(server => server.server === name));
driftError = '';
} catch (error) { driftError = (error as Error).message;
} finally {
scanning = false;
}
}
let driftCounts = $derived.by(() => {
if (!drift) return { untracked: 0, missing: 0, replace: 0 };
let u = 0, m = 0, r = 0;
for (const s of drift.servers) for (const f of s.files) {
if (f.driftKind === 'untracked') u++; else if (f.driftKind === 'replace') r++; else m++;
}
return { untracked: u, missing: m, replace: r };
});
// Auto-open on first scan if there's drift; leave user's toggle alone after.
let firstScan = $state(true);
$effect(() => {
if (drift && firstScan) {
firstScan = false;
if (drift.totalDrift > 0) driftOpen = true;
}
});
async function reviewDrift(all: boolean, targets = selected): Promise<void> {
reviewOpen = true; reviewing = true; plan = null; planError = ''; reviewAll = all;
try {
const response = await fetch('/api/drift', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ dryRun: true, servers: all ? undefined : targets }) });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not prepare the reconciliation plan.');
plan = body.plan;
} catch (error) { planError = (error as Error).message; }
finally { reviewing = false; }
}
async function runDrift(): Promise<void> {
if (!plan || running) return;
running = true; planError = ''; result = null;
try {
const res = await fetch('/api/drift', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ servers: reviewAll ? undefined : plan.servers, expectedFingerprint: plan.fingerprint })
});
const body = await res.json();
if (!res.ok || !body.ok) { if (body.plan) plan = body.plan; throw new Error(body.error ?? 'Reconciliation failed.'); }
result = body.result; reviewOpen = false;
await Promise.all([loadCampaigns(), scanDrift()]);
} catch (error) { planError = (error as Error).message; } finally {
running = false;
}
}
onMount(() => { void scanDrift(); return pollVisible(loadCampaigns, 5000); });
onMount(() => pollVisible(loadCampaigns, 5000));
function isActive(d: Deployment): boolean {
return d.servers.some((s) => s.restart_state === 'pending' || s.restart_state === 'failed');
@@ -174,8 +87,11 @@
return `${Math.floor(d / 86_400_000)}d ago`;
}
let active = $derived(deployments.filter(isActive));
let history = $derived(deployments.filter((d) => !isActive(d)));
const matching = $derived(deployments.filter(d => !serverFilter || d.servers.some(server =>
(server.server + ' ' + serverTitle(server.server)).toLowerCase().includes(serverFilter.toLowerCase())
)));
let active = $derived(matching.filter(isActive));
let history = $derived(matching.filter((d) => !isActive(d)));
function scriptSummary(d: Deployment): string {
if (d.scripts && d.scripts.length > 0) {
@@ -218,12 +134,6 @@
return Array.from(groups.entries());
});
// Drift filtering — spare the operator's eyes on huge fleets.
let filteredDriftServers = $derived(
drift?.servers.filter((s) =>
!driftFilter || (s.server + ' ' + serverTitle(s.server)).toLowerCase().includes(driftFilter.toLowerCase())
) ?? []
);
</script>
<svelte:head><title>Campaigns | Electric Cherry</title></svelte:head>
@@ -231,106 +141,20 @@
<div class="page">
<SectionHeader
title="Campaigns"
subtitle="Track file campaigns, reconcile drift, and review rollbacks."
subtitle="Track file campaigns, server activation, and rollbacks."
/>
{#if loadError}<div class="ui-error" role="alert">{loadError}<Button size="sm" onclick={loadCampaigns}>Retry</Button></div>{/if}
{#if loading}
<div class="loading">Loading…</div>
{:else}
<!-- Drift summary — collapsed by default when there's no problem. -->
<Card padding="lg">
{#if driftError}<div class="ui-error" role="alert">{driftError}<Button size="sm" onclick={scanDrift} disabled={scanning}>Retry scan</Button></div>{/if}
<div class="drift-summary">
<button class="drift-summary-l" aria-expanded={driftOpen} aria-controls="drift-body" onclick={() => driftOpen = !driftOpen}>
<span class="chev" class:open={driftOpen} aria-hidden="true">›</span>
<span class="drift-title">Drift</span>
{#if drift === null}
<Badge variant="neutral" size="sm">Not scanned</Badge>
{:else if drift.totalDrift === 0}
<Badge variant="online" size="sm">In sync</Badge>
{:else}
{#if driftCounts.untracked > 0}
<Badge variant="danger" size="sm">{driftCounts.untracked} untracked</Badge>
{/if}
{#if driftCounts.missing > 0}
<Badge variant="warning" size="sm">{driftCounts.missing} missing</Badge>
{/if}
{#if driftCounts.replace > 0}
<Badge variant="warning" size="sm">{driftCounts.replace} to replace</Badge>
{/if}
<span class="drift-servers">
across {drift.servers.length} server{drift.servers.length === 1 ? '' : 's'}
</span>
{/if}
</button>
<div class="drift-summary-r">
<Button variant="ghost" size="sm" onclick={scanDrift} disabled={scanning || running} loading={scanning}>
Rescan
</Button>
<Button
variant="danger"
size="sm"
onclick={() => reviewDrift(false)}
disabled={running || scanning || reviewing || !selected.length}
loading={running}
>
Review {selected.length} selected
</Button>
<Button size="sm" variant="ghost" onclick={() => reviewDrift(true)} disabled={running || scanning || reviewing || !drift?.totalDrift}>Review entire fleet</Button>
</div>
<div class="watchdog-link">
<div><h2>Automatic drift repair</h2><p>Watchdog detects drift and rebuilds affected servers with the promoted setup. Repair campaigns remain in this history.</p></div>
<Button href={serverFilter ? `/watchdog?kind=drift&server=${encodeURIComponent(serverFilter)}` : '/watchdog?kind=drift'}>Open Watchdog</Button>
</div>
{#if driftOpen}
<div class="drift-body" id="drift-body">
{#if drift}<p class="scan-context">Scanned {$timeFormat.dateTime(drift.scannedAt)}. Selection controls the action scope; search only changes the visible list.</p>{/if}
{#if drift && drift.totalDrift > 0}
<input
class="drift-filter" aria-label="Filter drift by server"
type="search"
placeholder="Filter by server name…"
bind:value={driftFilter}
oninput={event => { driftFilter = event.currentTarget.value; saveContext(); }}
/>
<div class="selection-actions"><Button size="sm" onclick={() => selected = [...new Set([...selected, ...filteredDriftServers.map(server => server.server)])]}>Select visible</Button><Button size="sm" variant="ghost" disabled={!selected.length} onclick={() => selected = []}>Clear selection</Button><span>{selected.length} selected</span></div>
{#each filteredDriftServers.slice(0,driftLimit) as s (s.server)}
<div class="drift-server">
<div class="drift-server-head">
<label class="server-name"><input type="checkbox" checked={selected.includes(s.server)} onchange={event => selected = event.currentTarget.checked ? [...selected, s.server] : selected.filter(name => name !== s.server)} /> <ServerIdentity name={s.server} /></label>
<span class="server-count">{s.files.length} entr{s.files.length === 1 ? 'y' : 'ies'}</span>
</div>
<div class="drift-files">
{#each s.files as f (f.driftKind + ':' + f.kind + ':' + f.relPath)}
<Badge
variant={f.driftKind === 'untracked' ? 'danger' : 'warning'}
size="sm"
mono
title="[{f.kind}] {f.driftKind} · {f.size}B · {f.sha256.slice(0, 12)}"
>
[{f.driftKind === 'missing' ? '+' : f.driftKind === 'replace' ? '↻' : '−'} {f.kind}] {f.relPath}
</Badge>
{/each}
</div>
</div>
{/each}
<ListFooter total={filteredDriftServers.length} bind:limit={driftLimit} label="servers" />
{:else if drift?.totalDrift === 0}
<EmptyState title="Inventory matches disk" description="Nothing to reconcile." />
{:else}
<EmptyState title="Scan hasn't run yet" description="Click Rescan to look for drift." />
{/if}
</div>
{/if}
{#if result}<div class="result" class:result-error={skipped.length > 0} role={skipped.length ? 'alert' : 'status'}>
<strong>{result.totalServers} server{result.totalServers === 1 ? '' : 's'} updated{skipped.length ? `, ${skipped.length} with skipped files` : ''}</strong>
<p>{result.totalRemoved} files removed; {result.totalRestored} installed. {result.totalServers ? 'Affected servers restart when empty, then health is verified.' : skipped.length ? 'No changes could be applied. Review the reasons below.' : 'The reviewed servers already match the intended files.'}</p>
{#if result.deploymentId}<a href="/campaigns/{result.deploymentId}">View campaign →</a>{/if}
{#each skipped as server}<div><a href="/servers/{encodeURIComponent(server.server)}?tab=diagnostics"><ServerIdentity name={server.server} /></a><ul>{#each server.skipped as reason}<li>{reason}</li>{/each}</ul></div>{/each}
{#if skipped.length}<Button onclick={() => reviewDrift(false, skipped.map(server => server.server))}>Review unresolved servers</Button>{/if}
</div>{/if}
</Card>
<label class="server-filter">Server <input type="search" bind:value={serverFilter} oninput={event => { serverFilter = event.currentTarget.value; saveContext(true); }} placeholder="Filter campaigns by server name" /></label>
<!-- Active pipelines -->
<section class="section">
@@ -376,22 +200,10 @@
{/each}
{/if}
</section>
<ListFooter total={history.length} bind:limit onchange={saveContext} label="campaigns" />
<ListFooter total={history.length} bind:limit onchange={() => saveContext()} label="campaigns" />
{/if}
</div>
<Dialog bind:open={reviewOpen} title="Review reconciliation" busy={running || reviewing} wide>
{#if planError}<p class="ui-error" role="alert">{planError}</p>{/if}
{#if reviewing}<p role="status">Comparing current files with their intended versions…</p>
{:else if plan}
<p><strong>{reviewAll ? 'Entire fleet' : 'Selected servers'}: {plan.servers.length} server{plan.servers.length === 1 ? '' : 's'}, {plan.files.length} file change{plan.files.length === 1 ? '' : 's'}.</strong></p>
<p>Files change immediately. Each affected server restarts after a fresh check confirms it is empty, then waits for health verification. Untracked files will be removed.</p>
<p class="scan-context">Reviewed {$timeFormat.dateTime(plan.scannedAt)}. Changes to this plan require another review.</p>
<div class="plan-files">{#each plan.servers as server}{@const files = plan.files.filter(file => file.server === server)}{#if files.length}<section><h3><ServerIdentity name={server} /></h3>{#each files as file}<div class="plan-file"><strong>{file.driftKind === 'untracked' ? 'Remove' : file.driftKind === 'missing' ? 'Restore' : 'Replace'} <code>{file.relPath}</code></strong><span>{file.beforeSize === null ? 'Absent' : `${file.beforeSize} bytes`} → {file.afterSize === null ? 'Removed' : `${file.afterSize} bytes`}</span><details><summary>File fingerprints</summary><p>Before: {file.beforeSha256 ?? 'Absent'}<br />After: {file.afterSha256 ?? 'Removed'}</p></details></div>{/each}</section>{/if}{/each}</div>
{/if}
{#snippet actions()}<Button variant="ghost" disabled={running || reviewing} onclick={() => reviewOpen = false}>Cancel</Button><Button variant="danger" disabled={running || reviewing || !plan?.files.length} loading={running} onclick={runDrift}>Apply {plan?.files.length ?? 0} reviewed changes</Button>{/snippet}
</Dialog>
{#snippet deployRow(d: Deployment)}
<article class="deploy-row">
{#if flash && flashId === d.id}<div class="result" class:result-error={flashError} role={flashError ? 'alert' : 'status'}>{flash}</div>{/if}
@@ -447,16 +259,13 @@
{/snippet}
<style>
.scan-context { color: var(--text-secondary); font-size: 13px; line-height: 1.6; }
.selection-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; margin-block: 12px; }
.result { padding: 16px; margin-top: 16px; border: 1px solid var(--state-online); border-radius: var(--radius-md); line-height: 1.6; }
.watchdog-link { display: flex; align-items: center; justify-content: space-between; gap: 24px; flex-wrap: wrap; }
.watchdog-link h2 { margin: 0 0 8px; font-size: var(--text-md); }
.watchdog-link p { margin: 0; max-width: 680px; font-size: var(--text-sm); color: var(--text-secondary); line-height: 1.6; }
.server-filter { display: grid; gap: 8px; max-width: 400px; font-size: var(--text-sm); color: var(--text-secondary); }
.server-filter input { padding: 10px 12px; background: var(--surface-sunken); color: var(--text-primary); border: 1px solid var(--border-control); border-radius: var(--radius-sm); font: inherit; }
.result { padding: 16px; border: 1px solid var(--state-online); border-radius: var(--radius-md); line-height: 1.6; }
.result-error { border-color: var(--state-warning); }
.result p { margin-block: 8px; }
.result a { color: var(--accent-primary-hover); }
.plan-files { display: grid; gap: 20px; }
.plan-file { display: grid; gap: 8px; padding-block: 12px; border-bottom: 1px solid var(--border-subtle); overflow-wrap: anywhere; }
.plan-file span, .plan-file details { font-size: 13px; color: var(--text-secondary); }
.server-name { display: flex; align-items: center; gap: 10px; }
.page {
display: flex;
flex-direction: column;
@@ -468,98 +277,6 @@
color: var(--text-tertiary);
}
/* Drift summary bar */
.drift-summary {
display: flex;
align-items: center;
justify-content: space-between;
gap: var(--space-3);
width: 100%;
padding: 0;
background: transparent;
border: none;
cursor: pointer;
color: inherit;
text-align: left;
font: inherit;
}
.drift-summary:focus-visible { outline: none; box-shadow: var(--focus-ring); border-radius: var(--radius-md); }
.drift-summary-l {
display: flex;
align-items: center;
gap: var(--space-3);
min-width: 0;
flex-wrap: wrap;
}
.drift-summary-r {
display: flex;
align-items: center;
gap: var(--space-2);
flex-shrink: 0;
}
.drift-title {
font-size: var(--text-md);
font-weight: var(--weight-semibold);
color: var(--text-primary);
}
.drift-servers {
font-size: var(--text-sm);
color: var(--text-tertiary);
}
.chev {
color: var(--text-tertiary);
font-size: var(--text-lg);
line-height: 1;
transition: transform var(--dur-base) var(--ease-standard);
display: inline-block;
}
.chev.open { transform: rotate(90deg); }
.drift-body {
margin-top: var(--space-5);
padding-top: var(--space-5);
border-top: 1px solid var(--border-subtle);
}
.drift-filter {
width: 100%;
padding: var(--space-3) var(--space-4);
margin-bottom: var(--space-4);
background: var(--surface-sunken);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-md);
color: var(--text-primary);
font-family: var(--font-body);
font-size: var(--text-sm);
transition: border-color var(--dur-fast) var(--ease-standard);
}
.drift-filter:focus {
outline: none;
border-color: var(--accent-primary);
}
.drift-server {
padding: var(--space-3) var(--space-4);
margin: var(--space-2) 0;
background: var(--surface-sunken);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-md);
}
.drift-server-head {
display: flex;
justify-content: space-between;
align-items: baseline;
margin-bottom: var(--space-2);
font-family: var(--font-mono);
font-size: var(--text-sm);
}
.server-name { color: var(--text-primary); }
.server-count { color: var(--text-tertiary); font-size: var(--text-xs); }
.drift-files {
display: flex;
flex-wrap: wrap;
gap: var(--space-1);
}
/* Sections */
.section {
display: flex;
@@ -709,10 +426,7 @@
@media (max-width: 640px) {
.row-head, .row-foot { flex-direction: column; align-items: flex-start; }
.drift-summary { flex-direction: column; align-items: flex-start; }
.drift-summary-r { width: 100%; justify-content: flex-end; }
}
.drift-summary-l { padding:0; background:transparent; border:0; color:inherit; text-align:left; cursor:pointer; }
.state-count { display:inline-flex; align-items:center; gap:5px; }
.action { color:var(--text-primary); text-decoration:underline; text-underline-offset:4px; }
</style>
src/routes/campaigns/[id]/+page.svelte
+5 -1
View File
@@ -35,6 +35,7 @@
servers: ServerRow[];
files: FileRow[];
events: ActivityEvent[];
watchdogIncidentId?: string | null;
}
let detail = $state<Detail | null>(null);
@@ -121,7 +122,9 @@
{/snippet}
</SectionHeader>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}</h2><p>Files are staged before activation. Servers restart when empty, then their health is verified.</p>{#if unresolved}<p>Review failed or waiting servers below. Diagnose connectivity or inspect startup logs from the affected row.</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}</h2><p>{detail.watchdogIncidentId ? 'Watchdog repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}</p>{#if unresolved}<p>Review failed or waiting servers below. Diagnose connectivity or inspect startup logs from the affected row.</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
{#if detail.watchdogIncidentId}<p class="watchdog-source">Started by Watchdog · <a href={`/watchdog/${encodeURIComponent(detail.watchdogIncidentId)}`}>View incident and recovery history</a></p>{/if}
<Card padding="lg">
<div class="meta-grid">
@@ -204,6 +207,7 @@
{/if}
<style>
.watchdog-source { margin: 0; color: var(--text-secondary); font-size: var(--text-sm); }.watchdog-source a { color: var(--accent-primary-hover); }
.outcome { padding: 20px; border: 1px solid var(--border-strong); border-radius: var(--radius-md); background: var(--surface); }.outcome h2 { font-size: 20px; margin: 0; }.outcome p { color: var(--text-secondary); line-height: 1.6; margin: 10px 0 0; }.srv-r a { color: var(--accent-primary-hover); font-size: 13px; }
.page {
display: flex;
src/routes/provisioning/[id]/+page.svelte
+2 -2
View File
@@ -22,7 +22,7 @@
stage: string; started_at: string; finished_at: string | null;
triggered_by: string; error: string | null; warnings: string; server_exists: number;
}
interface Detail { deployment: Deployment; events: { id: number; created_at: string; message: string }[]; sources: { kind: 'campaign' | 'beta' | 'maintenance'; source_id: string }[]; recipe: LaunchRecipeSummary | null; blueprintName: string | null; canResumeVerification: boolean; canResumeCleanup: boolean; canResumeRebuild: boolean; canCancelRemoval: boolean; collection: { id: string; name: string } | null }
interface Detail { deployment: Deployment; events: { id: number; created_at: string; message: string }[]; sources: { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string }[]; recipe: LaunchRecipeSummary | null; blueprintName: string | null; canResumeVerification: boolean; canResumeCleanup: boolean; canResumeRebuild: boolean; canCancelRemoval: boolean; collection: { id: string; name: string } | null }
let detail = $state<Detail | null>(null);
let error = $state('');
let refreshing = false;
@@ -152,7 +152,7 @@
{#each warnings as warning}<p class="warning">{warning}</p>{/each}
{#if deployment.expected_map}<p>Expected map: <strong>{deployment.expected_map}</strong></p>{/if}
<dl><div><dt>Started</dt><dd>{fmt(deployment.started_at)}</dd></div><div><dt>Finished</dt><dd>{deployment.finished_at ? fmt(deployment.finished_at) : 'In progress'}</dd></div><div><dt>Requested by</dt><dd>{deployment.triggered_by}</dd></div></dl>
{#if detail?.sources.length}<div class="recovery-links" aria-label="Requesting work">{#each detail.sources as source}<a href={source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}` : source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}` : '/settings#plutonium'}>{source.kind === 'campaign' ? 'Campaign' : source.kind === 'beta' ? 'Beta' : 'Maintenance'} {source.source_id}</a>{/each}</div>{/if}
{#if detail?.sources.length}<div class="recovery-links" aria-label="Requesting work">{#each detail.sources as source}<a href={source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}` : source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}` : source.kind === 'watchdog' ? `/watchdog/${encodeURIComponent(source.source_id)}` : '/settings#plutonium'}>{source.kind === 'campaign' ? 'Campaign' : source.kind === 'beta' ? 'Beta' : source.kind === 'watchdog' ? 'Watchdog incident' : 'Maintenance'} {source.source_id}</a>{/each}</div>{/if}
</section>
{#if detail?.recipe}<section class="saved-recipe"><div class="top"><h2>{detail.blueprintName ? `Launched from ${detail.blueprintName}` : 'Recorded launch files'}</h2><Button variant="ghost" onclick={() => open = true}>Launch another server →</Button></div><p class="muted">Launch this map with the latest promoted files using a fresh server ID and port. This does not repair the existing server.</p><LaunchRecipe recipe={detail.recipe} historical /></section><DeployServerDialog bind:open sourceDeploymentId={deployment.id} />{/if}
<OperationHistory steps={steps.map((label,index): OperationStep => ({label,state:deployment.state === 'cancelled' ? 'pending' : index < currentStep ? 'done' : index === currentStep ? (!active && ['failed','interrupted'].includes(deployment.state) ? 'failed' : 'current') : 'pending'}))} events={detail?.events ?? []} {active} />
src/routes/watchdog/+page.svelte
+168
View File
@@ -0,0 +1,168 @@
<script lang="ts">
import { onMount } from 'svelte';
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import { watchdogStateLabels, type WatchdogOverview, type WatchdogState } from '$lib/watchdog.js';
import { pollVisible } from '$ui/poll.js';
import { timeFormat } from '$ui/stores/time-zone.js';
import { serverTitle } from '$ui/server-labels.js';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import WatchdogStatus from '$ui/components/WatchdogStatus.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
import Button from '$ui/components/primitives/Button.svelte';
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
let overview = $state<WatchdogOverview | null>(null);
let loadError = $state('');
let actionError = $state('');
let notice = $state('');
let busy = $state<'scan' | 'toggle' | null>(null);
let filterState = $state('active');
let kind = $state('all');
let server = $state('');
let limit = $state(25);
let request = 0;
const states = Object.keys(watchdogStateLabels) as WatchdogState[];
const activeStates = ['open', 'recovering', 'attention'];
const active = $derived(overview?.incidents.filter(incident => activeStates.includes(incident.state)).length ?? 0);
const attention = $derived(overview?.incidents.filter(incident => incident.state === 'attention').length ?? 0);
const matching = $derived(overview?.incidents.filter(incident =>
(filterState === 'all' || (filterState === 'active' ? activeStates.includes(incident.state) : incident.state === filterState)) &&
(kind === 'all' || incident.kind === kind) &&
(!server || `${incident.server} ${serverTitle(incident.server)}`.toLowerCase().includes(server.toLowerCase()))
) ?? []);
$effect(() => {
const params = $page.url.searchParams;
const selectedState = params.get('state') ?? 'active';
filterState = ['active', 'all', ...states].includes(selectedState) ? selectedState : 'active';
kind = ['health', 'drift'].includes(params.get('kind') ?? '') ? params.get('kind')! : 'all';
server = params.get('server') ?? '';
limit = Math.max(25, Number(params.get('limit')) || 25);
});
function saveContext(reset = false) {
if (reset) limit = 25;
const url = new URL($page.url);
for (const [key, value] of Object.entries({ state: filterState === 'active' ? '' : filterState, kind: kind === 'all' ? '' : kind, server, limit: limit > 25 ? String(limit) : '' })) {
if (value) url.searchParams.set(key, value); else url.searchParams.delete(key);
}
void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
}
async function load() {
if (busy) return;
const current = ++request;
try {
const response = await fetch('/api/watchdog');
if (!response.ok) throw new Error('Could not refresh Watchdog.');
const body = await response.json() as WatchdogOverview;
if (current !== request) return;
overview = body; loadError = '';
} catch (error) {
if (current === request) loadError = (error as Error).message;
}
}
async function update(action: 'scan' | 'toggle') {
if (busy || !overview) return;
busy = action; actionError = ''; notice = '';
const current = ++request;
const enabled = !overview.enabled;
try {
const response = await fetch('/api/watchdog', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(action === 'scan' ? { action: 'scan' } : { enabled })
});
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not update Watchdog.');
if (current !== request) return;
overview = body; loadError = '';
notice = action === 'scan' ? (body.scanError ? 'Scan finished with errors. Review the scan details below.' : 'Scan complete.') : enabled ? 'Automatic recovery resumed.' : 'Automatic recovery paused. Queued Watchdog recovery is held; already submitted operations continue.';
} catch (error) {
if (current === request) actionError = (error as Error).message;
} finally { if (current === request) busy = null; }
}
onMount(() => {
const stop = pollVisible(load, 5000);
return () => { stop(); request++; };
});
</script>
<svelte:head><title>Watchdog | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title="Watchdog" subtitle="Automatic server recovery and drift repair.">
{#snippet actions()}
<Button disabled={!overview || !!busy} loading={busy === 'scan'} onclick={() => update('scan')}>Scan now</Button>
<Button disabled={!overview || !!busy} loading={busy === 'toggle'} onclick={() => update('toggle')}>{overview?.enabled ? 'Pause automation' : 'Resume automation'}</Button>
{/snippet}
</SectionHeader>
{#if loadError}<div class="ui-error" role="alert">{loadError} {overview ? 'Showing the last successful refresh.' : ''}<Button size="sm" disabled={!!busy} onclick={load}>Retry</Button></div>{/if}
{#if actionError}<p class="ui-error" role="alert">{actionError}</p>{/if}
{#if notice}<p class="notice" role="status">{notice}</p>{/if}
{#if overview}
<section class="automation" aria-label="Automation status">
<div class="automation-heading"><h2>Automatic recovery</h2><Badge variant={overview.enabled ? 'online' : 'warning'}>{overview.enabled ? 'Enabled' : 'Paused'}</Badge></div>
<p>{overview.enabled ? 'Watchdog restarts unhealthy servers, then rebuilds if recovery fails.' : 'Watchdog continues detecting incidents while automatic recovery is paused. Queued Watchdog recovery is held; already submitted operations continue.'}</p>
<p>Drift repair rebuilds the server with the latest promoted setup and records a campaign. <strong>Rebuilds delete the existing instance and all of its files.</strong> Interruption requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry.</p>
<div class="meta"><span>{active} active incident{active === 1 ? '' : 's'} · {attention} need attention</span><span>Last scan: {overview.lastScanAt ? $timeFormat.dateTime(overview.lastScanAt) : 'Not yet scanned'}</span></div>
{#if overview.scanError}<p class="ui-error" role="alert">Last scan: {overview.scanError}</p>{/if}
</section>
<section aria-label="Incidents">
<div class="filters">
<label>Status<select bind:value={filterState} onchange={event => { filterState = event.currentTarget.value; saveContext(true); }}><option value="active">Active incidents</option><option value="all">All incidents</option>{#each states as value}<option value={value}>{watchdogStateLabels[value]}</option>{/each}</select></label>
<label>Reason<select bind:value={kind} onchange={event => { kind = event.currentTarget.value; saveContext(true); }}><option value="all">All reasons</option><option value="health">Server health</option><option value="drift">File drift</option></select></label>
<label class="server-filter">Server<input type="search" bind:value={server} oninput={event => { server = event.currentTarget.value; saveContext(true); }} placeholder="Filter by server name" /></label>
</div>
<h2>Incidents <span class="muted">{matching.length}</span></h2>
{#if !matching.length}
<EmptyState title={filterState === 'active' && kind === 'all' && !server ? 'No active incidents' : 'No matching incidents'} description={filterState === 'active' && kind === 'all' && !server ? 'Detected health and drift problems will appear here automatically.' : 'Change the filters to review other incidents.'} />
{:else}
<div class="incidents">
{#each matching.slice(0, limit) as incident (incident.id)}
<article class="incident">
<div class="row-heading"><a class="incident-link" href={`/watchdog/${encodeURIComponent(incident.id)}`}><ServerIdentity name={incident.server} /><span class="sr-only">Watchdog incident {incident.id}</span></a><WatchdogStatus state={incident.state} /></div>
<p>{incident.summary}</p>
<div class="meta"><Badge variant="neutral">{incident.kind === 'drift' ? 'File drift' : 'Server health'}</Badge><span>Detected {$timeFormat.dateTime(incident.first_seen_at)}</span><span>Last seen {$timeFormat.dateTime(incident.last_seen_at)}</span></div>
<div class="links"><a href={`/watchdog/${encodeURIComponent(incident.id)}`}>View recovery and activity</a><a href={`/servers/${encodeURIComponent(incident.server)}?tab=diagnostics`}>Diagnose server</a></div>
</article>
{/each}
</div>
<ListFooter total={matching.length} bind:limit onchange={() => saveContext()} label="incidents" />
{/if}
</section>
{:else if !loadError}<p class="muted" role="status">Loading Watchdog…</p>{/if}
</div>
<style>
.page, section, .incidents { display: grid; gap: 20px; }
.page { gap: 28px; }
.automation, .incident { border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); padding: 24px; }
.automation { gap: 12px; }
.automation-heading, .row-heading, .meta, .links, .filters { display: flex; gap: 12px 20px; flex-wrap: wrap; align-items: center; }
.row-heading { justify-content: space-between; }
.incident { display: grid; gap: 14px; }
.incident-link { color: var(--text-primary); text-decoration: none; font-weight: var(--weight-semibold); }
.incident-link:hover { color: var(--accent-primary-hover); text-decoration: underline; }
h2 { margin: 0; font-size: var(--text-md); }
h2 .muted { margin-left: 8px; }
p { margin: 0; line-height: 1.6; overflow-wrap: anywhere; }
.automation p { max-width: 960px; font-size: var(--text-sm); color: var(--text-secondary); }
.automation strong { color: var(--text-primary); font-weight: var(--weight-medium); }
.meta, .muted { color: var(--text-secondary); font-size: var(--text-xs); }
.links { font-size: var(--text-sm); }
a { color: var(--accent-primary-hover); }
.filters { align-items: flex-end; }
label { display: grid; gap: 8px; font-size: var(--text-xs); color: var(--text-secondary); }
.server-filter { flex: 1; min-width: 200px; }
input, select { font: inherit; font-size: var(--text-sm); min-height: 40px; color: var(--text-primary); background: var(--surface-sunken); border: 1px solid var(--border-control); border-radius: var(--radius-sm); padding: 9px 12px; }
.notice { color: var(--text-secondary); font-size: var(--text-sm); }
@media (max-width: 640px) { .automation, .incident { padding: 18px; } label { width: 100%; } }
</style>
src/routes/watchdog/[id]/+page.svelte
+205
View File
@@ -0,0 +1,205 @@
<script lang="ts">
import { onMount, untrack } from 'svelte';
import { page } from '$app/stores';
import type { WatchdogDetail, WatchdogEvidence } from '$lib/watchdog.js';
import { pollVisible } from '$ui/poll.js';
import { timeFormat } from '$ui/stores/time-zone.js';
import { serverTitle } from '$ui/server-labels.js';
import WatchdogStatus from '$ui/components/WatchdogStatus.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
import Button from '$ui/components/primitives/Button.svelte';
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
let detail = $state<WatchdogDetail | null>(null);
let loadError = $state('');
let actionError = $state('');
let notice = $state('');
let busy = $state<'retry' | 'dismiss' | null>(null);
let eventLimit = $state(25);
let fileLimit = $state(25);
let request = 0;
const incident = $derived(detail?.incident);
const evidence = $derived.by((): WatchdogEvidence => {
try {
const value = JSON.parse(incident?.details ?? '{}');
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
} catch { return {}; }
});
const files = $derived(evidence.files ?? []);
const actions = $derived(detail?.actions.slice().reverse() ?? []);
const events = $derived(detail?.events.slice().reverse() ?? []);
const activeOperation = $derived(actions.some(action => ['queued', 'waiting', 'running', 'awaiting-health'].includes(action.operation_state ?? '')));
const canRetry = $derived(incident?.state === 'attention' || incident?.state === 'dismissed');
const canDismiss = $derived(incident?.state === 'open' || incident?.state === 'attention');
const explanation = $derived.by(() => {
if (!incident) return '';
if (incident.state === 'resolved') return incident.kind === 'drift' ? 'The observed file drift has cleared.' : 'The server passed the checks for this incident.';
if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.';
if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.';
if (incident.state === 'recovering') return 'Recovery is in progress. Follow the provisioning operations below for eligibility checks, progress, and health verification.';
return 'Watchdog detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server.';
});
async function load() {
if (busy) return;
const id = $page.params.id ?? '';
const current = ++request;
try {
const response = await fetch(`/api/watchdog/${encodeURIComponent(id)}`);
if (!response.ok) throw new Error(response.status === 404 ? 'Watchdog incident not found.' : 'Could not refresh this incident.');
const body = await response.json() as WatchdogDetail;
if (current !== request || id !== $page.params.id) return;
detail = body; loadError = '';
} catch (error) {
if (current === request && id === $page.params.id) loadError = (error as Error).message;
}
}
async function update(action: 'retry' | 'dismiss') {
if (busy || !incident) return;
const id = incident.id;
const current = ++request;
busy = action; actionError = ''; notice = '';
try {
const response = await fetch(`/api/watchdog/${encodeURIComponent(id)}`, {
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action })
});
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not update this incident.');
if (current !== request || id !== $page.params.id) return;
detail = body; loadError = '';
notice = action === 'retry' ? 'Recovery retry requested. Watchdog will recheck the server before taking action.' : 'Incident dismissed.';
} catch (error) {
if (current === request && id === $page.params.id) actionError = (error as Error).message;
} finally { if (current === request) busy = null; }
}
$effect(() => {
$page.params.id;
request++; detail = null; loadError = ''; actionError = ''; notice = ''; busy = null;
eventLimit = 25; fileLimit = 25;
untrack(() => { void load(); });
});
onMount(() => {
const stop = pollVisible(load, 5000, false);
return () => { stop(); request++; };
});
</script>
<svelte:head><title>{incident ? `${serverTitle(incident.server)} · Watchdog` : 'Watchdog incident'} | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title={incident ? `${incident.kind === 'drift' ? 'File drift' : 'Server health'} · ${serverTitle(incident.server)}` : 'Watchdog incident'} subtitle={incident?.id}>
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Watchdog', href: '/watchdog' }, { label: 'Incident' }]} />{/snippet}
{#snippet actions()}
{#if canRetry}<Button disabled={!!busy || activeOperation} loading={busy === 'retry'} onclick={() => update('retry')}>Retry recovery</Button>{/if}
{#if canDismiss}<Button variant="ghost" disabled={!!busy || activeOperation} loading={busy === 'dismiss'} onclick={() => update('dismiss')}>Dismiss incident</Button>{/if}
{/snippet}
</SectionHeader>
{#if loadError}<div class="ui-error" role="alert">{loadError} {detail ? 'Showing the last successful refresh.' : ''}<Button size="sm" disabled={!!busy} onclick={load}>Retry refresh</Button></div>{/if}
{#if actionError}<p class="ui-error" role="alert">{actionError}</p>{/if}
{#if notice}<p class="notice" role="status">{notice}</p>{/if}
{#if incident && detail}
<section class="summary" aria-label="Incident status">
<div class="row-heading"><WatchdogStatus state={incident.state} /><a href={`/servers/${encodeURIComponent(incident.server)}`}>View server</a></div>
<h2>{incident.summary}</h2>
<p>{explanation}</p>
{#if activeOperation && (canRetry || canDismiss)}<p>Complete or cancel the active provisioning operation before retrying or dismissing this incident.</p>{/if}
{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}
<dl>
<div><dt>First detected</dt><dd>{$timeFormat.dateTime(incident.first_seen_at)}</dd></div>
<div><dt>Last observed</dt><dd>{$timeFormat.dateTime(incident.last_seen_at)}</dd></div>
{#if incident.finished_at}<div><dt>Finished</dt><dd>{$timeFormat.dateTime(incident.finished_at)}</dd></div>{/if}
{#if evidence.containerState}<div><dt>Observed Komodo state</dt><dd>{evidence.containerState}</dd></div>{/if}
</dl>
{#if evidence.reason && evidence.reason !== incident.summary}<p class="observed-reason">Observed reason: {evidence.reason}</p>{/if}
<div class="links"><a href={`/servers/${encodeURIComponent(incident.server)}?tab=diagnostics`}>Diagnose connection</a><a href={`/servers/${encodeURIComponent(incident.server)}?tab=logs`}>Open server logs</a></div>
</section>
<section aria-labelledby="recovery-title">
<h2 id="recovery-title">Recovery operations</h2>
{#if !actions.length}<EmptyState title="No recovery operations yet" description="Provisioning operations and drift campaigns will appear here when recovery starts." />
{:else}<div class="operations">
{#each actions as action (action.id)}
<article class="operation">
<div class="row-heading"><h3>{action.action === 'restart' ? 'Restart server' : 'Rebuild server'}</h3>{#if action.operation_state}<StatusBadge state={action.operation_state} />{/if}</div>
{#if action.operation_stage}<p>{action.operation_stage}</p>{/if}
{#if action.operation_error}<p class="error">{action.operation_error}</p>{/if}
<time datetime={action.created_at}>{$timeFormat.dateTime(action.created_at)}</time>
<div class="links"><a href={`/provisioning/${encodeURIComponent(action.operation_id)}`}>View provisioning</a>{#if action.campaign_id}<a href={`/campaigns/${encodeURIComponent(action.campaign_id)}`}>View repair campaign</a>{/if}</div>
</article>
{/each}
</div>{/if}
</section>
{#if incident.kind === 'drift'}
<section aria-labelledby="files-title">
<h2 id="files-title">Observed file drift <span class="muted">{files.length}</span></h2>
<p class="muted">These differences describe the observed problem. Recovery rebuilds the whole server instance.</p>
{#if !files.length}<EmptyState title="No file differences recorded" />
{:else}<div class="files">
{#each files.slice(0, fileLimit) as file (`${file.kind}:${file.driftKind}:${file.relPath}`)}
<article class="file">
<div class="file-heading"><Badge variant={file.driftKind === 'untracked' ? 'danger' : 'warning'}>{file.driftKind === 'replace' ? 'Modified' : file.driftKind === 'missing' ? 'Missing' : 'Untracked'}</Badge><code>{file.relPath}</code><span class="muted">{file.kind}</span></div>
<details><summary>Observed and expected file</summary><dl>
<div><dt>Observed</dt><dd>{file.beforeSize === null ? 'Absent' : `${file.beforeSize} bytes`}{#if file.beforeSha256}<code>{file.beforeSha256}</code>{/if}</dd></div>
<div><dt>Expected</dt><dd>{file.afterSize === null ? 'Absent' : `${file.afterSize} bytes`}{#if file.afterSha256}<code>{file.afterSha256}</code>{/if}</dd></div>
</dl></details>
</article>
{/each}
</div><ListFooter total={files.length} bind:limit={fileLimit} label="files" />{/if}
</section>
{/if}
<section aria-labelledby="activity-title">
<h2 id="activity-title">Activity history <span class="muted">{events.length}</span></h2>
<p class="muted">Newest first · updates automatically</p>
{#if !events.length}<EmptyState title="No activity recorded yet" />
{:else}<ol class="timeline">
{#each events.slice(0, eventLimit) as event (event.id)}
<li><time datetime={event.created_at}>{$timeFormat.dateTime(event.created_at)}</time><p>{event.message}</p></li>
{/each}
</ol><ListFooter total={events.length} bind:limit={eventLimit} label="events" />{/if}
</section>
{:else if !loadError}<p class="muted" role="status">Loading incident…</p>{/if}
</div>
<style>
.page, section, .operations, .files { display: grid; gap: 20px; }
.page { gap: 28px; }
.summary, .operation, .file { padding: 24px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); }
.operation, .file { display: grid; gap: 12px; }
.summary { gap: 16px; }
.row-heading, .links, .file-heading { display: flex; gap: 12px 20px; align-items: center; flex-wrap: wrap; }
.row-heading { justify-content: space-between; }
h2, h3, p { margin: 0; }
h2 { font-size: var(--text-md); }
h3 { font-size: var(--text-sm); }
p { line-height: 1.6; overflow-wrap: anywhere; }
h2 .muted { margin-left: 8px; }
.summary p, .operation p { font-size: var(--text-sm); color: var(--text-secondary); }
.summary strong { color: var(--text-primary); font-weight: var(--weight-medium); }
.links, .row-heading a { font-size: var(--text-sm); }
a { color: var(--accent-primary-hover); }
dl { display: flex; gap: 16px 32px; flex-wrap: wrap; margin: 0; }
dt, .muted, time, summary, .notice { color: var(--text-secondary); font-size: var(--text-xs); }
dd { margin: 6px 0 0; font-size: var(--text-sm); overflow-wrap: anywhere; }
code { font-family: var(--font-mono); font-size: var(--text-xs); overflow-wrap: anywhere; }
dd code { display: block; margin-top: 8px; color: var(--text-secondary); }
details { border-top: 1px solid var(--border-subtle); padding-top: 12px; }
summary { cursor: pointer; }
details[open] summary { margin-bottom: 16px; }
.operation .error { color: var(--state-danger); }
.timeline { list-style: none; padding: 0 0 0 18px; margin: 0 0 0 6px; border-left: 2px solid var(--border-strong); }
.timeline li { position: relative; padding: 0 0 24px 12px; }
.timeline li:last-child { padding-bottom: 0; }
.timeline li::before { content: ''; position: absolute; left: -24px; top: 5px; width: 10px; height: 10px; border-radius: 50%; background: var(--accent-primary); }
.timeline p { margin-top: 6px; font-size: var(--text-sm); }
@media (max-width: 640px) { .summary, .operation, .file { padding: 18px; } }
</style>