josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve deleted server history

Browse Source 
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.
This commit is contained in:
josh
2026-09-27 21:48:55 -04:00
parent 53b29f726c
commit 981fb27c1c
61 changed files with 929 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/routes/maintenance/+page.svelte
+10 -9
View File
@@ -9,6 +9,7 @@
import { timeFormat, timeZone } from '$ui/stores/time-zone.js';
import { serverTitle } from '$ui/server-labels.js';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Tabs from '$ui/components/primitives/Tabs.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
@@ -40,16 +41,16 @@
const versions = $derived(data?.runtime.servers.filter(v => !server || `${v.server} ${serverTitle(v.server)}`.toLowerCase().includes(server.toLowerCase())) ?? []);
const outdated = $derived(versions.filter(v => v.update_status === 'outdated'));
const unknown = $derived(versions.filter(v => v.update_status === 'unknown'));
const activeJobs = $derived(data?.runtime.jobs.filter(j => ['queued', 'restarting'].includes(j.state)) ?? []);
const activeJobs = $derived(data?.runtime.jobs.filter(j => !j.server_deleted_at && ['queued', 'restarting'].includes(j.state)) ?? []);
const activities = $derived([
...(data?.recovery.incidents ?? []).map(i => ({
key: i.id, kind: i.kind, server: i.server, at: i.first_seen_at,
key: i.id, kind: i.kind, server: i.server, server_deleted_at: i.server_deleted_at, at: i.first_seen_at,
title: i.kind === 'health' ? 'Health recovery' : 'Configuration drift',
state: ['open', 'recovering'].includes(i.state) ? 'active' : i.state === 'attention' ? 'attention' : 'done',
label: watchdogStateLabels[i.state], detail: i.state === 'recovering' ? i.operation_stage ?? i.summary : i.summary, href: `/maintenance/${encodeURIComponent(i.id)}`, job: null
})),
...(data?.runtime.jobs ?? []).map(j => ({
key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, at: j.requested_at,
key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, server_deleted_at: j.server_deleted_at, at: j.requested_at,
title: j.kind === 'schedule' ? 'Scheduled restart' : j.kind === 'update' ? 'Plutonium update' : 'Legacy maintenance',
state: ['queued', 'restarting'].includes(j.state) ? 'active' : j.state === 'failed' ? 'attention' : 'done',
label: j.state === 'failed' ? 'Needs attention' : j.state === 'done' ? 'Completed' : j.state === 'cancelled' ? 'Cancelled' : j.state === 'restarting' ? 'Recovering' : 'Queued',
@@ -58,12 +59,12 @@
}))
].sort((a, b) => b.at.localeCompare(a.at)));
const isLatest = (item: (typeof activities)[number]) => !item.job || !data?.runtime.jobs.some(j => j.server === item.server && j.kind === item.job!.kind && j.id > item.job!.id);
const attention = $derived(activities.filter(a => a.state === 'attention' && isLatest(a)).length);
const waiting = $derived(activities.filter(a => a.state === 'active').length);
const attention = $derived(activities.filter(a => !a.server_deleted_at && a.state === 'attention' && isLatest(a)).length);
const waiting = $derived(activities.filter(a => !a.server_deleted_at && a.state === 'active').length);
const matching = $derived(activities.filter(a =>
(kind === 'all' || a.kind === kind) &&
(!server || `${a.server} ${serverTitle(a.server)}`.toLowerCase().includes(server.toLowerCase())) &&
(view === 'overview' ? a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status)
(view === 'overview' ? !a.server_deleted_at && a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status)
));
$effect(() => {
const params = $page.url.searchParams;
@@ -186,12 +187,12 @@
<h2>{view === 'overview' ? 'Active work & attention' : 'Maintenance activity'}</h2>
<div class="activity">{#each matching.slice(0, limit) as item (item.key)}
<article class="card" id={item.key}>
<div class="row-heading"><h3>{item.title} · <ServerIdentity name={item.server} /></h3><Badge variant={item.state === 'attention' ? 'danger' : item.state === 'active' ? 'info' : 'neutral'}>{item.label}</Badge></div>
<div class="row-heading"><h3>{item.title} · <HistoricalServerIdentity name={item.server} deletedAt={item.server_deleted_at} /></h3><Badge variant={item.state === 'attention' ? 'danger' : item.state === 'active' ? 'info' : 'neutral'}>{item.label}</Badge></div>
{#if item.detail}<p>{item.detail}</p>{/if}
<p class="muted">{$timeFormat.dateTime(item.at)}{item.job?.target_revision ? ` · Expected r${item.job.target_revision} or newer` : ''}</p>
<div class="links">{#if item.href}<a href={item.href}>{item.job ? 'View restart operation' : 'View evidence and recovery'}</a>{/if}
{#if item.job?.state === 'queued'}<Button size="sm" disabled={busy} onclick={() => act({ action: 'cancel', id: item.job!.id })}>Cancel restart</Button>{/if}
{#if item.job?.state === 'failed' && item.job.operation_state === 'done'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'retry', id: item.job!.id })}>Retry after inspection</Button>{/if}
{#if item.job?.state === 'queued' && !item.server_deleted_at}<Button size="sm" disabled={busy} onclick={() => act({ action: 'cancel', id: item.job!.id })}>Cancel restart</Button>{/if}
{#if item.job?.state === 'failed' && !item.server_deleted_at && item.job.operation_state === 'done'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'retry', id: item.job!.id })}>Retry after inspection</Button>{/if}
</div>
</article>
{:else}<EmptyState title={view === 'overview' ? 'No active maintenance in this view' : 'No matching activity'} description="Findings, scheduled restarts, and update verification appear here." />{/each}</div>
src/routes/maintenance/[id]/+page.svelte
+6 -5
View File
@@ -33,10 +33,11 @@
const actions = $derived(detail?.actions.slice().reverse() ?? []);
const events = $derived(detail?.events.slice().reverse() ?? []);
const activeOperation = $derived(actions.some(action => ['queued', 'waiting', 'running', 'awaiting-health'].includes(action.operation_state ?? '')));
const canRetry = $derived(incident?.state === 'attention' || incident?.state === 'dismissed');
const canDismiss = $derived(incident?.state === 'open' || incident?.state === 'attention');
const canRetry = $derived(!incident?.server_deleted_at && (incident?.state === 'attention' || incident?.state === 'dismissed'));
const canDismiss = $derived(!incident?.server_deleted_at && (incident?.state === 'open' || incident?.state === 'attention'));
const explanation = $derived.by(() => {
if (!incident) return '';
if (incident.server_deleted_at) return 'This server was deleted. Its recorded finding, evidence, and recovery results are preserved below.';
if (incident.state === 'resolved') return incident.kind === 'drift' ? 'The observed file drift has cleared.' : 'The server passed the checks for this incident.';
if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.';
if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.';
@@ -107,11 +108,11 @@
{#if incident && detail}
<section class="summary" aria-label="Incident status">
<div class="row-heading"><WatchdogStatus state={incident.state} /><a href={`/servers/${encodeURIComponent(incident.server)}`}>View server</a></div>
<div class="row-heading"><WatchdogStatus state={incident.state} />{#if incident.server_deleted_at}<Badge>Deleted</Badge>{:else}<a href={`/servers/${encodeURIComponent(incident.server)}`}>View server</a>{/if}</div>
<h2>{incident.summary}</h2>
<p>{explanation}</p>
{#if activeOperation && (canRetry || canDismiss)}<p>Complete or cancel the active provisioning operation before retrying or dismissing this incident.</p>{/if}
{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}
{#if !incident.server_deleted_at}{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}{/if}
<dl>
<div><dt>First detected</dt><dd>{$timeFormat.dateTime(incident.first_seen_at)}</dd></div>
<div><dt>Last observed</dt><dd>{$timeFormat.dateTime(incident.last_seen_at)}</dd></div>
@@ -119,7 +120,7 @@
{#if evidence.containerState}<div><dt>Observed Komodo state</dt><dd>{evidence.containerState}</dd></div>{/if}
</dl>
{#if evidence.reason && evidence.reason !== incident.summary}<p class="observed-reason">Observed reason: {evidence.reason}</p>{/if}
<div class="links"><a href={`/servers/${encodeURIComponent(incident.server)}?tab=diagnostics`}>Diagnose connection</a><a href={`/servers/${encodeURIComponent(incident.server)}?tab=logs`}>Open server logs</a></div>
{#if !incident.server_deleted_at}<div class="links"><a href={`/servers/${encodeURIComponent(incident.server)}?tab=diagnostics`}>Diagnose connection</a><a href={`/servers/${encodeURIComponent(incident.server)}?tab=logs`}>Open server logs</a></div>{/if}
</section>
<section aria-labelledby="recovery-title">