feat: unify recovery, updates, and schedules under Maintenance
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
import { redirect } from '@sveltejs/kit';
|
||||
export function load({ params, url }: { params: { id: string }; url: URL }) { redirect(308, `/maintenance/${encodeURIComponent(params.id)}${url.search}`); }
|
||||
@@ -1,205 +0,0 @@
|
||||
<script lang="ts">
|
||||
import { onMount, untrack } from 'svelte';
|
||||
import { page } from '$app/stores';
|
||||
import type { WatchdogDetail, WatchdogEvidence } from '$lib/watchdog.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { timeFormat } from '$ui/stores/time-zone.js';
|
||||
import { serverTitle } from '$ui/server-labels.js';
|
||||
import WatchdogStatus from '$ui/components/WatchdogStatus.svelte';
|
||||
import Badge from '$ui/components/primitives/Badge.svelte';
|
||||
import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
|
||||
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
|
||||
|
||||
let detail = $state<WatchdogDetail | null>(null);
|
||||
let loadError = $state('');
|
||||
let actionError = $state('');
|
||||
let notice = $state('');
|
||||
let busy = $state<'retry' | 'dismiss' | null>(null);
|
||||
let eventLimit = $state(25);
|
||||
let fileLimit = $state(25);
|
||||
let request = 0;
|
||||
const incident = $derived(detail?.incident);
|
||||
const evidence = $derived.by((): WatchdogEvidence => {
|
||||
try {
|
||||
const value = JSON.parse(incident?.details ?? '{}');
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
||||
} catch { return {}; }
|
||||
});
|
||||
const files = $derived(evidence.files ?? []);
|
||||
const actions = $derived(detail?.actions.slice().reverse() ?? []);
|
||||
const events = $derived(detail?.events.slice().reverse() ?? []);
|
||||
const activeOperation = $derived(actions.some(action => ['queued', 'waiting', 'running', 'awaiting-health'].includes(action.operation_state ?? '')));
|
||||
const canRetry = $derived(incident?.state === 'attention' || incident?.state === 'dismissed');
|
||||
const canDismiss = $derived(incident?.state === 'open' || incident?.state === 'attention');
|
||||
const explanation = $derived.by(() => {
|
||||
if (!incident) return '';
|
||||
if (incident.state === 'resolved') return incident.kind === 'drift' ? 'The observed file drift has cleared.' : 'The server passed the checks for this incident.';
|
||||
if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.';
|
||||
if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.';
|
||||
if (incident.state === 'recovering') return 'Recovery is in progress. Follow the provisioning operations below for eligibility checks, progress, and health verification.';
|
||||
return 'Watchdog detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server.';
|
||||
});
|
||||
|
||||
async function load() {
|
||||
if (busy) return;
|
||||
const id = $page.params.id ?? '';
|
||||
const current = ++request;
|
||||
try {
|
||||
const response = await fetch(`/api/watchdog/${encodeURIComponent(id)}`);
|
||||
if (!response.ok) throw new Error(response.status === 404 ? 'Watchdog incident not found.' : 'Could not refresh this incident.');
|
||||
const body = await response.json() as WatchdogDetail;
|
||||
if (current !== request || id !== $page.params.id) return;
|
||||
detail = body; loadError = '';
|
||||
} catch (error) {
|
||||
if (current === request && id === $page.params.id) loadError = (error as Error).message;
|
||||
}
|
||||
}
|
||||
|
||||
async function update(action: 'retry' | 'dismiss') {
|
||||
if (busy || !incident) return;
|
||||
const id = incident.id;
|
||||
const current = ++request;
|
||||
busy = action; actionError = ''; notice = '';
|
||||
try {
|
||||
const response = await fetch(`/api/watchdog/${encodeURIComponent(id)}`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action })
|
||||
});
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? 'Could not update this incident.');
|
||||
if (current !== request || id !== $page.params.id) return;
|
||||
detail = body; loadError = '';
|
||||
notice = action === 'retry' ? 'Recovery retry requested. Watchdog will recheck the server before taking action.' : 'Incident dismissed.';
|
||||
} catch (error) {
|
||||
if (current === request && id === $page.params.id) actionError = (error as Error).message;
|
||||
} finally { if (current === request) busy = null; }
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
$page.params.id;
|
||||
request++; detail = null; loadError = ''; actionError = ''; notice = ''; busy = null;
|
||||
eventLimit = 25; fileLimit = 25;
|
||||
untrack(() => { void load(); });
|
||||
});
|
||||
onMount(() => {
|
||||
const stop = pollVisible(load, 5000, false);
|
||||
return () => { stop(); request++; };
|
||||
});
|
||||
</script>
|
||||
|
||||
<svelte:head><title>{incident ? `${serverTitle(incident.server)} · Watchdog` : 'Watchdog incident'} | Electric Cherry</title></svelte:head>
|
||||
|
||||
<div class="page">
|
||||
<SectionHeader title={incident ? `${incident.kind === 'drift' ? 'File drift' : 'Server health'} · ${serverTitle(incident.server)}` : 'Watchdog incident'} subtitle={incident?.id}>
|
||||
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Watchdog', href: '/watchdog' }, { label: 'Incident' }]} />{/snippet}
|
||||
{#snippet actions()}
|
||||
{#if canRetry}<Button disabled={!!busy || activeOperation} loading={busy === 'retry'} onclick={() => update('retry')}>Retry recovery</Button>{/if}
|
||||
{#if canDismiss}<Button variant="ghost" disabled={!!busy || activeOperation} loading={busy === 'dismiss'} onclick={() => update('dismiss')}>Dismiss incident</Button>{/if}
|
||||
{/snippet}
|
||||
</SectionHeader>
|
||||
|
||||
{#if loadError}<div class="ui-error" role="alert">{loadError} {detail ? 'Showing the last successful refresh.' : ''}<Button size="sm" disabled={!!busy} onclick={load}>Retry refresh</Button></div>{/if}
|
||||
{#if actionError}<p class="ui-error" role="alert">{actionError}</p>{/if}
|
||||
{#if notice}<p class="notice" role="status">{notice}</p>{/if}
|
||||
|
||||
{#if incident && detail}
|
||||
<section class="summary" aria-label="Incident status">
|
||||
<div class="row-heading"><WatchdogStatus state={incident.state} /><a href={`/servers/${encodeURIComponent(incident.server)}`}>View server</a></div>
|
||||
<h2>{incident.summary}</h2>
|
||||
<p>{explanation}</p>
|
||||
{#if activeOperation && (canRetry || canDismiss)}<p>Complete or cancel the active provisioning operation before retrying or dismissing this incident.</p>{/if}
|
||||
{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}
|
||||
<dl>
|
||||
<div><dt>First detected</dt><dd>{$timeFormat.dateTime(incident.first_seen_at)}</dd></div>
|
||||
<div><dt>Last observed</dt><dd>{$timeFormat.dateTime(incident.last_seen_at)}</dd></div>
|
||||
{#if incident.finished_at}<div><dt>Finished</dt><dd>{$timeFormat.dateTime(incident.finished_at)}</dd></div>{/if}
|
||||
{#if evidence.containerState}<div><dt>Observed Komodo state</dt><dd>{evidence.containerState}</dd></div>{/if}
|
||||
</dl>
|
||||
{#if evidence.reason && evidence.reason !== incident.summary}<p class="observed-reason">Observed reason: {evidence.reason}</p>{/if}
|
||||
<div class="links"><a href={`/servers/${encodeURIComponent(incident.server)}?tab=diagnostics`}>Diagnose connection</a><a href={`/servers/${encodeURIComponent(incident.server)}?tab=logs`}>Open server logs</a></div>
|
||||
</section>
|
||||
|
||||
<section aria-labelledby="recovery-title">
|
||||
<h2 id="recovery-title">Recovery operations</h2>
|
||||
{#if !actions.length}<EmptyState title="No recovery operations yet" description="Provisioning operations and drift campaigns will appear here when recovery starts." />
|
||||
{:else}<div class="operations">
|
||||
{#each actions as action (action.id)}
|
||||
<article class="operation">
|
||||
<div class="row-heading"><h3>{action.action === 'restart' ? 'Restart server' : 'Rebuild server'}</h3>{#if action.operation_state}<StatusBadge state={action.operation_state} />{/if}</div>
|
||||
{#if action.operation_stage}<p>{action.operation_stage}</p>{/if}
|
||||
{#if action.operation_error}<p class="error">{action.operation_error}</p>{/if}
|
||||
<time datetime={action.created_at}>{$timeFormat.dateTime(action.created_at)}</time>
|
||||
<div class="links"><a href={`/provisioning/${encodeURIComponent(action.operation_id)}`}>View provisioning</a>{#if action.campaign_id}<a href={`/campaigns/${encodeURIComponent(action.campaign_id)}`}>View repair campaign</a>{/if}</div>
|
||||
</article>
|
||||
{/each}
|
||||
</div>{/if}
|
||||
</section>
|
||||
|
||||
{#if incident.kind === 'drift'}
|
||||
<section aria-labelledby="files-title">
|
||||
<h2 id="files-title">Observed file drift <span class="muted">{files.length}</span></h2>
|
||||
<p class="muted">These differences describe the observed problem. Recovery rebuilds the whole server instance.</p>
|
||||
{#if !files.length}<EmptyState title="No file differences recorded" />
|
||||
{:else}<div class="files">
|
||||
{#each files.slice(0, fileLimit) as file (`${file.kind}:${file.driftKind}:${file.relPath}`)}
|
||||
<article class="file">
|
||||
<div class="file-heading"><Badge variant={file.driftKind === 'untracked' ? 'danger' : 'warning'}>{file.driftKind === 'replace' ? 'Modified' : file.driftKind === 'missing' ? 'Missing' : 'Untracked'}</Badge><code>{file.relPath}</code><span class="muted">{file.kind}</span></div>
|
||||
<details><summary>Observed and expected file</summary><dl>
|
||||
<div><dt>Observed</dt><dd>{file.beforeSize === null ? 'Absent' : `${file.beforeSize} bytes`}{#if file.beforeSha256}<code>{file.beforeSha256}</code>{/if}</dd></div>
|
||||
<div><dt>Expected</dt><dd>{file.afterSize === null ? 'Absent' : `${file.afterSize} bytes`}{#if file.afterSha256}<code>{file.afterSha256}</code>{/if}</dd></div>
|
||||
</dl></details>
|
||||
</article>
|
||||
{/each}
|
||||
</div><ListFooter total={files.length} bind:limit={fileLimit} label="files" />{/if}
|
||||
</section>
|
||||
{/if}
|
||||
|
||||
<section aria-labelledby="activity-title">
|
||||
<h2 id="activity-title">Activity history <span class="muted">{events.length}</span></h2>
|
||||
<p class="muted">Newest first · updates automatically</p>
|
||||
{#if !events.length}<EmptyState title="No activity recorded yet" />
|
||||
{:else}<ol class="timeline">
|
||||
{#each events.slice(0, eventLimit) as event (event.id)}
|
||||
<li><time datetime={event.created_at}>{$timeFormat.dateTime(event.created_at)}</time><p>{event.message}</p></li>
|
||||
{/each}
|
||||
</ol><ListFooter total={events.length} bind:limit={eventLimit} label="events" />{/if}
|
||||
</section>
|
||||
{:else if !loadError}<p class="muted" role="status">Loading incident…</p>{/if}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.page, section, .operations, .files { display: grid; gap: 20px; }
|
||||
.page { gap: 28px; }
|
||||
.summary, .operation, .file { padding: 24px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); }
|
||||
.operation, .file { display: grid; gap: 12px; }
|
||||
.summary { gap: 16px; }
|
||||
.row-heading, .links, .file-heading { display: flex; gap: 12px 20px; align-items: center; flex-wrap: wrap; }
|
||||
.row-heading { justify-content: space-between; }
|
||||
h2, h3, p { margin: 0; }
|
||||
h2 { font-size: var(--text-md); }
|
||||
h3 { font-size: var(--text-sm); }
|
||||
p { line-height: 1.6; overflow-wrap: anywhere; }
|
||||
h2 .muted { margin-left: 8px; }
|
||||
.summary p, .operation p { font-size: var(--text-sm); color: var(--text-secondary); }
|
||||
.summary strong { color: var(--text-primary); font-weight: var(--weight-medium); }
|
||||
.links, .row-heading a { font-size: var(--text-sm); }
|
||||
a { color: var(--accent-primary-hover); }
|
||||
dl { display: flex; gap: 16px 32px; flex-wrap: wrap; margin: 0; }
|
||||
dt, .muted, time, summary, .notice { color: var(--text-secondary); font-size: var(--text-xs); }
|
||||
dd { margin: 6px 0 0; font-size: var(--text-sm); overflow-wrap: anywhere; }
|
||||
code { font-family: var(--font-mono); font-size: var(--text-xs); overflow-wrap: anywhere; }
|
||||
dd code { display: block; margin-top: 8px; color: var(--text-secondary); }
|
||||
details { border-top: 1px solid var(--border-subtle); padding-top: 12px; }
|
||||
summary { cursor: pointer; }
|
||||
details[open] summary { margin-bottom: 16px; }
|
||||
.operation .error { color: var(--state-danger); }
|
||||
.timeline { list-style: none; padding: 0 0 0 18px; margin: 0 0 0 6px; border-left: 2px solid var(--border-strong); }
|
||||
.timeline li { position: relative; padding: 0 0 24px 12px; }
|
||||
.timeline li:last-child { padding-bottom: 0; }
|
||||
.timeline li::before { content: ''; position: absolute; left: -24px; top: 5px; width: 10px; height: 10px; border-radius: 50%; background: var(--accent-primary); }
|
||||
.timeline p { margin-top: 6px; font-size: var(--text-sm); }
|
||||
@media (max-width: 640px) { .summary, .operation, .file { padding: 18px; } }
|
||||
</style>
|
||||
Reference in New Issue
Block a user