feat: unify recovery, updates, and schedules under Maintenance
This commit is contained in:
@@ -0,0 +1,221 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { page } from '$app/stores';
|
||||
import { goto } from '$app/navigation';
|
||||
import type { MaintenanceOverview, MaintenancePolicy } from '$lib/maintenance.js';
|
||||
import { maintenanceTimeInZone, PLUTONIUM_MANIFEST_URL } from '$lib/plutonium.js';
|
||||
import { watchdogStateLabels } from '$lib/watchdog.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { timeFormat, timeZone } from '$ui/stores/time-zone.js';
|
||||
import { serverTitle } from '$ui/server-labels.js';
|
||||
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import Tabs from '$ui/components/primitives/Tabs.svelte';
|
||||
import Badge from '$ui/components/primitives/Badge.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
|
||||
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
|
||||
|
||||
let data = $state<MaintenanceOverview | null>(null);
|
||||
let error = $state('');
|
||||
let notice = $state('');
|
||||
let busy = $state(false);
|
||||
let refreshing = false;
|
||||
let view = $state('overview');
|
||||
let kind = $state('all');
|
||||
let server = $state('');
|
||||
let status = $state('all');
|
||||
let limit = $state(25);
|
||||
let scheduleEnabled = $state(false);
|
||||
let scheduleTime = $state('06:00');
|
||||
let scheduleLoaded = false;
|
||||
let displayedZone = $state('');
|
||||
let request = 0;
|
||||
const tabs = [{ id: 'overview', label: 'Overview' }, { id: 'policies', label: 'Policies & schedules' }, { id: 'activity', label: 'Activity' }];
|
||||
const policies: { id: MaintenancePolicy; title: string; description: string }[] = [
|
||||
{ id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Rebuilding deletes instance files and installs the latest promoted setup.' },
|
||||
{ id: 'drift', title: 'Configuration drift', description: 'Rebuild servers whose managed files differ from the promoted setup. Record the differences in a linked Campaign. Rebuilding deletes instance files.' },
|
||||
{ id: 'updates', title: 'Automatic Plutonium updates', description: 'Update outdated running servers on restart. One automatic attempt per published revision; failed verification requires inspection. Disabled by default.' }
|
||||
];
|
||||
const versions = $derived(data?.runtime.servers.filter(v => !server || `${v.server} ${serverTitle(v.server)}`.toLowerCase().includes(server.toLowerCase())) ?? []);
|
||||
const outdated = $derived(versions.filter(v => v.update_status === 'outdated'));
|
||||
const unknown = $derived(versions.filter(v => v.update_status === 'unknown'));
|
||||
const activeJobs = $derived(data?.runtime.jobs.filter(j => ['queued', 'restarting'].includes(j.state)) ?? []);
|
||||
const activities = $derived([
|
||||
...(data?.recovery.incidents ?? []).map(i => ({
|
||||
key: i.id, kind: i.kind, server: i.server, at: i.first_seen_at,
|
||||
title: i.kind === 'health' ? 'Health recovery' : 'Configuration drift',
|
||||
state: ['open', 'recovering'].includes(i.state) ? 'active' : i.state === 'attention' ? 'attention' : 'done',
|
||||
label: watchdogStateLabels[i.state], detail: i.state === 'recovering' ? i.operation_stage ?? i.summary : i.summary, href: `/maintenance/${encodeURIComponent(i.id)}`, job: null
|
||||
})),
|
||||
...(data?.runtime.jobs ?? []).map(j => ({
|
||||
key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, at: j.requested_at,
|
||||
title: j.kind === 'schedule' ? 'Scheduled restart' : j.kind === 'update' ? 'Plutonium update' : 'Legacy maintenance',
|
||||
state: ['queued', 'restarting'].includes(j.state) ? 'active' : j.state === 'failed' ? 'attention' : 'done',
|
||||
label: j.state === 'failed' ? 'Needs attention' : j.state === 'done' ? 'Completed' : j.state === 'cancelled' ? 'Cancelled' : j.state === 'restarting' ? 'Recovering' : 'Queued',
|
||||
detail: ['queued', 'restarting'].includes(j.state) ? j.operation_stage ?? j.detail : j.detail,
|
||||
href: j.provisioning_id ? `/provisioning/${encodeURIComponent(j.provisioning_id)}` : '', job: j
|
||||
}))
|
||||
].sort((a, b) => b.at.localeCompare(a.at)));
|
||||
const isLatest = (item: (typeof activities)[number]) => !item.job || !data?.runtime.jobs.some(j => j.server === item.server && j.kind === item.job!.kind && j.id > item.job!.id);
|
||||
const attention = $derived(activities.filter(a => a.state === 'attention' && isLatest(a)).length);
|
||||
const waiting = $derived(activities.filter(a => a.state === 'active').length);
|
||||
const matching = $derived(activities.filter(a =>
|
||||
(kind === 'all' || a.kind === kind) &&
|
||||
(!server || `${a.server} ${serverTitle(a.server)}`.toLowerCase().includes(server.toLowerCase())) &&
|
||||
(view === 'overview' ? a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status)
|
||||
));
|
||||
$effect(() => {
|
||||
const params = $page.url.searchParams;
|
||||
view = ['overview', 'policies', 'activity'].includes(params.get('view') ?? '') ? params.get('view')! : 'overview';
|
||||
kind = ['health', 'drift', 'updates', 'schedule'].includes(params.get('kind') ?? '') ? params.get('kind')! : 'all';
|
||||
server = params.get('server') ?? '';
|
||||
status = ['active', 'attention', 'done'].includes(params.get('status') ?? '') ? params.get('status')! : 'all';
|
||||
limit = Math.max(25, Number(params.get('limit')) || 25);
|
||||
});
|
||||
$effect(() => {
|
||||
if (data && displayedZone !== $timeZone) {
|
||||
scheduleTime = maintenanceTimeInZone(data.runtime.schedule, $timeZone);
|
||||
displayedZone = $timeZone;
|
||||
}
|
||||
});
|
||||
function saveContext(reset = true) {
|
||||
if (reset) limit = 25;
|
||||
const url = new URL($page.url);
|
||||
for (const [key, value] of Object.entries({ view: view === 'overview' ? '' : view, kind: kind === 'all' ? '' : kind, server, status: status === 'all' ? '' : status, limit: limit > 25 ? String(limit) : '' })) {
|
||||
if (value) url.searchParams.set(key, value); else url.searchParams.delete(key);
|
||||
}
|
||||
void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
|
||||
}
|
||||
function receive(body: MaintenanceOverview) {
|
||||
data = body;
|
||||
if (!scheduleLoaded) { scheduleEnabled = body.runtime.schedule.enabled; scheduleLoaded = true; }
|
||||
}
|
||||
async function load() {
|
||||
if (busy || refreshing) return;
|
||||
refreshing = true;
|
||||
const current = ++request;
|
||||
try {
|
||||
const response = await fetch('/api/maintenance');
|
||||
if (!response.ok) throw new Error('Could not refresh maintenance.');
|
||||
const body = await response.json();
|
||||
if (current === request) { receive(body); error = ''; }
|
||||
} catch (cause) { if (current === request) error = (cause as Error).message; }
|
||||
finally { refreshing = false; }
|
||||
}
|
||||
async function act(body: Record<string, unknown>) {
|
||||
if (busy) return;
|
||||
busy = true; notice = ''; error = ''; request++;
|
||||
try {
|
||||
const response = await fetch('/api/maintenance', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
|
||||
const result = await response.json();
|
||||
if (!response.ok) throw new Error(result.error ?? 'Could not update maintenance.');
|
||||
receive(result); notice = result.notice;
|
||||
} catch (cause) { error = (cause as Error).message; }
|
||||
finally { busy = false; }
|
||||
}
|
||||
onMount(() => pollVisible(load, 5000));
|
||||
</script>
|
||||
|
||||
<svelte:head><title>Maintenance | Electric Cherry</title></svelte:head>
|
||||
<div class="page">
|
||||
<SectionHeader title="Maintenance" subtitle="Keep servers healthy, current, and ready for the next match.">
|
||||
{#snippet actions()}
|
||||
<Button disabled={busy} onclick={() => act({ action: 'scan' })}>Check now</Button>
|
||||
{#if data}<Button variant="ghost" disabled={busy} onclick={() => act({ action: 'pause', paused: !data!.policies.paused })}>{data.policies.paused ? 'Resume maintenance' : 'Pause all maintenance'}</Button>{/if}
|
||||
{/snippet}
|
||||
</SectionHeader>
|
||||
{#if error}<p class="ui-error" role="alert">{error} {data ? 'Showing the last successful refresh.' : ''} <Button size="sm" onclick={load} disabled={busy}>Retry refresh</Button></p>{/if}
|
||||
{#if notice}<p role="status">{notice}</p>{/if}
|
||||
{#if data?.policies.paused}<div class="paused"><strong>All maintenance is paused.</strong> Checks and verification continue. New actions, scheduled runs, and queued maintenance are held. Submitted operations finish recovery. Campaigns, betas, and manual Provisioning actions continue.</div>{/if}
|
||||
<Tabs {tabs} bind:value={view} onchange={() => saveContext()} ariaLabel="Maintenance views" idPrefix="maintenance-tabs" panelId="maintenance-panel" />
|
||||
<div id="maintenance-panel" role="tabpanel" aria-labelledby={`maintenance-tabs-${view}`} class="panels">
|
||||
{#if data}
|
||||
{#if view === 'policies'}
|
||||
<p class="muted">Policies apply to the current fleet and respect intentionally stopped servers. Disabling a recovery or automatic update policy holds its queued actions. Already submitted operations continue verification.</p>
|
||||
<div class="policies">{#each policies as policy}
|
||||
<section class="card">
|
||||
<div class="row-heading"><h2>{policy.title}</h2><Badge variant={data.policies[policy.id] ? 'info' : 'neutral'}>{data.policies[policy.id] ? 'Enabled' : 'Disabled'}</Badge></div>
|
||||
<p>{policy.description}</p>
|
||||
<Button disabled={busy} onclick={() => act({ action: 'policy', policy: policy.id, enabled: !data!.policies[policy.id] })}>{data.policies[policy.id] ? 'Disable' : 'Enable'} {policy.title.toLowerCase()}</Button>
|
||||
</section>
|
||||
{/each}</div>
|
||||
<section class="card" id="schedule">
|
||||
<h2>Scheduled restarts</h2>
|
||||
<p>Restart the running fleet daily, even without a new release. Intentionally stopped servers are excluded. Busy servers wait until empty and may restart later.</p>
|
||||
<form onsubmit={event => { event.preventDefault(); void act({ action: 'schedule', enabled: scheduleEnabled, time: scheduleTime, timeZone: $timeZone }); }}>
|
||||
<label class="toggle"><input type="checkbox" bind:checked={scheduleEnabled} /> Daily fleet restart</label>
|
||||
<label>Queue time ({$timeZone})<input type="time" bind:value={scheduleTime} required /></label>
|
||||
<Button type="submit" disabled={busy}>Save schedule</Button>
|
||||
</form>
|
||||
<p class="muted">Saving follows daylight saving changes in your selected timezone. Skipped times run at the next valid minute; repeated times run once. Missed days become one run on resume.</p>
|
||||
{#if data.runtime.schedule.enabled && data.runtime.schedule.timeZone !== $timeZone}<p>Saved for {data.runtime.schedule.time} in {data.runtime.schedule.timeZone}. Save to use {$timeZone}.</p>{/if}
|
||||
{#if data.runtime.schedule.nextAt}<p>Next run: {$timeFormat.dateTime(data.runtime.schedule.nextAt)}{data.policies.paused ? ' · Held while maintenance is paused' : ''}</p>{/if}
|
||||
<p class="muted">Disabling the schedule stops future runs. Existing requests remain in Activity, where unsubmitted restarts can be cancelled.</p>
|
||||
</section>
|
||||
{:else}
|
||||
{#if view === 'overview'}
|
||||
<div class="summary"><span><strong>{attention}</strong> need attention</span><span><strong>{waiting}</strong> active</span><span><strong>{outdated.length}</strong> outdated</span><span><strong>{unknown.length}</strong> versions unverified</span></div>
|
||||
{#if data.runtime.schedule.nextAt}<p class="muted">Next scheduled restart: {$timeFormat.dateTime(data.runtime.schedule.nextAt)} · <a href="/maintenance?view=policies#schedule">Manage schedule</a></p>{/if}
|
||||
{#if data.recovery.lastScanAt}<p class="muted">Health and drift checked {$timeFormat.dateTime(data.recovery.lastScanAt)}</p>{/if}
|
||||
{#if data.recovery.scanError}<p class="ui-error" role="alert">{data.recovery.scanError}</p>{/if}
|
||||
{/if}
|
||||
<div class="filters">
|
||||
<label>Capability<select bind:value={kind} onchange={() => saveContext()}><option value="all">All capabilities</option><option value="health">Health recovery</option><option value="drift">Configuration drift</option><option value="updates">Plutonium updates</option><option value="schedule">Scheduled restarts</option></select></label>
|
||||
<label>Server<input bind:value={server} oninput={() => saveContext()} placeholder="Filter servers" /></label>
|
||||
{#if view === 'activity'}<label>Status<select bind:value={status} onchange={() => saveContext()}><option value="all">All activity</option><option value="active">Active</option><option value="attention">Needs attention</option><option value="done">Finished</option></select></label>{/if}
|
||||
</div>
|
||||
{#if view === 'overview' && ['all', 'updates'].includes(kind)}
|
||||
<section class="card" id="updates">
|
||||
<div class="row-heading"><h2>Plutonium updates</h2><Button disabled={busy || data.policies.paused || !outdated.some(v => !activeJobs.some(j => j.server === v.server))} onclick={() => act({ action: 'update', servers: outdated.map(v => v.server) })}>Update outdated servers on restart</Button></div>
|
||||
<p>Published revision: <strong>{data.runtime.release.revision ? `r${data.runtime.release.revision}` : 'Unknown'}</strong> · <a href={PLUTONIUM_MANIFEST_URL} target="_blank" rel="noreferrer">Production manifest</a> · <a href="https://plutonium.pw/docs/changelog/" target="_blank" rel="noreferrer">Changelog</a></p>
|
||||
{#if data.runtime.release.checkedAt}<p class="muted">Checked {$timeFormat.dateTime(data.runtime.release.checkedAt)}</p>{/if}
|
||||
{#if data.runtime.release.error}<p class="ui-error" role="alert">{data.runtime.release.error} Displayed release information is cached.</p>{/if}
|
||||
<p class="muted">Plutainer applies the available release at startup when auto-update is enabled. Maintenance verifies the running revision after recovery. Unknown versions require a successful check before automatic updates.</p>
|
||||
<div class="table-wrap"><table><thead><tr><th>Server</th><th>Running revision</th><th>Status</th><th>Action</th></tr></thead><tbody>
|
||||
{#each versions as version (version.server)}
|
||||
{@const pending = activeJobs.find(j => j.server === version.server)}
|
||||
<tr><td><a href={`/servers/${encodeURIComponent(version.server)}`}><ServerIdentity name={version.server} /></a></td><td>{version.revision ? `r${version.revision}` : 'Unknown'}{#if version.checked_at}<small>{$timeFormat.dateTime(version.checked_at)}</small>{/if}{#if version.error}<small>{version.error}</small>{/if}</td>
|
||||
<td>{version.update_status === 'outdated' ? 'Update available' : version.update_status === 'current' ? 'Up to date' : version.update_status === 'ahead' ? 'Ahead of published release' : version.update_status === 'stopped' ? 'Intentionally stopped' : 'Unverified'}</td>
|
||||
<td>{#if pending}<a href={pending.provisioning_id ? `/provisioning/${encodeURIComponent(pending.provisioning_id)}` : '#activity'}>Restart pending</a>{:else if version.update_status === 'outdated'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'update', servers: [version.server] })}>Update on restart</Button>{/if}</td></tr>
|
||||
{:else}<tr><td colspan="4">No matching fleet servers.</td></tr>{/each}
|
||||
</tbody></table></div>
|
||||
</section>
|
||||
{/if}
|
||||
<section id="activity">
|
||||
<h2>{view === 'overview' ? 'Active work & attention' : 'Maintenance activity'}</h2>
|
||||
<div class="activity">{#each matching.slice(0, limit) as item (item.key)}
|
||||
<article class="card" id={item.key}>
|
||||
<div class="row-heading"><h3>{item.title} · <ServerIdentity name={item.server} /></h3><Badge variant={item.state === 'attention' ? 'danger' : item.state === 'active' ? 'info' : 'neutral'}>{item.label}</Badge></div>
|
||||
{#if item.detail}<p>{item.detail}</p>{/if}
|
||||
<p class="muted">{$timeFormat.dateTime(item.at)}{item.job?.target_revision ? ` · Expected r${item.job.target_revision} or newer` : ''}</p>
|
||||
<div class="links">{#if item.href}<a href={item.href}>{item.job ? 'View restart operation' : 'View evidence and recovery'}</a>{/if}
|
||||
{#if item.job?.state === 'queued'}<Button size="sm" disabled={busy} onclick={() => act({ action: 'cancel', id: item.job!.id })}>Cancel restart</Button>{/if}
|
||||
{#if item.job?.state === 'failed' && item.job.operation_state === 'done'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'retry', id: item.job!.id })}>Retry after inspection</Button>{/if}
|
||||
</div>
|
||||
</article>
|
||||
{:else}<EmptyState title={view === 'overview' ? 'No active maintenance in this view' : 'No matching activity'} description="Findings, scheduled restarts, and update verification appear here." />{/each}</div>
|
||||
<ListFooter total={matching.length} bind:limit onchange={() => saveContext(false)} label="activities" />
|
||||
</section>
|
||||
{/if}
|
||||
{:else if !error}<p role="status">Loading maintenance…</p>{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.page, .panels, section, .activity, .policies { display: grid; gap: 20px; }
|
||||
.page { gap: 28px; }
|
||||
.card { padding: 24px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); display: grid; gap: 16px; }
|
||||
.row-heading, .summary, .filters, form, .links { display: flex; gap: 16px 24px; align-items: center; flex-wrap: wrap; }
|
||||
.row-heading { justify-content: space-between; }
|
||||
.summary, .paused { padding: 20px; background: var(--surface-raised); border: 1px solid var(--border-strong); border-radius: var(--radius-md); }
|
||||
.paused { line-height: 1.7; }
|
||||
h2 { font-size: var(--text-md); } h3 { font-size: var(--text-sm); } h2, h3, p { margin: 0; }
|
||||
p, label, td, th { font-size: var(--text-sm); line-height: 1.6; }
|
||||
.muted, small { color: var(--text-secondary); } small { display: block; }
|
||||
a { color: var(--accent-primary-hover); } label { display: grid; gap: 6px; } .toggle { display: flex; align-items: center; }
|
||||
input:not([type='checkbox']), select { color: inherit; background: var(--surface); padding: 9px; border: 1px solid var(--border-strong); border-radius: var(--radius-sm); max-width: 100%; }
|
||||
.table-wrap { overflow-x: auto; } table { width: 100%; border-collapse: collapse; text-align: left; } td, th { padding: 12px 8px; border-bottom: 1px solid var(--border-subtle); }
|
||||
.activity article, section { scroll-margin-top: 100px; }
|
||||
@media (max-width: 640px) { .card { padding: 18px; } }
|
||||
</style>
|
||||
@@ -0,0 +1,205 @@
|
||||
<script lang="ts">
|
||||
import { onMount, untrack } from 'svelte';
|
||||
import { page } from '$app/stores';
|
||||
import type { WatchdogDetail, WatchdogEvidence } from '$lib/watchdog.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { timeFormat } from '$ui/stores/time-zone.js';
|
||||
import { serverTitle } from '$ui/server-labels.js';
|
||||
import WatchdogStatus from '$ui/components/WatchdogStatus.svelte';
|
||||
import Badge from '$ui/components/primitives/Badge.svelte';
|
||||
import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
|
||||
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
|
||||
|
||||
let detail = $state<WatchdogDetail | null>(null);
|
||||
let loadError = $state('');
|
||||
let actionError = $state('');
|
||||
let notice = $state('');
|
||||
let busy = $state<'retry' | 'dismiss' | null>(null);
|
||||
let eventLimit = $state(25);
|
||||
let fileLimit = $state(25);
|
||||
let request = 0;
|
||||
const incident = $derived(detail?.incident);
|
||||
const evidence = $derived.by((): WatchdogEvidence => {
|
||||
try {
|
||||
const value = JSON.parse(incident?.details ?? '{}');
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
||||
} catch { return {}; }
|
||||
});
|
||||
const files = $derived(evidence.files ?? []);
|
||||
const actions = $derived(detail?.actions.slice().reverse() ?? []);
|
||||
const events = $derived(detail?.events.slice().reverse() ?? []);
|
||||
const activeOperation = $derived(actions.some(action => ['queued', 'waiting', 'running', 'awaiting-health'].includes(action.operation_state ?? '')));
|
||||
const canRetry = $derived(incident?.state === 'attention' || incident?.state === 'dismissed');
|
||||
const canDismiss = $derived(incident?.state === 'open' || incident?.state === 'attention');
|
||||
const explanation = $derived.by(() => {
|
||||
if (!incident) return '';
|
||||
if (incident.state === 'resolved') return incident.kind === 'drift' ? 'The observed file drift has cleared.' : 'The server passed the checks for this incident.';
|
||||
if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.';
|
||||
if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.';
|
||||
if (incident.state === 'recovering') return 'Recovery is in progress. Follow the provisioning operations below for eligibility checks, progress, and health verification.';
|
||||
return 'Maintenance detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server.';
|
||||
});
|
||||
|
||||
async function load() {
|
||||
if (busy) return;
|
||||
const id = $page.params.id ?? '';
|
||||
const current = ++request;
|
||||
try {
|
||||
const response = await fetch(`/api/maintenance/${encodeURIComponent(id)}`);
|
||||
if (!response.ok) throw new Error(response.status === 404 ? 'Maintenance finding not found.' : 'Could not refresh this incident.');
|
||||
const body = await response.json() as WatchdogDetail;
|
||||
if (current !== request || id !== $page.params.id) return;
|
||||
detail = body; loadError = '';
|
||||
} catch (error) {
|
||||
if (current === request && id === $page.params.id) loadError = (error as Error).message;
|
||||
}
|
||||
}
|
||||
|
||||
async function update(action: 'retry' | 'dismiss') {
|
||||
if (busy || !incident) return;
|
||||
const id = incident.id;
|
||||
const current = ++request;
|
||||
busy = action; actionError = ''; notice = '';
|
||||
try {
|
||||
const response = await fetch(`/api/maintenance/${encodeURIComponent(id)}`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action })
|
||||
});
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? 'Could not update this incident.');
|
||||
if (current !== request || id !== $page.params.id) return;
|
||||
detail = body; loadError = '';
|
||||
notice = action === 'retry' ? 'Recovery retry requested. Maintenance will recheck the server before taking action.' : 'Incident dismissed.';
|
||||
} catch (error) {
|
||||
if (current === request && id === $page.params.id) actionError = (error as Error).message;
|
||||
} finally { if (current === request) busy = null; }
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
$page.params.id;
|
||||
request++; detail = null; loadError = ''; actionError = ''; notice = ''; busy = null;
|
||||
eventLimit = 25; fileLimit = 25;
|
||||
untrack(() => { void load(); });
|
||||
});
|
||||
onMount(() => {
|
||||
const stop = pollVisible(load, 5000, false);
|
||||
return () => { stop(); request++; };
|
||||
});
|
||||
</script>
|
||||
|
||||
<svelte:head><title>{incident ? `${serverTitle(incident.server)} · Maintenance` : 'Maintenance finding'} | Electric Cherry</title></svelte:head>
|
||||
|
||||
<div class="page">
|
||||
<SectionHeader title={incident ? `${incident.kind === 'drift' ? 'File drift' : 'Server health'} · ${serverTitle(incident.server)}` : 'Maintenance finding'} subtitle={incident?.id}>
|
||||
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Maintenance', href: '/maintenance' }, { label: 'Incident' }]} />{/snippet}
|
||||
{#snippet actions()}
|
||||
{#if canRetry}<Button disabled={!!busy || activeOperation} loading={busy === 'retry'} onclick={() => update('retry')}>Retry recovery</Button>{/if}
|
||||
{#if canDismiss}<Button variant="ghost" disabled={!!busy || activeOperation} loading={busy === 'dismiss'} onclick={() => update('dismiss')}>Dismiss incident</Button>{/if}
|
||||
{/snippet}
|
||||
</SectionHeader>
|
||||
|
||||
{#if loadError}<div class="ui-error" role="alert">{loadError} {detail ? 'Showing the last successful refresh.' : ''}<Button size="sm" disabled={!!busy} onclick={load}>Retry refresh</Button></div>{/if}
|
||||
{#if actionError}<p class="ui-error" role="alert">{actionError}</p>{/if}
|
||||
{#if notice}<p class="notice" role="status">{notice}</p>{/if}
|
||||
|
||||
{#if incident && detail}
|
||||
<section class="summary" aria-label="Incident status">
|
||||
<div class="row-heading"><WatchdogStatus state={incident.state} /><a href={`/servers/${encodeURIComponent(incident.server)}`}>View server</a></div>
|
||||
<h2>{incident.summary}</h2>
|
||||
<p>{explanation}</p>
|
||||
{#if activeOperation && (canRetry || canDismiss)}<p>Complete or cancel the active provisioning operation before retrying or dismissing this incident.</p>{/if}
|
||||
{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}
|
||||
<dl>
|
||||
<div><dt>First detected</dt><dd>{$timeFormat.dateTime(incident.first_seen_at)}</dd></div>
|
||||
<div><dt>Last observed</dt><dd>{$timeFormat.dateTime(incident.last_seen_at)}</dd></div>
|
||||
{#if incident.finished_at}<div><dt>Finished</dt><dd>{$timeFormat.dateTime(incident.finished_at)}</dd></div>{/if}
|
||||
{#if evidence.containerState}<div><dt>Observed Komodo state</dt><dd>{evidence.containerState}</dd></div>{/if}
|
||||
</dl>
|
||||
{#if evidence.reason && evidence.reason !== incident.summary}<p class="observed-reason">Observed reason: {evidence.reason}</p>{/if}
|
||||
<div class="links"><a href={`/servers/${encodeURIComponent(incident.server)}?tab=diagnostics`}>Diagnose connection</a><a href={`/servers/${encodeURIComponent(incident.server)}?tab=logs`}>Open server logs</a></div>
|
||||
</section>
|
||||
|
||||
<section aria-labelledby="recovery-title">
|
||||
<h2 id="recovery-title">Recovery operations</h2>
|
||||
{#if !actions.length}<EmptyState title="No recovery operations yet" description="Provisioning operations and drift campaigns will appear here when recovery starts." />
|
||||
{:else}<div class="operations">
|
||||
{#each actions as action (action.id)}
|
||||
<article class="operation">
|
||||
<div class="row-heading"><h3>{action.action === 'restart' ? 'Restart server' : 'Rebuild server'}</h3>{#if action.operation_state}<StatusBadge state={action.operation_state} />{/if}</div>
|
||||
{#if action.operation_stage}<p>{action.operation_stage}</p>{/if}
|
||||
{#if action.operation_error}<p class="error">{action.operation_error}</p>{/if}
|
||||
<time datetime={action.created_at}>{$timeFormat.dateTime(action.created_at)}</time>
|
||||
<div class="links"><a href={`/provisioning/${encodeURIComponent(action.operation_id)}`}>View provisioning</a>{#if action.campaign_id}<a href={`/campaigns/${encodeURIComponent(action.campaign_id)}`}>View repair campaign</a>{/if}</div>
|
||||
</article>
|
||||
{/each}
|
||||
</div>{/if}
|
||||
</section>
|
||||
|
||||
{#if incident.kind === 'drift'}
|
||||
<section aria-labelledby="files-title">
|
||||
<h2 id="files-title">Observed file drift <span class="muted">{files.length}</span></h2>
|
||||
<p class="muted">These differences describe the observed problem. Recovery rebuilds the whole server instance.</p>
|
||||
{#if !files.length}<EmptyState title="No file differences recorded" />
|
||||
{:else}<div class="files">
|
||||
{#each files.slice(0, fileLimit) as file (`${file.kind}:${file.driftKind}:${file.relPath}`)}
|
||||
<article class="file">
|
||||
<div class="file-heading"><Badge variant={file.driftKind === 'untracked' ? 'danger' : 'warning'}>{file.driftKind === 'replace' ? 'Modified' : file.driftKind === 'missing' ? 'Missing' : 'Untracked'}</Badge><code>{file.relPath}</code><span class="muted">{file.kind}</span></div>
|
||||
<details><summary>Observed and expected file</summary><dl>
|
||||
<div><dt>Observed</dt><dd>{file.beforeSize === null ? 'Absent' : `${file.beforeSize} bytes`}{#if file.beforeSha256}<code>{file.beforeSha256}</code>{/if}</dd></div>
|
||||
<div><dt>Expected</dt><dd>{file.afterSize === null ? 'Absent' : `${file.afterSize} bytes`}{#if file.afterSha256}<code>{file.afterSha256}</code>{/if}</dd></div>
|
||||
</dl></details>
|
||||
</article>
|
||||
{/each}
|
||||
</div><ListFooter total={files.length} bind:limit={fileLimit} label="files" />{/if}
|
||||
</section>
|
||||
{/if}
|
||||
|
||||
<section aria-labelledby="activity-title">
|
||||
<h2 id="activity-title">Activity history <span class="muted">{events.length}</span></h2>
|
||||
<p class="muted">Newest first · updates automatically</p>
|
||||
{#if !events.length}<EmptyState title="No activity recorded yet" />
|
||||
{:else}<ol class="timeline">
|
||||
{#each events.slice(0, eventLimit) as event (event.id)}
|
||||
<li><time datetime={event.created_at}>{$timeFormat.dateTime(event.created_at)}</time><p>{event.message}</p></li>
|
||||
{/each}
|
||||
</ol><ListFooter total={events.length} bind:limit={eventLimit} label="events" />{/if}
|
||||
</section>
|
||||
{:else if !loadError}<p class="muted" role="status">Loading incident…</p>{/if}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.page, section, .operations, .files { display: grid; gap: 20px; }
|
||||
.page { gap: 28px; }
|
||||
.summary, .operation, .file { padding: 24px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); }
|
||||
.operation, .file { display: grid; gap: 12px; }
|
||||
.summary { gap: 16px; }
|
||||
.row-heading, .links, .file-heading { display: flex; gap: 12px 20px; align-items: center; flex-wrap: wrap; }
|
||||
.row-heading { justify-content: space-between; }
|
||||
h2, h3, p { margin: 0; }
|
||||
h2 { font-size: var(--text-md); }
|
||||
h3 { font-size: var(--text-sm); }
|
||||
p { line-height: 1.6; overflow-wrap: anywhere; }
|
||||
h2 .muted { margin-left: 8px; }
|
||||
.summary p, .operation p { font-size: var(--text-sm); color: var(--text-secondary); }
|
||||
.summary strong { color: var(--text-primary); font-weight: var(--weight-medium); }
|
||||
.links, .row-heading a { font-size: var(--text-sm); }
|
||||
a { color: var(--accent-primary-hover); }
|
||||
dl { display: flex; gap: 16px 32px; flex-wrap: wrap; margin: 0; }
|
||||
dt, .muted, time, summary, .notice { color: var(--text-secondary); font-size: var(--text-xs); }
|
||||
dd { margin: 6px 0 0; font-size: var(--text-sm); overflow-wrap: anywhere; }
|
||||
code { font-family: var(--font-mono); font-size: var(--text-xs); overflow-wrap: anywhere; }
|
||||
dd code { display: block; margin-top: 8px; color: var(--text-secondary); }
|
||||
details { border-top: 1px solid var(--border-subtle); padding-top: 12px; }
|
||||
summary { cursor: pointer; }
|
||||
details[open] summary { margin-bottom: 16px; }
|
||||
.operation .error { color: var(--state-danger); }
|
||||
.timeline { list-style: none; padding: 0 0 0 18px; margin: 0 0 0 6px; border-left: 2px solid var(--border-strong); }
|
||||
.timeline li { position: relative; padding: 0 0 24px 12px; }
|
||||
.timeline li:last-child { padding-bottom: 0; }
|
||||
.timeline li::before { content: ''; position: absolute; left: -24px; top: 5px; width: 10px; height: 10px; border-radius: 50%; background: var(--accent-primary); }
|
||||
.timeline p { margin-top: 6px; font-size: var(--text-sm); }
|
||||
@media (max-width: 640px) { .summary, .operation, .file { padding: 18px; } }
|
||||
</style>
|
||||
Reference in New Issue
Block a user