feat: unify recovery, updates, and schedules under Maintenance
This commit is contained in:
@@ -0,0 +1,221 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { page } from '$app/stores';
|
||||
import { goto } from '$app/navigation';
|
||||
import type { MaintenanceOverview, MaintenancePolicy } from '$lib/maintenance.js';
|
||||
import { maintenanceTimeInZone, PLUTONIUM_MANIFEST_URL } from '$lib/plutonium.js';
|
||||
import { watchdogStateLabels } from '$lib/watchdog.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { timeFormat, timeZone } from '$ui/stores/time-zone.js';
|
||||
import { serverTitle } from '$ui/server-labels.js';
|
||||
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import Tabs from '$ui/components/primitives/Tabs.svelte';
|
||||
import Badge from '$ui/components/primitives/Badge.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
|
||||
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
|
||||
|
||||
let data = $state<MaintenanceOverview | null>(null);
|
||||
let error = $state('');
|
||||
let notice = $state('');
|
||||
let busy = $state(false);
|
||||
let refreshing = false;
|
||||
let view = $state('overview');
|
||||
let kind = $state('all');
|
||||
let server = $state('');
|
||||
let status = $state('all');
|
||||
let limit = $state(25);
|
||||
let scheduleEnabled = $state(false);
|
||||
let scheduleTime = $state('06:00');
|
||||
let scheduleLoaded = false;
|
||||
let displayedZone = $state('');
|
||||
let request = 0;
|
||||
const tabs = [{ id: 'overview', label: 'Overview' }, { id: 'policies', label: 'Policies & schedules' }, { id: 'activity', label: 'Activity' }];
|
||||
const policies: { id: MaintenancePolicy; title: string; description: string }[] = [
|
||||
{ id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Rebuilding deletes instance files and installs the latest promoted setup.' },
|
||||
{ id: 'drift', title: 'Configuration drift', description: 'Rebuild servers whose managed files differ from the promoted setup. Record the differences in a linked Campaign. Rebuilding deletes instance files.' },
|
||||
{ id: 'updates', title: 'Automatic Plutonium updates', description: 'Update outdated running servers on restart. One automatic attempt per published revision; failed verification requires inspection. Disabled by default.' }
|
||||
];
|
||||
const versions = $derived(data?.runtime.servers.filter(v => !server || `${v.server} ${serverTitle(v.server)}`.toLowerCase().includes(server.toLowerCase())) ?? []);
|
||||
const outdated = $derived(versions.filter(v => v.update_status === 'outdated'));
|
||||
const unknown = $derived(versions.filter(v => v.update_status === 'unknown'));
|
||||
const activeJobs = $derived(data?.runtime.jobs.filter(j => ['queued', 'restarting'].includes(j.state)) ?? []);
|
||||
const activities = $derived([
|
||||
...(data?.recovery.incidents ?? []).map(i => ({
|
||||
key: i.id, kind: i.kind, server: i.server, at: i.first_seen_at,
|
||||
title: i.kind === 'health' ? 'Health recovery' : 'Configuration drift',
|
||||
state: ['open', 'recovering'].includes(i.state) ? 'active' : i.state === 'attention' ? 'attention' : 'done',
|
||||
label: watchdogStateLabels[i.state], detail: i.state === 'recovering' ? i.operation_stage ?? i.summary : i.summary, href: `/maintenance/${encodeURIComponent(i.id)}`, job: null
|
||||
})),
|
||||
...(data?.runtime.jobs ?? []).map(j => ({
|
||||
key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, at: j.requested_at,
|
||||
title: j.kind === 'schedule' ? 'Scheduled restart' : j.kind === 'update' ? 'Plutonium update' : 'Legacy maintenance',
|
||||
state: ['queued', 'restarting'].includes(j.state) ? 'active' : j.state === 'failed' ? 'attention' : 'done',
|
||||
label: j.state === 'failed' ? 'Needs attention' : j.state === 'done' ? 'Completed' : j.state === 'cancelled' ? 'Cancelled' : j.state === 'restarting' ? 'Recovering' : 'Queued',
|
||||
detail: ['queued', 'restarting'].includes(j.state) ? j.operation_stage ?? j.detail : j.detail,
|
||||
href: j.provisioning_id ? `/provisioning/${encodeURIComponent(j.provisioning_id)}` : '', job: j
|
||||
}))
|
||||
].sort((a, b) => b.at.localeCompare(a.at)));
|
||||
const isLatest = (item: (typeof activities)[number]) => !item.job || !data?.runtime.jobs.some(j => j.server === item.server && j.kind === item.job!.kind && j.id > item.job!.id);
|
||||
const attention = $derived(activities.filter(a => a.state === 'attention' && isLatest(a)).length);
|
||||
const waiting = $derived(activities.filter(a => a.state === 'active').length);
|
||||
const matching = $derived(activities.filter(a =>
|
||||
(kind === 'all' || a.kind === kind) &&
|
||||
(!server || `${a.server} ${serverTitle(a.server)}`.toLowerCase().includes(server.toLowerCase())) &&
|
||||
(view === 'overview' ? a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status)
|
||||
));
|
||||
$effect(() => {
|
||||
const params = $page.url.searchParams;
|
||||
view = ['overview', 'policies', 'activity'].includes(params.get('view') ?? '') ? params.get('view')! : 'overview';
|
||||
kind = ['health', 'drift', 'updates', 'schedule'].includes(params.get('kind') ?? '') ? params.get('kind')! : 'all';
|
||||
server = params.get('server') ?? '';
|
||||
status = ['active', 'attention', 'done'].includes(params.get('status') ?? '') ? params.get('status')! : 'all';
|
||||
limit = Math.max(25, Number(params.get('limit')) || 25);
|
||||
});
|
||||
$effect(() => {
|
||||
if (data && displayedZone !== $timeZone) {
|
||||
scheduleTime = maintenanceTimeInZone(data.runtime.schedule, $timeZone);
|
||||
displayedZone = $timeZone;
|
||||
}
|
||||
});
|
||||
function saveContext(reset = true) {
|
||||
if (reset) limit = 25;
|
||||
const url = new URL($page.url);
|
||||
for (const [key, value] of Object.entries({ view: view === 'overview' ? '' : view, kind: kind === 'all' ? '' : kind, server, status: status === 'all' ? '' : status, limit: limit > 25 ? String(limit) : '' })) {
|
||||
if (value) url.searchParams.set(key, value); else url.searchParams.delete(key);
|
||||
}
|
||||
void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
|
||||
}
|
||||
function receive(body: MaintenanceOverview) {
|
||||
data = body;
|
||||
if (!scheduleLoaded) { scheduleEnabled = body.runtime.schedule.enabled; scheduleLoaded = true; }
|
||||
}
|
||||
async function load() {
|
||||
if (busy || refreshing) return;
|
||||
refreshing = true;
|
||||
const current = ++request;
|
||||
try {
|
||||
const response = await fetch('/api/maintenance');
|
||||
if (!response.ok) throw new Error('Could not refresh maintenance.');
|
||||
const body = await response.json();
|
||||
if (current === request) { receive(body); error = ''; }
|
||||
} catch (cause) { if (current === request) error = (cause as Error).message; }
|
||||
finally { refreshing = false; }
|
||||
}
|
||||
async function act(body: Record<string, unknown>) {
|
||||
if (busy) return;
|
||||
busy = true; notice = ''; error = ''; request++;
|
||||
try {
|
||||
const response = await fetch('/api/maintenance', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
|
||||
const result = await response.json();
|
||||
if (!response.ok) throw new Error(result.error ?? 'Could not update maintenance.');
|
||||
receive(result); notice = result.notice;
|
||||
} catch (cause) { error = (cause as Error).message; }
|
||||
finally { busy = false; }
|
||||
}
|
||||
onMount(() => pollVisible(load, 5000));
|
||||
</script>
|
||||
|
||||
<svelte:head><title>Maintenance | Electric Cherry</title></svelte:head>
|
||||
<div class="page">
|
||||
<SectionHeader title="Maintenance" subtitle="Keep servers healthy, current, and ready for the next match.">
|
||||
{#snippet actions()}
|
||||
<Button disabled={busy} onclick={() => act({ action: 'scan' })}>Check now</Button>
|
||||
{#if data}<Button variant="ghost" disabled={busy} onclick={() => act({ action: 'pause', paused: !data!.policies.paused })}>{data.policies.paused ? 'Resume maintenance' : 'Pause all maintenance'}</Button>{/if}
|
||||
{/snippet}
|
||||
</SectionHeader>
|
||||
{#if error}<p class="ui-error" role="alert">{error} {data ? 'Showing the last successful refresh.' : ''} <Button size="sm" onclick={load} disabled={busy}>Retry refresh</Button></p>{/if}
|
||||
{#if notice}<p role="status">{notice}</p>{/if}
|
||||
{#if data?.policies.paused}<div class="paused"><strong>All maintenance is paused.</strong> Checks and verification continue. New actions, scheduled runs, and queued maintenance are held. Submitted operations finish recovery. Campaigns, betas, and manual Provisioning actions continue.</div>{/if}
|
||||
<Tabs {tabs} bind:value={view} onchange={() => saveContext()} ariaLabel="Maintenance views" idPrefix="maintenance-tabs" panelId="maintenance-panel" />
|
||||
<div id="maintenance-panel" role="tabpanel" aria-labelledby={`maintenance-tabs-${view}`} class="panels">
|
||||
{#if data}
|
||||
{#if view === 'policies'}
|
||||
<p class="muted">Policies apply to the current fleet and respect intentionally stopped servers. Disabling a recovery or automatic update policy holds its queued actions. Already submitted operations continue verification.</p>
|
||||
<div class="policies">{#each policies as policy}
|
||||
<section class="card">
|
||||
<div class="row-heading"><h2>{policy.title}</h2><Badge variant={data.policies[policy.id] ? 'info' : 'neutral'}>{data.policies[policy.id] ? 'Enabled' : 'Disabled'}</Badge></div>
|
||||
<p>{policy.description}</p>
|
||||
<Button disabled={busy} onclick={() => act({ action: 'policy', policy: policy.id, enabled: !data!.policies[policy.id] })}>{data.policies[policy.id] ? 'Disable' : 'Enable'} {policy.title.toLowerCase()}</Button>
|
||||
</section>
|
||||
{/each}</div>
|
||||
<section class="card" id="schedule">
|
||||
<h2>Scheduled restarts</h2>
|
||||
<p>Restart the running fleet daily, even without a new release. Intentionally stopped servers are excluded. Busy servers wait until empty and may restart later.</p>
|
||||
<form onsubmit={event => { event.preventDefault(); void act({ action: 'schedule', enabled: scheduleEnabled, time: scheduleTime, timeZone: $timeZone }); }}>
|
||||
<label class="toggle"><input type="checkbox" bind:checked={scheduleEnabled} /> Daily fleet restart</label>
|
||||
<label>Queue time ({$timeZone})<input type="time" bind:value={scheduleTime} required /></label>
|
||||
<Button type="submit" disabled={busy}>Save schedule</Button>
|
||||
</form>
|
||||
<p class="muted">Saving follows daylight saving changes in your selected timezone. Skipped times run at the next valid minute; repeated times run once. Missed days become one run on resume.</p>
|
||||
{#if data.runtime.schedule.enabled && data.runtime.schedule.timeZone !== $timeZone}<p>Saved for {data.runtime.schedule.time} in {data.runtime.schedule.timeZone}. Save to use {$timeZone}.</p>{/if}
|
||||
{#if data.runtime.schedule.nextAt}<p>Next run: {$timeFormat.dateTime(data.runtime.schedule.nextAt)}{data.policies.paused ? ' · Held while maintenance is paused' : ''}</p>{/if}
|
||||
<p class="muted">Disabling the schedule stops future runs. Existing requests remain in Activity, where unsubmitted restarts can be cancelled.</p>
|
||||
</section>
|
||||
{:else}
|
||||
{#if view === 'overview'}
|
||||
<div class="summary"><span><strong>{attention}</strong> need attention</span><span><strong>{waiting}</strong> active</span><span><strong>{outdated.length}</strong> outdated</span><span><strong>{unknown.length}</strong> versions unverified</span></div>
|
||||
{#if data.runtime.schedule.nextAt}<p class="muted">Next scheduled restart: {$timeFormat.dateTime(data.runtime.schedule.nextAt)} · <a href="/maintenance?view=policies#schedule">Manage schedule</a></p>{/if}
|
||||
{#if data.recovery.lastScanAt}<p class="muted">Health and drift checked {$timeFormat.dateTime(data.recovery.lastScanAt)}</p>{/if}
|
||||
{#if data.recovery.scanError}<p class="ui-error" role="alert">{data.recovery.scanError}</p>{/if}
|
||||
{/if}
|
||||
<div class="filters">
|
||||
<label>Capability<select bind:value={kind} onchange={() => saveContext()}><option value="all">All capabilities</option><option value="health">Health recovery</option><option value="drift">Configuration drift</option><option value="updates">Plutonium updates</option><option value="schedule">Scheduled restarts</option></select></label>
|
||||
<label>Server<input bind:value={server} oninput={() => saveContext()} placeholder="Filter servers" /></label>
|
||||
{#if view === 'activity'}<label>Status<select bind:value={status} onchange={() => saveContext()}><option value="all">All activity</option><option value="active">Active</option><option value="attention">Needs attention</option><option value="done">Finished</option></select></label>{/if}
|
||||
</div>
|
||||
{#if view === 'overview' && ['all', 'updates'].includes(kind)}
|
||||
<section class="card" id="updates">
|
||||
<div class="row-heading"><h2>Plutonium updates</h2><Button disabled={busy || data.policies.paused || !outdated.some(v => !activeJobs.some(j => j.server === v.server))} onclick={() => act({ action: 'update', servers: outdated.map(v => v.server) })}>Update outdated servers on restart</Button></div>
|
||||
<p>Published revision: <strong>{data.runtime.release.revision ? `r${data.runtime.release.revision}` : 'Unknown'}</strong> · <a href={PLUTONIUM_MANIFEST_URL} target="_blank" rel="noreferrer">Production manifest</a> · <a href="https://plutonium.pw/docs/changelog/" target="_blank" rel="noreferrer">Changelog</a></p>
|
||||
{#if data.runtime.release.checkedAt}<p class="muted">Checked {$timeFormat.dateTime(data.runtime.release.checkedAt)}</p>{/if}
|
||||
{#if data.runtime.release.error}<p class="ui-error" role="alert">{data.runtime.release.error} Displayed release information is cached.</p>{/if}
|
||||
<p class="muted">Plutainer applies the available release at startup when auto-update is enabled. Maintenance verifies the running revision after recovery. Unknown versions require a successful check before automatic updates.</p>
|
||||
<div class="table-wrap"><table><thead><tr><th>Server</th><th>Running revision</th><th>Status</th><th>Action</th></tr></thead><tbody>
|
||||
{#each versions as version (version.server)}
|
||||
{@const pending = activeJobs.find(j => j.server === version.server)}
|
||||
<tr><td><a href={`/servers/${encodeURIComponent(version.server)}`}><ServerIdentity name={version.server} /></a></td><td>{version.revision ? `r${version.revision}` : 'Unknown'}{#if version.checked_at}<small>{$timeFormat.dateTime(version.checked_at)}</small>{/if}{#if version.error}<small>{version.error}</small>{/if}</td>
|
||||
<td>{version.update_status === 'outdated' ? 'Update available' : version.update_status === 'current' ? 'Up to date' : version.update_status === 'ahead' ? 'Ahead of published release' : version.update_status === 'stopped' ? 'Intentionally stopped' : 'Unverified'}</td>
|
||||
<td>{#if pending}<a href={pending.provisioning_id ? `/provisioning/${encodeURIComponent(pending.provisioning_id)}` : '#activity'}>Restart pending</a>{:else if version.update_status === 'outdated'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'update', servers: [version.server] })}>Update on restart</Button>{/if}</td></tr>
|
||||
{:else}<tr><td colspan="4">No matching fleet servers.</td></tr>{/each}
|
||||
</tbody></table></div>
|
||||
</section>
|
||||
{/if}
|
||||
<section id="activity">
|
||||
<h2>{view === 'overview' ? 'Active work & attention' : 'Maintenance activity'}</h2>
|
||||
<div class="activity">{#each matching.slice(0, limit) as item (item.key)}
|
||||
<article class="card" id={item.key}>
|
||||
<div class="row-heading"><h3>{item.title} · <ServerIdentity name={item.server} /></h3><Badge variant={item.state === 'attention' ? 'danger' : item.state === 'active' ? 'info' : 'neutral'}>{item.label}</Badge></div>
|
||||
{#if item.detail}<p>{item.detail}</p>{/if}
|
||||
<p class="muted">{$timeFormat.dateTime(item.at)}{item.job?.target_revision ? ` · Expected r${item.job.target_revision} or newer` : ''}</p>
|
||||
<div class="links">{#if item.href}<a href={item.href}>{item.job ? 'View restart operation' : 'View evidence and recovery'}</a>{/if}
|
||||
{#if item.job?.state === 'queued'}<Button size="sm" disabled={busy} onclick={() => act({ action: 'cancel', id: item.job!.id })}>Cancel restart</Button>{/if}
|
||||
{#if item.job?.state === 'failed' && item.job.operation_state === 'done'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'retry', id: item.job!.id })}>Retry after inspection</Button>{/if}
|
||||
</div>
|
||||
</article>
|
||||
{:else}<EmptyState title={view === 'overview' ? 'No active maintenance in this view' : 'No matching activity'} description="Findings, scheduled restarts, and update verification appear here." />{/each}</div>
|
||||
<ListFooter total={matching.length} bind:limit onchange={() => saveContext(false)} label="activities" />
|
||||
</section>
|
||||
{/if}
|
||||
{:else if !error}<p role="status">Loading maintenance…</p>{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.page, .panels, section, .activity, .policies { display: grid; gap: 20px; }
|
||||
.page { gap: 28px; }
|
||||
.card { padding: 24px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); display: grid; gap: 16px; }
|
||||
.row-heading, .summary, .filters, form, .links { display: flex; gap: 16px 24px; align-items: center; flex-wrap: wrap; }
|
||||
.row-heading { justify-content: space-between; }
|
||||
.summary, .paused { padding: 20px; background: var(--surface-raised); border: 1px solid var(--border-strong); border-radius: var(--radius-md); }
|
||||
.paused { line-height: 1.7; }
|
||||
h2 { font-size: var(--text-md); } h3 { font-size: var(--text-sm); } h2, h3, p { margin: 0; }
|
||||
p, label, td, th { font-size: var(--text-sm); line-height: 1.6; }
|
||||
.muted, small { color: var(--text-secondary); } small { display: block; }
|
||||
a { color: var(--accent-primary-hover); } label { display: grid; gap: 6px; } .toggle { display: flex; align-items: center; }
|
||||
input:not([type='checkbox']), select { color: inherit; background: var(--surface); padding: 9px; border: 1px solid var(--border-strong); border-radius: var(--radius-sm); max-width: 100%; }
|
||||
.table-wrap { overflow-x: auto; } table { width: 100%; border-collapse: collapse; text-align: left; } td, th { padding: 12px 8px; border-bottom: 1px solid var(--border-subtle); }
|
||||
.activity article, section { scroll-margin-top: 100px; }
|
||||
@media (max-width: 640px) { .card { padding: 18px; } }
|
||||
</style>
|
||||
Reference in New Issue
Block a user