josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: unify recovery, updates, and schedules under Maintenance

Browse Source
This commit is contained in:
josh
2026-09-27 16:00:18 -04:00
parent 8ea696417f
commit fc46cb9da4
42 changed files with 1256 additions and 833 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/routes/+layout.svelte
+4 -4
View File
@@ -52,7 +52,7 @@
// Two-key nav shortcuts (Gmail-style): `g f` fleet, `g d` provisioning,
// `g b` betas, `g s` scripts. Ignored while typing in inputs.
const shortcuts: Record<string, string> = {
f: '/', d: '/provisioning', a: '/campaigns', w: '/watchdog', b: '/betas',
f: '/', d: '/provisioning', a: '/campaigns', w: '/maintenance', b: '/betas',
k: '/server-keys', s: '/scripts', c: '/configs', p: '/plugins', l: '/leaderboard', h: '/settings', m: '/metrics'
};
let armed = false;
@@ -99,7 +99,7 @@
const tabs = [
{ href: '/', label: 'Fleet', match: (p: string) => p === '/' || p.startsWith('/servers/') },
{ href: '/scripts', label: 'Library', match: (p: string) => /^\/(scripts|configs|plugins)(\/|$)/.test(p) },
{ href: '/provisioning', label: 'Operations', match: (p: string) => /^\/(provisioning|deployments|collections|campaigns|watchdog|server-keys)(\/|$)/.test(p) },
{ href: '/provisioning', label: 'Operations', match: (p: string) => /^\/(provisioning|deployments|collections|campaigns|maintenance|watchdog|server-keys)(\/|$)/.test(p) },
{ href: '/betas', label: 'Betas', match: (p: string) => p.startsWith('/betas') },
{ href: '/settings', label: 'Settings', match: (p: string) => p.startsWith('/settings') },
{ href: '/leaderboard', label: 'Leaderboard', match: (p: string) => p.startsWith('/leaderboard') },
@@ -109,7 +109,7 @@
const secondary = $derived(tabs[1].match($page.url.pathname)
? [{ href: '/scripts', label: 'Scripts' }, { href: '/configs', label: 'Configs' }, { href: '/plugins', label: 'Plugins' }]
: tabs[2].match($page.url.pathname)
? [{ href: '/provisioning', label: 'Provisioning' }, { href: '/collections', label: 'Collections' }, { href: '/campaigns', label: 'Campaigns' }, { href: '/watchdog', label: 'Watchdog' }, { href: '/server-keys', label: 'Keys' }]
? [{ href: '/provisioning', label: 'Provisioning' }, { href: '/collections', label: 'Collections' }, { href: '/campaigns', label: 'Campaigns' }, { href: '/maintenance', label: 'Maintenance' }, { href: '/server-keys', label: 'Keys' }]
: tabs[4].match($page.url.pathname) ? [{ href: '/settings', label: 'Preferences' }] : []);
</script>
@@ -149,7 +149,7 @@
<Confirmation />
<NotificationHistory bind:open={notificationsOpen} />
<Dialog bind:open={helpOpen} title="Keyboard shortcuts" description="Press g, then the destination key. Shortcuts are inactive while typing or using a dialog.">
<dl class="shortcuts">{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Provisioning'],['g a','Campaigns'],['g w','Watchdog'],['g b','Betas'],['g h','Settings'],['g k','Keys'],['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}<div><dt><kbd>{keys}</kbd></dt><dd>{label}</dd></div>{/each}</dl>
<dl class="shortcuts">{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Provisioning'],['g a','Campaigns'],['g w','Maintenance'],['g b','Betas'],['g h','Settings'],['g k','Keys'],['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}<div><dt><kbd>{keys}</kbd></dt><dd>{label}</dd></div>{/each}</dl>
</Dialog>
</div>
src/routes/+page.svelte
+2
View File
@@ -2,6 +2,7 @@
import { onMount, untrack } from 'svelte';
import { afterNavigate, replaceState } from '$app/navigation';
import { page } from '$app/stores';
import RuntimeVersion from '$ui/components/RuntimeVersion.svelte';
import ServerCard from '$ui/components/ServerCard.svelte';
import DeployServerDialog from '$ui/components/DeployServerDialog.svelte';
import Button from '$ui/components/primitives/Button.svelte';
@@ -239,6 +240,7 @@
{/snippet}
{#snippet serverActions(server: FleetServer)}
<RuntimeVersion name={server.name} version={server.plutonium} after={server.telemetry_after} ready={server.health === 'online'} />
<button class="pin" class:selected={pinned.includes(server.name)} aria-pressed={pinned.includes(server.name)} aria-label="Pin {serverTitle(server.name)}" onclick={() => togglePin(server.name)}>{pinned.includes(server.name) ? '★ Pinned' : '☆ Pin'}</button>
{/snippet}
src/routes/api/maintenance/+server.ts
+47
View File
@@ -0,0 +1,47 @@
import { json } from '@sveltejs/kit';
import { maintenanceOverview } from '$lib/server/maintenance/index.js';
import { pauseMaintenance, setMaintenancePolicy } from '$lib/server/maintenance/policies.js';
import { watchdogTick } from '$lib/server/maintenance/recovery.js';
import { cancelMaintenance, checkPlutonium, queueUpdates, retryMaintenance, saveMaintenanceSchedule } from '$lib/server/maintenance/runtime.js';
import { validTimeZone } from '$lib/time-zone.js';
import { kick } from '$lib/server/pipeline/worker.js';
export function GET() { return json(maintenanceOverview()); }
export async function POST({ request }: { request: Request }) {
const body = await request.json().catch(() => null);
if (!body || typeof body !== 'object' || Array.isArray(body)) return json({ error: 'Invalid request.' }, { status: 400 });
let notice = '';
try {
switch (body.action) {
case 'pause':
if (typeof body.paused !== 'boolean') throw new Error('Provide a paused boolean.');
pauseMaintenance(body.paused);
notice = body.paused ? 'Maintenance paused. Submitted operations continue recovery.' : 'Maintenance resumed.';
break;
case 'policy':
if (!['health', 'drift', 'updates'].includes(body.policy) || typeof body.enabled !== 'boolean') throw new Error('Choose a policy and an enabled boolean.');
setMaintenancePolicy(body.policy, body.enabled); notice = 'Policy saved.';
break;
case 'scan':
await Promise.all([watchdogTick(true), checkPlutonium()]); notice = 'Checks finished.';
break;
case 'update':
if (body.servers !== undefined && (!Array.isArray(body.servers) || !body.servers.length || body.servers.some((v: unknown) => typeof v !== 'string'))) throw new Error('Choose existing fleet servers.');
notice = `${queueUpdates(body.servers)} update restart(s) queued. Occupied servers wait until empty.`;
break;
case 'schedule':
if (typeof body.enabled !== 'boolean' || typeof body.time !== 'string' || !validTimeZone(body.timeZone)) throw new Error('Provide enabled, a time, and a valid timezone.');
saveMaintenanceSchedule(body.enabled, body.time, body.timeZone); notice = 'Schedule saved. Existing requests are retained.';
break;
case 'cancel':
if (!Number.isSafeInteger(body.id) || !cancelMaintenance(body.id)) return json({ error: 'Only unsubmitted restarts can be cancelled.' }, { status: 409 });
notice = 'Queued restart cancelled.'; break;
case 'retry':
if (!Number.isSafeInteger(body.id)) throw new Error('Choose a maintenance activity.');
notice = `${retryMaintenance(body.id)} restart(s) queued for verification.`; break;
default: throw new Error('Unknown maintenance action.');
}
} catch (error) { return json({ error: (error as Error).message }, { status: 400 }); }
void kick().catch(error => console.error('[maintenance] worker kick failed:', error));
return json({ ...maintenanceOverview(), notice });
}
src/routes/api/maintenance/[id]/+server.ts
+16
View File
@@ -0,0 +1,16 @@
import { json } from '@sveltejs/kit';
import { watchdogDetail, updateWatchdogIncident } from '$lib/server/maintenance/recovery.js';
import { kick } from '$lib/server/pipeline/worker.js';
export function GET({ params }: { params: { id: string } }) {
const detail = watchdogDetail(params.id);
return detail ? json(detail) : json({ error: 'Maintenance finding not found.' }, { status: 404 });
}
export async function POST({ params, request }: { params: { id: string }; request: Request }) {
const body = await request.json().catch(() => null);
if (!body || !['retry','dismiss'].includes(body.action)) return json({ error: 'Choose retry or dismiss.' }, { status: 400 });
const result = updateWatchdogIncident(params.id, body.action);
if (!result.ok) return json({ error: result.error }, { status: result.status });
void kick().catch(err => console.error('[maintenance] worker kick failed:', err));
return json(watchdogDetail(params.id));
}
src/routes/api/servers/+server.ts
+3
View File
@@ -6,12 +6,15 @@ import { serverObservation } from '$lib/server/fleet/status.js';
import { queueCreateServer } from '$lib/server/provisioning/index.js';
import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
import type { GameMode, ZombieGame } from '$lib/server/fleet/map-aliases.js';
import { runningVersions } from '$lib/server/integrations/plutonium.js';
export async function GET() {
const rconHost = config.rcon.host;
const versions = new Map(runningVersions().map(version => [version.server, version]));
const servers = listServers().map((s) => {
return {
...serverObservation(s),
plutonium: versions.get(s.name),
// Derived endpoint — same logic as endpointFor() but without asking
// Komodo. If we don't have a port yet, endpoint is null.
endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
src/routes/campaigns/[id]/+page.svelte
+2 -2
View File
@@ -122,9 +122,9 @@
{/snippet}
</SectionHeader>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}</h2><p>{detail.watchdogIncidentId ? 'Watchdog repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}</p>{#if unresolved}<p>Review failed or waiting servers below. Diagnose connectivity or inspect startup logs from the affected row.</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}</h2><p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}</p>{#if unresolved}<p>Review failed or waiting servers below. Diagnose connectivity or inspect startup logs from the affected row.</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
{#if detail.watchdogIncidentId}<p class="watchdog-source">Started by Watchdog · <a href={`/watchdog/${encodeURIComponent(detail.watchdogIncidentId)}`}>View incident and recovery history</a></p>{/if}
{#if detail.watchdogIncidentId}<p class="watchdog-source">Started by Maintenance · <a href={`/maintenance/${encodeURIComponent(detail.watchdogIncidentId)}`}>View incident and recovery history</a></p>{/if}
<Card padding="lg">
<div class="meta-grid">
src/routes/maintenance/+page.svelte
+221
View File
@@ -0,0 +1,221 @@
<script lang="ts">
import { onMount } from 'svelte';
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import type { MaintenanceOverview, MaintenancePolicy } from '$lib/maintenance.js';
import { maintenanceTimeInZone, PLUTONIUM_MANIFEST_URL } from '$lib/plutonium.js';
import { watchdogStateLabels } from '$lib/watchdog.js';
import { pollVisible } from '$ui/poll.js';
import { timeFormat, timeZone } from '$ui/stores/time-zone.js';
import { serverTitle } from '$ui/server-labels.js';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Tabs from '$ui/components/primitives/Tabs.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
import Button from '$ui/components/primitives/Button.svelte';
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
let data = $state<MaintenanceOverview | null>(null);
let error = $state('');
let notice = $state('');
let busy = $state(false);
let refreshing = false;
let view = $state('overview');
let kind = $state('all');
let server = $state('');
let status = $state('all');
let limit = $state(25);
let scheduleEnabled = $state(false);
let scheduleTime = $state('06:00');
let scheduleLoaded = false;
let displayedZone = $state('');
let request = 0;
const tabs = [{ id: 'overview', label: 'Overview' }, { id: 'policies', label: 'Policies & schedules' }, { id: 'activity', label: 'Activity' }];
const policies: { id: MaintenancePolicy; title: string; description: string }[] = [
{ id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Rebuilding deletes instance files and installs the latest promoted setup.' },
{ id: 'drift', title: 'Configuration drift', description: 'Rebuild servers whose managed files differ from the promoted setup. Record the differences in a linked Campaign. Rebuilding deletes instance files.' },
{ id: 'updates', title: 'Automatic Plutonium updates', description: 'Update outdated running servers on restart. One automatic attempt per published revision; failed verification requires inspection. Disabled by default.' }
];
const versions = $derived(data?.runtime.servers.filter(v => !server || `${v.server} ${serverTitle(v.server)}`.toLowerCase().includes(server.toLowerCase())) ?? []);
const outdated = $derived(versions.filter(v => v.update_status === 'outdated'));
const unknown = $derived(versions.filter(v => v.update_status === 'unknown'));
const activeJobs = $derived(data?.runtime.jobs.filter(j => ['queued', 'restarting'].includes(j.state)) ?? []);
const activities = $derived([
...(data?.recovery.incidents ?? []).map(i => ({
key: i.id, kind: i.kind, server: i.server, at: i.first_seen_at,
title: i.kind === 'health' ? 'Health recovery' : 'Configuration drift',
state: ['open', 'recovering'].includes(i.state) ? 'active' : i.state === 'attention' ? 'attention' : 'done',
label: watchdogStateLabels[i.state], detail: i.state === 'recovering' ? i.operation_stage ?? i.summary : i.summary, href: `/maintenance/${encodeURIComponent(i.id)}`, job: null
})),
...(data?.runtime.jobs ?? []).map(j => ({
key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, at: j.requested_at,
title: j.kind === 'schedule' ? 'Scheduled restart' : j.kind === 'update' ? 'Plutonium update' : 'Legacy maintenance',
state: ['queued', 'restarting'].includes(j.state) ? 'active' : j.state === 'failed' ? 'attention' : 'done',
label: j.state === 'failed' ? 'Needs attention' : j.state === 'done' ? 'Completed' : j.state === 'cancelled' ? 'Cancelled' : j.state === 'restarting' ? 'Recovering' : 'Queued',
detail: ['queued', 'restarting'].includes(j.state) ? j.operation_stage ?? j.detail : j.detail,
href: j.provisioning_id ? `/provisioning/${encodeURIComponent(j.provisioning_id)}` : '', job: j
}))
].sort((a, b) => b.at.localeCompare(a.at)));
const isLatest = (item: (typeof activities)[number]) => !item.job || !data?.runtime.jobs.some(j => j.server === item.server && j.kind === item.job!.kind && j.id > item.job!.id);
const attention = $derived(activities.filter(a => a.state === 'attention' && isLatest(a)).length);
const waiting = $derived(activities.filter(a => a.state === 'active').length);
const matching = $derived(activities.filter(a =>
(kind === 'all' || a.kind === kind) &&
(!server || `${a.server} ${serverTitle(a.server)}`.toLowerCase().includes(server.toLowerCase())) &&
(view === 'overview' ? a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status)
));
$effect(() => {
const params = $page.url.searchParams;
view = ['overview', 'policies', 'activity'].includes(params.get('view') ?? '') ? params.get('view')! : 'overview';
kind = ['health', 'drift', 'updates', 'schedule'].includes(params.get('kind') ?? '') ? params.get('kind')! : 'all';
server = params.get('server') ?? '';
status = ['active', 'attention', 'done'].includes(params.get('status') ?? '') ? params.get('status')! : 'all';
limit = Math.max(25, Number(params.get('limit')) || 25);
});
$effect(() => {
if (data && displayedZone !== $timeZone) {
scheduleTime = maintenanceTimeInZone(data.runtime.schedule, $timeZone);
displayedZone = $timeZone;
}
});
function saveContext(reset = true) {
if (reset) limit = 25;
const url = new URL($page.url);
for (const [key, value] of Object.entries({ view: view === 'overview' ? '' : view, kind: kind === 'all' ? '' : kind, server, status: status === 'all' ? '' : status, limit: limit > 25 ? String(limit) : '' })) {
if (value) url.searchParams.set(key, value); else url.searchParams.delete(key);
}
void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
}
function receive(body: MaintenanceOverview) {
data = body;
if (!scheduleLoaded) { scheduleEnabled = body.runtime.schedule.enabled; scheduleLoaded = true; }
}
async function load() {
if (busy || refreshing) return;
refreshing = true;
const current = ++request;
try {
const response = await fetch('/api/maintenance');
if (!response.ok) throw new Error('Could not refresh maintenance.');
const body = await response.json();
if (current === request) { receive(body); error = ''; }
} catch (cause) { if (current === request) error = (cause as Error).message; }
finally { refreshing = false; }
}
async function act(body: Record<string, unknown>) {
if (busy) return;
busy = true; notice = ''; error = ''; request++;
try {
const response = await fetch('/api/maintenance', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
const result = await response.json();
if (!response.ok) throw new Error(result.error ?? 'Could not update maintenance.');
receive(result); notice = result.notice;
} catch (cause) { error = (cause as Error).message; }
finally { busy = false; }
}
onMount(() => pollVisible(load, 5000));
</script>
<svelte:head><title>Maintenance | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title="Maintenance" subtitle="Keep servers healthy, current, and ready for the next match.">
{#snippet actions()}
<Button disabled={busy} onclick={() => act({ action: 'scan' })}>Check now</Button>
{#if data}<Button variant="ghost" disabled={busy} onclick={() => act({ action: 'pause', paused: !data!.policies.paused })}>{data.policies.paused ? 'Resume maintenance' : 'Pause all maintenance'}</Button>{/if}
{/snippet}
</SectionHeader>
{#if error}<p class="ui-error" role="alert">{error} {data ? 'Showing the last successful refresh.' : ''} <Button size="sm" onclick={load} disabled={busy}>Retry refresh</Button></p>{/if}
{#if notice}<p role="status">{notice}</p>{/if}
{#if data?.policies.paused}<div class="paused"><strong>All maintenance is paused.</strong> Checks and verification continue. New actions, scheduled runs, and queued maintenance are held. Submitted operations finish recovery. Campaigns, betas, and manual Provisioning actions continue.</div>{/if}
<Tabs {tabs} bind:value={view} onchange={() => saveContext()} ariaLabel="Maintenance views" idPrefix="maintenance-tabs" panelId="maintenance-panel" />
<div id="maintenance-panel" role="tabpanel" aria-labelledby={`maintenance-tabs-${view}`} class="panels">
{#if data}
{#if view === 'policies'}
<p class="muted">Policies apply to the current fleet and respect intentionally stopped servers. Disabling a recovery or automatic update policy holds its queued actions. Already submitted operations continue verification.</p>
<div class="policies">{#each policies as policy}
<section class="card">
<div class="row-heading"><h2>{policy.title}</h2><Badge variant={data.policies[policy.id] ? 'info' : 'neutral'}>{data.policies[policy.id] ? 'Enabled' : 'Disabled'}</Badge></div>
<p>{policy.description}</p>
<Button disabled={busy} onclick={() => act({ action: 'policy', policy: policy.id, enabled: !data!.policies[policy.id] })}>{data.policies[policy.id] ? 'Disable' : 'Enable'} {policy.title.toLowerCase()}</Button>
</section>
{/each}</div>
<section class="card" id="schedule">
<h2>Scheduled restarts</h2>
<p>Restart the running fleet daily, even without a new release. Intentionally stopped servers are excluded. Busy servers wait until empty and may restart later.</p>
<form onsubmit={event => { event.preventDefault(); void act({ action: 'schedule', enabled: scheduleEnabled, time: scheduleTime, timeZone: $timeZone }); }}>
<label class="toggle"><input type="checkbox" bind:checked={scheduleEnabled} /> Daily fleet restart</label>
<label>Queue time ({$timeZone})<input type="time" bind:value={scheduleTime} required /></label>
<Button type="submit" disabled={busy}>Save schedule</Button>
</form>
<p class="muted">Saving follows daylight saving changes in your selected timezone. Skipped times run at the next valid minute; repeated times run once. Missed days become one run on resume.</p>
{#if data.runtime.schedule.enabled && data.runtime.schedule.timeZone !== $timeZone}<p>Saved for {data.runtime.schedule.time} in {data.runtime.schedule.timeZone}. Save to use {$timeZone}.</p>{/if}
{#if data.runtime.schedule.nextAt}<p>Next run: {$timeFormat.dateTime(data.runtime.schedule.nextAt)}{data.policies.paused ? ' · Held while maintenance is paused' : ''}</p>{/if}
<p class="muted">Disabling the schedule stops future runs. Existing requests remain in Activity, where unsubmitted restarts can be cancelled.</p>
</section>
{:else}
{#if view === 'overview'}
<div class="summary"><span><strong>{attention}</strong> need attention</span><span><strong>{waiting}</strong> active</span><span><strong>{outdated.length}</strong> outdated</span><span><strong>{unknown.length}</strong> versions unverified</span></div>
{#if data.runtime.schedule.nextAt}<p class="muted">Next scheduled restart: {$timeFormat.dateTime(data.runtime.schedule.nextAt)} · <a href="/maintenance?view=policies#schedule">Manage schedule</a></p>{/if}
{#if data.recovery.lastScanAt}<p class="muted">Health and drift checked {$timeFormat.dateTime(data.recovery.lastScanAt)}</p>{/if}
{#if data.recovery.scanError}<p class="ui-error" role="alert">{data.recovery.scanError}</p>{/if}
{/if}
<div class="filters">
<label>Capability<select bind:value={kind} onchange={() => saveContext()}><option value="all">All capabilities</option><option value="health">Health recovery</option><option value="drift">Configuration drift</option><option value="updates">Plutonium updates</option><option value="schedule">Scheduled restarts</option></select></label>
<label>Server<input bind:value={server} oninput={() => saveContext()} placeholder="Filter servers" /></label>
{#if view === 'activity'}<label>Status<select bind:value={status} onchange={() => saveContext()}><option value="all">All activity</option><option value="active">Active</option><option value="attention">Needs attention</option><option value="done">Finished</option></select></label>{/if}
</div>
{#if view === 'overview' && ['all', 'updates'].includes(kind)}
<section class="card" id="updates">
<div class="row-heading"><h2>Plutonium updates</h2><Button disabled={busy || data.policies.paused || !outdated.some(v => !activeJobs.some(j => j.server === v.server))} onclick={() => act({ action: 'update', servers: outdated.map(v => v.server) })}>Update outdated servers on restart</Button></div>
<p>Published revision: <strong>{data.runtime.release.revision ? `r${data.runtime.release.revision}` : 'Unknown'}</strong> · <a href={PLUTONIUM_MANIFEST_URL} target="_blank" rel="noreferrer">Production manifest</a> · <a href="https://plutonium.pw/docs/changelog/" target="_blank" rel="noreferrer">Changelog</a></p>
{#if data.runtime.release.checkedAt}<p class="muted">Checked {$timeFormat.dateTime(data.runtime.release.checkedAt)}</p>{/if}
{#if data.runtime.release.error}<p class="ui-error" role="alert">{data.runtime.release.error} Displayed release information is cached.</p>{/if}
<p class="muted">Plutainer applies the available release at startup when auto-update is enabled. Maintenance verifies the running revision after recovery. Unknown versions require a successful check before automatic updates.</p>
<div class="table-wrap"><table><thead><tr><th>Server</th><th>Running revision</th><th>Status</th><th>Action</th></tr></thead><tbody>
{#each versions as version (version.server)}
{@const pending = activeJobs.find(j => j.server === version.server)}
<tr><td><a href={`/servers/${encodeURIComponent(version.server)}`}><ServerIdentity name={version.server} /></a></td><td>{version.revision ? `r${version.revision}` : 'Unknown'}{#if version.checked_at}<small>{$timeFormat.dateTime(version.checked_at)}</small>{/if}{#if version.error}<small>{version.error}</small>{/if}</td>
<td>{version.update_status === 'outdated' ? 'Update available' : version.update_status === 'current' ? 'Up to date' : version.update_status === 'ahead' ? 'Ahead of published release' : version.update_status === 'stopped' ? 'Intentionally stopped' : 'Unverified'}</td>
<td>{#if pending}<a href={pending.provisioning_id ? `/provisioning/${encodeURIComponent(pending.provisioning_id)}` : '#activity'}>Restart pending</a>{:else if version.update_status === 'outdated'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'update', servers: [version.server] })}>Update on restart</Button>{/if}</td></tr>
{:else}<tr><td colspan="4">No matching fleet servers.</td></tr>{/each}
</tbody></table></div>
</section>
{/if}
<section id="activity">
<h2>{view === 'overview' ? 'Active work & attention' : 'Maintenance activity'}</h2>
<div class="activity">{#each matching.slice(0, limit) as item (item.key)}
<article class="card" id={item.key}>
<div class="row-heading"><h3>{item.title} · <ServerIdentity name={item.server} /></h3><Badge variant={item.state === 'attention' ? 'danger' : item.state === 'active' ? 'info' : 'neutral'}>{item.label}</Badge></div>
{#if item.detail}<p>{item.detail}</p>{/if}
<p class="muted">{$timeFormat.dateTime(item.at)}{item.job?.target_revision ? ` · Expected r${item.job.target_revision} or newer` : ''}</p>
<div class="links">{#if item.href}<a href={item.href}>{item.job ? 'View restart operation' : 'View evidence and recovery'}</a>{/if}
{#if item.job?.state === 'queued'}<Button size="sm" disabled={busy} onclick={() => act({ action: 'cancel', id: item.job!.id })}>Cancel restart</Button>{/if}
{#if item.job?.state === 'failed' && item.job.operation_state === 'done'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'retry', id: item.job!.id })}>Retry after inspection</Button>{/if}
</div>
</article>
{:else}<EmptyState title={view === 'overview' ? 'No active maintenance in this view' : 'No matching activity'} description="Findings, scheduled restarts, and update verification appear here." />{/each}</div>
<ListFooter total={matching.length} bind:limit onchange={() => saveContext(false)} label="activities" />
</section>
{/if}
{:else if !error}<p role="status">Loading maintenance…</p>{/if}
</div>
</div>
<style>
.page, .panels, section, .activity, .policies { display: grid; gap: 20px; }
.page { gap: 28px; }
.card { padding: 24px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); display: grid; gap: 16px; }
.row-heading, .summary, .filters, form, .links { display: flex; gap: 16px 24px; align-items: center; flex-wrap: wrap; }
.row-heading { justify-content: space-between; }
.summary, .paused { padding: 20px; background: var(--surface-raised); border: 1px solid var(--border-strong); border-radius: var(--radius-md); }
.paused { line-height: 1.7; }
h2 { font-size: var(--text-md); } h3 { font-size: var(--text-sm); } h2, h3, p { margin: 0; }
p, label, td, th { font-size: var(--text-sm); line-height: 1.6; }
.muted, small { color: var(--text-secondary); } small { display: block; }
a { color: var(--accent-primary-hover); } label { display: grid; gap: 6px; } .toggle { display: flex; align-items: center; }
input:not([type='checkbox']), select { color: inherit; background: var(--surface); padding: 9px; border: 1px solid var(--border-strong); border-radius: var(--radius-sm); max-width: 100%; }
.table-wrap { overflow-x: auto; } table { width: 100%; border-collapse: collapse; text-align: left; } td, th { padding: 12px 8px; border-bottom: 1px solid var(--border-subtle); }
.activity article, section { scroll-margin-top: 100px; }
@media (max-width: 640px) { .card { padding: 18px; } }
</style>
src/routes/watchdog/[id]/+page.svelte → src/routes/maintenance/[id]/+page.svelte
+8 -8
View File
@@ -41,7 +41,7 @@
if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.';
if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.';
if (incident.state === 'recovering') return 'Recovery is in progress. Follow the provisioning operations below for eligibility checks, progress, and health verification.';
return 'Watchdog detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server.';
return 'Maintenance detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server.';
});
async function load() {
@@ -49,8 +49,8 @@
const id = $page.params.id ?? '';
const current = ++request;
try {
const response = await fetch(`/api/watchdog/${encodeURIComponent(id)}`);
if (!response.ok) throw new Error(response.status === 404 ? 'Watchdog incident not found.' : 'Could not refresh this incident.');
const response = await fetch(`/api/maintenance/${encodeURIComponent(id)}`);
if (!response.ok) throw new Error(response.status === 404 ? 'Maintenance finding not found.' : 'Could not refresh this incident.');
const body = await response.json() as WatchdogDetail;
if (current !== request || id !== $page.params.id) return;
detail = body; loadError = '';
@@ -65,14 +65,14 @@
const current = ++request;
busy = action; actionError = ''; notice = '';
try {
const response = await fetch(`/api/watchdog/${encodeURIComponent(id)}`, {
const response = await fetch(`/api/maintenance/${encodeURIComponent(id)}`, {
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action })
});
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not update this incident.');
if (current !== request || id !== $page.params.id) return;
detail = body; loadError = '';
notice = action === 'retry' ? 'Recovery retry requested. Watchdog will recheck the server before taking action.' : 'Incident dismissed.';
notice = action === 'retry' ? 'Recovery retry requested. Maintenance will recheck the server before taking action.' : 'Incident dismissed.';
} catch (error) {
if (current === request && id === $page.params.id) actionError = (error as Error).message;
} finally { if (current === request) busy = null; }
@@ -90,11 +90,11 @@
});
</script>
<svelte:head><title>{incident ? `${serverTitle(incident.server)} · Watchdog` : 'Watchdog incident'} | Electric Cherry</title></svelte:head>
<svelte:head><title>{incident ? `${serverTitle(incident.server)} · Maintenance` : 'Maintenance finding'} | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title={incident ? `${incident.kind === 'drift' ? 'File drift' : 'Server health'} · ${serverTitle(incident.server)}` : 'Watchdog incident'} subtitle={incident?.id}>
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Watchdog', href: '/watchdog' }, { label: 'Incident' }]} />{/snippet}
<SectionHeader title={incident ? `${incident.kind === 'drift' ? 'File drift' : 'Server health'} · ${serverTitle(incident.server)}` : 'Maintenance finding'} subtitle={incident?.id}>
{#snippet crumbs()}<Breadcrumbs items={[{ label: 'Maintenance', href: '/maintenance' }, { label: 'Incident' }]} />{/snippet}
{#snippet actions()}
{#if canRetry}<Button disabled={!!busy || activeOperation} loading={busy === 'retry'} onclick={() => update('retry')}>Retry recovery</Button>{/if}
{#if canDismiss}<Button variant="ghost" disabled={!!busy || activeOperation} loading={busy === 'dismiss'} onclick={() => update('dismiss')}>Dismiss incident</Button>{/if}
src/routes/provisioning/[id]/+page.svelte
+1 -1
View File
@@ -152,7 +152,7 @@
{#each warnings as warning}<p class="warning">{warning}</p>{/each}
{#if deployment.expected_map}<p>Expected map: <strong>{deployment.expected_map}</strong></p>{/if}
<dl><div><dt>Started</dt><dd>{fmt(deployment.started_at)}</dd></div><div><dt>Finished</dt><dd>{deployment.finished_at ? fmt(deployment.finished_at) : 'In progress'}</dd></div><div><dt>Requested by</dt><dd>{deployment.triggered_by}</dd></div></dl>
{#if detail?.sources.length}<div class="recovery-links" aria-label="Requesting work">{#each detail.sources as source}<a href={source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}` : source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}` : source.kind === 'watchdog' ? `/watchdog/${encodeURIComponent(source.source_id)}` : '/settings#plutonium'}>{source.kind === 'campaign' ? 'Campaign' : source.kind === 'beta' ? 'Beta' : source.kind === 'watchdog' ? 'Watchdog incident' : 'Maintenance'} {source.source_id}</a>{/each}</div>{/if}
{#if detail?.sources.length}<div class="recovery-links" aria-label="Requesting work">{#each detail.sources as source}<a href={source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}` : source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}` : source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}` : `/maintenance?view=activity#job-${source.source_id}`}>{source.kind === 'campaign' ? 'Campaign' : source.kind === 'beta' ? 'Beta' : source.kind === 'watchdog' ? 'Maintenance finding' : 'Maintenance'} {source.source_id}</a>{/each}</div>{/if}
</section>
{#if detail?.recipe}<section class="saved-recipe"><div class="top"><h2>{detail.blueprintName ? `Launched from ${detail.blueprintName}` : 'Recorded launch files'}</h2><Button variant="ghost" onclick={() => open = true}>Launch another server →</Button></div><p class="muted">Launch this map with the latest promoted files using a fresh server ID and port. This does not repair the existing server.</p><LaunchRecipe recipe={detail.recipe} historical /></section><DeployServerDialog bind:open sourceDeploymentId={deployment.id} />{/if}
<OperationHistory steps={steps.map((label,index): OperationStep => ({label,state:deployment.state === 'cancelled' ? 'pending' : index < currentStep ? 'done' : index === currentStep ? (!active && ['failed','interrupted'].includes(deployment.state) ? 'failed' : 'current') : 'pending'}))} events={detail?.events ?? []} {active} />
src/routes/servers/[name]/+page.svelte
+3
View File
@@ -1,4 +1,5 @@
<script lang="ts">
import RuntimeVersion from '$ui/components/RuntimeVersion.svelte';
import MatchHistory from '$ui/components/MatchHistory.svelte';
import RoundTelemetry from '$ui/components/RoundTelemetry.svelte';
import TelemetryDiagnostics from '$ui/components/TelemetryDiagnostics.svelte';
@@ -70,6 +71,7 @@
status?: { players: number; map: string | null; gametype: string | null; roster: RosterPlayer[] };
}
interface Server {
plutonium?: import('$lib/plutonium.js').PlutoniumVersion;
telemetry_after?: string | null;
status_source?: 'telemetry' | 'rcon' | null;
rcon_status?: import('$lib/server-observation.js').GameObservation;
@@ -251,6 +253,7 @@
</SectionHeader>
<CopyValue value={name} label="Server ID" />
<RuntimeVersion {name} version={server.plutonium} after={server.telemetry_after} ready={effectiveHealth === 'online'} />
<div class="metrics">
<div class="metric-cell name-cell">
<StatusPip health={effectiveHealth} size="md" />
src/routes/settings/+page.svelte
+1 -2
View File
@@ -3,7 +3,6 @@
import { availableTimeZones, DEFAULT_TIME_ZONE } from '$lib/time-zone.js';
import { onMount } from 'svelte';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import PlutoniumMaintenance from '$ui/components/PlutoniumMaintenance.svelte';
let exampleDate = $state<Date | null>(null);
const zones = $derived([...new Set([...availableTimeZones(), $timeZone])].sort());
onMount(() => { exampleDate = new Date(); });
@@ -23,7 +22,7 @@
{#if exampleDate}<p>Example: {$timeFormat.dateTime(exampleDate)}</p>{/if}
{#if $timeZonePersistenceError}<p role="status">Browser storage is unavailable. Your timezone applies for this visit but cannot be saved.</p>{/if}
</section>
<PlutoniumMaintenance />
<p id="plutonium">Server updates and restart schedules are now in <a href="/maintenance">Operations → Maintenance</a>.</p>
</section>
<style>
src/routes/watchdog/+page.server.ts
+2
View File
@@ -0,0 +1,2 @@
import { redirect } from '@sveltejs/kit';
export function load({ url }: { url: URL }) { redirect(308, `/maintenance${url.search}`); }
src/routes/watchdog/+page.svelte
-168
View File
@@ -1,168 +0,0 @@
<script lang="ts">
import { onMount } from 'svelte';
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import { watchdogStateLabels, type WatchdogOverview, type WatchdogState } from '$lib/watchdog.js';
import { pollVisible } from '$ui/poll.js';
import { timeFormat } from '$ui/stores/time-zone.js';
import { serverTitle } from '$ui/server-labels.js';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import WatchdogStatus from '$ui/components/WatchdogStatus.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
import Button from '$ui/components/primitives/Button.svelte';
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
let overview = $state<WatchdogOverview | null>(null);
let loadError = $state('');
let actionError = $state('');
let notice = $state('');
let busy = $state<'scan' | 'toggle' | null>(null);
let filterState = $state('active');
let kind = $state('all');
let server = $state('');
let limit = $state(25);
let request = 0;
const states = Object.keys(watchdogStateLabels) as WatchdogState[];
const activeStates = ['open', 'recovering', 'attention'];
const active = $derived(overview?.incidents.filter(incident => activeStates.includes(incident.state)).length ?? 0);
const attention = $derived(overview?.incidents.filter(incident => incident.state === 'attention').length ?? 0);
const matching = $derived(overview?.incidents.filter(incident =>
(filterState === 'all' || (filterState === 'active' ? activeStates.includes(incident.state) : incident.state === filterState)) &&
(kind === 'all' || incident.kind === kind) &&
(!server || `${incident.server} ${serverTitle(incident.server)}`.toLowerCase().includes(server.toLowerCase()))
) ?? []);
$effect(() => {
const params = $page.url.searchParams;
const selectedState = params.get('state') ?? 'active';
filterState = ['active', 'all', ...states].includes(selectedState) ? selectedState : 'active';
kind = ['health', 'drift'].includes(params.get('kind') ?? '') ? params.get('kind')! : 'all';
server = params.get('server') ?? '';
limit = Math.max(25, Number(params.get('limit')) || 25);
});
function saveContext(reset = false) {
if (reset) limit = 25;
const url = new URL($page.url);
for (const [key, value] of Object.entries({ state: filterState === 'active' ? '' : filterState, kind: kind === 'all' ? '' : kind, server, limit: limit > 25 ? String(limit) : '' })) {
if (value) url.searchParams.set(key, value); else url.searchParams.delete(key);
}
void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
}
async function load() {
if (busy) return;
const current = ++request;
try {
const response = await fetch('/api/watchdog');
if (!response.ok) throw new Error('Could not refresh Watchdog.');
const body = await response.json() as WatchdogOverview;
if (current !== request) return;
overview = body; loadError = '';
} catch (error) {
if (current === request) loadError = (error as Error).message;
}
}
async function update(action: 'scan' | 'toggle') {
if (busy || !overview) return;
busy = action; actionError = ''; notice = '';
const current = ++request;
const enabled = !overview.enabled;
try {
const response = await fetch('/api/watchdog', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(action === 'scan' ? { action: 'scan' } : { enabled })
});
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not update Watchdog.');
if (current !== request) return;
overview = body; loadError = '';
notice = action === 'scan' ? (body.scanError ? 'Scan finished with errors. Review the scan details below.' : 'Scan complete.') : enabled ? 'Automatic recovery resumed.' : 'Automatic recovery paused. Queued Watchdog recovery is held; already submitted operations continue.';
} catch (error) {
if (current === request) actionError = (error as Error).message;
} finally { if (current === request) busy = null; }
}
onMount(() => {
const stop = pollVisible(load, 5000);
return () => { stop(); request++; };
});
</script>
<svelte:head><title>Watchdog | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title="Watchdog" subtitle="Automatic server recovery and drift repair.">
{#snippet actions()}
<Button disabled={!overview || !!busy} loading={busy === 'scan'} onclick={() => update('scan')}>Scan now</Button>
<Button disabled={!overview || !!busy} loading={busy === 'toggle'} onclick={() => update('toggle')}>{overview?.enabled ? 'Pause automation' : 'Resume automation'}</Button>
{/snippet}
</SectionHeader>
{#if loadError}<div class="ui-error" role="alert">{loadError} {overview ? 'Showing the last successful refresh.' : ''}<Button size="sm" disabled={!!busy} onclick={load}>Retry</Button></div>{/if}
{#if actionError}<p class="ui-error" role="alert">{actionError}</p>{/if}
{#if notice}<p class="notice" role="status">{notice}</p>{/if}
{#if overview}
<section class="automation" aria-label="Automation status">
<div class="automation-heading"><h2>Automatic recovery</h2><Badge variant={overview.enabled ? 'online' : 'warning'}>{overview.enabled ? 'Enabled' : 'Paused'}</Badge></div>
<p>{overview.enabled ? 'Watchdog restarts unhealthy servers, then rebuilds if recovery fails.' : 'Watchdog continues detecting incidents while automatic recovery is paused. Queued Watchdog recovery is held; already submitted operations continue.'}</p>
<p>Drift repair rebuilds the server with the latest promoted setup and records a campaign. <strong>Rebuilds delete the existing instance and all of its files.</strong> Interruption requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry.</p>
<div class="meta"><span>{active} active incident{active === 1 ? '' : 's'} · {attention} need attention</span><span>Last scan: {overview.lastScanAt ? $timeFormat.dateTime(overview.lastScanAt) : 'Not yet scanned'}</span></div>
{#if overview.scanError}<p class="ui-error" role="alert">Last scan: {overview.scanError}</p>{/if}
</section>
<section aria-label="Incidents">
<div class="filters">
<label>Status<select bind:value={filterState} onchange={event => { filterState = event.currentTarget.value; saveContext(true); }}><option value="active">Active incidents</option><option value="all">All incidents</option>{#each states as value}<option value={value}>{watchdogStateLabels[value]}</option>{/each}</select></label>
<label>Reason<select bind:value={kind} onchange={event => { kind = event.currentTarget.value; saveContext(true); }}><option value="all">All reasons</option><option value="health">Server health</option><option value="drift">File drift</option></select></label>
<label class="server-filter">Server<input type="search" bind:value={server} oninput={event => { server = event.currentTarget.value; saveContext(true); }} placeholder="Filter by server name" /></label>
</div>
<h2>Incidents <span class="muted">{matching.length}</span></h2>
{#if !matching.length}
<EmptyState title={filterState === 'active' && kind === 'all' && !server ? 'No active incidents' : 'No matching incidents'} description={filterState === 'active' && kind === 'all' && !server ? 'Detected health and drift problems will appear here automatically.' : 'Change the filters to review other incidents.'} />
{:else}
<div class="incidents">
{#each matching.slice(0, limit) as incident (incident.id)}
<article class="incident">
<div class="row-heading"><a class="incident-link" href={`/watchdog/${encodeURIComponent(incident.id)}`}><ServerIdentity name={incident.server} /><span class="sr-only">Watchdog incident {incident.id}</span></a><WatchdogStatus state={incident.state} /></div>
<p>{incident.summary}</p>
<div class="meta"><Badge variant="neutral">{incident.kind === 'drift' ? 'File drift' : 'Server health'}</Badge><span>Detected {$timeFormat.dateTime(incident.first_seen_at)}</span><span>Last seen {$timeFormat.dateTime(incident.last_seen_at)}</span></div>
<div class="links"><a href={`/watchdog/${encodeURIComponent(incident.id)}`}>View recovery and activity</a><a href={`/servers/${encodeURIComponent(incident.server)}?tab=diagnostics`}>Diagnose server</a></div>
</article>
{/each}
</div>
<ListFooter total={matching.length} bind:limit onchange={() => saveContext()} label="incidents" />
{/if}
</section>
{:else if !loadError}<p class="muted" role="status">Loading Watchdog…</p>{/if}
</div>
<style>
.page, section, .incidents { display: grid; gap: 20px; }
.page { gap: 28px; }
.automation, .incident { border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); padding: 24px; }
.automation { gap: 12px; }
.automation-heading, .row-heading, .meta, .links, .filters { display: flex; gap: 12px 20px; flex-wrap: wrap; align-items: center; }
.row-heading { justify-content: space-between; }
.incident { display: grid; gap: 14px; }
.incident-link { color: var(--text-primary); text-decoration: none; font-weight: var(--weight-semibold); }
.incident-link:hover { color: var(--accent-primary-hover); text-decoration: underline; }
h2 { margin: 0; font-size: var(--text-md); }
h2 .muted { margin-left: 8px; }
p { margin: 0; line-height: 1.6; overflow-wrap: anywhere; }
.automation p { max-width: 960px; font-size: var(--text-sm); color: var(--text-secondary); }
.automation strong { color: var(--text-primary); font-weight: var(--weight-medium); }
.meta, .muted { color: var(--text-secondary); font-size: var(--text-xs); }
.links { font-size: var(--text-sm); }
a { color: var(--accent-primary-hover); }
.filters { align-items: flex-end; }
label { display: grid; gap: 8px; font-size: var(--text-xs); color: var(--text-secondary); }
.server-filter { flex: 1; min-width: 200px; }
input, select { font: inherit; font-size: var(--text-sm); min-height: 40px; color: var(--text-primary); background: var(--surface-sunken); border: 1px solid var(--border-control); border-radius: var(--radius-sm); padding: 9px 12px; }
.notice { color: var(--text-secondary); font-size: var(--text-sm); }
@media (max-width: 640px) { .automation, .incident { padding: 18px; } label { width: 100%; } }
</style>
src/routes/watchdog/[id]/+page.server.ts
+2
View File
@@ -0,0 +1,2 @@
import { redirect } from '@sveltejs/kit';
export function load({ params, url }: { params: { id: string }; url: URL }) { redirect(308, `/maintenance/${encodeURIComponent(params.id)}${url.search}`); }