Add Plutonium version checks and scheduled rolling restarts

README.md

@@ -62,6 +62,40 @@ server actions or send browser push notifications after the app closes.
 Plugins support named release versions, GitHub DLL imports and update discovery.
 See [Plugin releases](docs/plugin-releases.md) for the import and beta workflow.
 
+## Plutonium updates and scheduled restarts
+
+Open **Settings → Plutonium updates & restarts** to check the latest published
+revision, inspect each server's running revision, and queue a server or fleet
+restart. Release and running-version checks also run hourly. The published
+revision comes from the [official changelog](https://plutonium.pw/docs/changelog/);
+running revisions come from RCON `version`. Unsupported replies and failed
+queries appear as unknown rather than being treated as up to date. A failed
+changelog request retains the last release with a visible error.
+
+Daily fleet restarts are optional and disabled by default. Choose a **UTC** queue
+time and save the schedule; the next run is also shown in your display timezone.
+Every due run queues the current fleet, regardless of whether an update exists.
+The schedule and jobs survive app restarts. Missed days become one run, and a
+server can have only one pending maintenance restart. Disabling the schedule
+does not cancel work already queued; use **Cancel** for those jobs.
+
+Maintenance waits for a fresh zero-player RCON response and for existing
+deployment/beta work, then uses the shared rolling restart limit. Unreachable or
+occupied servers remain queued. After container and RCON recovery, Electric
+Cherry queries the running revision again. If a fresh published revision was
+known when queued, recovery on an older or unknown revision is reported as a
+failed verification, without an automatic restart loop. Ambiguous restart
+failures likewise require inspection before a new request. Activity is retained
+for 30 days, with pending jobs and the latest 100 records shown in Settings.
+
+Updates rely on Plutainer's startup updater (`PLUTAINER_AUTO_UPDATE` must be
+enabled); Electric Cherry does not download binaries or pull a container image.
+See [Plutainer's startup flow](https://github.com/Ayymoss/Plutainer/blob/main/CLAUDE.md)
+and [environment migration reference](https://github.com/Ayymoss/Plutainer/blob/main/MIGRATION.md).
+The worker starts on the first application request, so keep the app running and
+reachable for scheduled maintenance. Validate RCON revision replies and updater
+behavior on your live T5/T6 servers before relying on unattended maintenance.
+
 ## How restarts stay safe
 
 Betas support configurable gameplay, participation, stability and manual-review

src/lib/plutonium.ts

@@ -0,0 +1,55 @@
+export interface MaintenanceSchedule {
+    enabled: boolean;
+    time: string;
+    nextAt: string | null;
+}
+export interface PlutoniumRelease {
+    revision: number | null;
+    checkedAt: string | null;
+    error: string | null;
+}
+export interface PlutoniumVersion {
+    server: string;
+    revision: number | null;
+    checked_at: string | null;
+    error: string | null;
+}
+export interface MaintenanceJob {
+    id: number;
+    server: string;
+    state: 'queued' | 'restarting' | 'done' | 'failed' | 'cancelled';
+    target_revision: number | null;
+    requested_at: string;
+    restarted_at: string | null;
+    finished_at: string | null;
+    detail: string | null;
+}
+export interface PlutoniumStatus {
+    release: PlutoniumRelease;
+    schedule: MaintenanceSchedule;
+    servers: PlutoniumVersion[];
+    jobs: MaintenanceJob[];
+}
+
+/** Accept explicit revision tokens only, never engine versions or build dates. */
+export function parsePlutoniumRevision(text: string): number | null {
+    const match = text.replace(/\^[0-9]/g, '').match(/\br([1-9]\d{2,8})\b/i);
+    return match ? Number(match[1]) : null;
+}
+
+export function parseChangelogRevision(html: string): number {
+    const headings = [...html.matchAll(/<h2\b[^>]*>([\s\S]*?)<\/h2>/gi)];
+    const revisions = headings.map(h => parsePlutoniumRevision(h[1].replace(/<[^>]*>/g, ' ')))
+        .filter((r): r is number => r !== null);
+    if (!revisions.length) throw new Error('The Plutonium changelog did not contain a release heading.');
+    return Math.max(...revisions);
+}
+
+export function nextMaintenanceAt(time: string, now = new Date()): string {
+    if (!/^([01]\d|2[0-3]):[0-5]\d$/.test(time)) throw new Error('Choose a valid UTC time (HH:mm).');
+    const next = new Date(now);
+    const [hour, minute] = time.split(':').map(Number);
+    next.setUTCHours(hour, minute, 0, 0);
+    if (next <= now) next.setUTCDate(next.getUTCDate() + 1);
+    return next.toISOString();
+}

src/lib/server/db/schema.sql

@@ -362,6 +362,24 @@ CREATE TABLE IF NOT EXISTS settings (
     key   TEXT PRIMARY KEY,
     value TEXT NOT NULL
 );
+CREATE TABLE IF NOT EXISTS plutonium_versions (
+    server TEXT PRIMARY KEY REFERENCES servers(name) ON DELETE CASCADE,
+    revision INTEGER,
+    checked_at TEXT NOT NULL,
+    error TEXT
+);
+CREATE TABLE IF NOT EXISTS maintenance_jobs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    server TEXT NOT NULL,
+    state TEXT NOT NULL DEFAULT 'queued',
+    target_revision INTEGER,
+    requested_at TEXT NOT NULL,
+    restarted_at TEXT,
+    finished_at TEXT,
+    detail TEXT
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_maintenance_pending ON maintenance_jobs(server)
+    WHERE state IN ('queued', 'restarting');
 CREATE TABLE IF NOT EXISTS server_telemetry (
     server TEXT PRIMARY KEY REFERENCES servers(name) ON DELETE CASCADE,
     round INTEGER NOT NULL,

src/lib/server/fleet/plutonium.ts

@@ -0,0 +1,173 @@
+import { db } from '../db/index.js';
+import { config, komodoConfigured, rconConfigured } from '../config.js';
+import { getServer, listServers } from './servers.js';
+import { rconGetPlayerCount, rconPlutoniumVersion } from '../integrations/rcon.js';
+import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from '../pipeline/restart-rollout.js';
+import { nextMaintenanceAt, parseChangelogRevision } from '../../plutonium.js';
+import type { MaintenanceJob, MaintenanceSchedule, PlutoniumRelease, PlutoniumStatus, PlutoniumVersion } from '../../plutonium.js';
+
+const CHANGELOG = 'https://plutonium.pw/docs/changelog/';
+const HOUR = 60 * 60 * 1000;
+const nowIso = () => new Date().toISOString();
+function readSetting<T>(key: string, fallback: T): T {
+    const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined;
+    return row ? JSON.parse(row.value) : fallback;
+}
+function writeSetting(key: string, value: unknown): void {
+    db().prepare('INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value')
+        .run(key, JSON.stringify(value));
+}
+function release(): PlutoniumRelease {
+    return readSetting('plutonium_release', { revision: null, checkedAt: null, error: null });
+}
+function schedule(): MaintenanceSchedule {
+    return readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null });
+}
+export function saveMaintenanceSchedule(enabled: boolean, time: string): void {
+    const nextAt = nextMaintenanceAt(time);
+    writeSetting('plutonium_schedule', { enabled, time, nextAt: enabled ? nextAt : null });
+}
+export function plutoniumStatus(): PlutoniumStatus {
+    const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
+    return {
+        release: release(), schedule: schedule(),
+        servers: listServers().map(s => versions.find(v => v.server === s.name)
+            ?? { server: s.name, revision: null, checked_at: null, error: null }),
+        jobs: db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting')
+            OR id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) ORDER BY id DESC`).all() as MaintenanceJob[]
+    };
+}
+
+async function probeVersion(server: string): Promise<number | null> {
+    const recovering = db().prepare('SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(server);
+    const result = recovering ? { revision: null, error: 'Waiting for restart recovery.' }
+        : await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
+    // Clear the old revision on failure: it is no longer evidence of what is running.
+    db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?)
+        ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`)
+        .run(server, result.revision, nowIso(), result.error);
+    return result.revision;
+}
+
+let checking: Promise<void> | null = null;
+export function checkPlutonium(): Promise<void> {
+    if (checking) return checking;
+    checking = check().finally(() => { checking = null; });
+    return checking;
+}
+async function check(): Promise<void> {
+    const previous = release();
+    try {
+        const response = await fetch(CHANGELOG, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' });
+        if (!response.ok) throw new Error(`Changelog request failed (HTTP ${response.status}).`);
+        const html = await response.text();
+        if (html.length > 2_000_000) throw new Error('Changelog response was too large.');
+        writeSetting('plutonium_release', { revision: parseChangelogRevision(html), checkedAt: nowIso(), error: null });
+    } catch (err) {
+        writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message });
+    }
+    const servers = listServers();
+    for (let i = 0; i < servers.length; i += 4) {
+        await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name)));
+    }
+}
+
+export function queueMaintenance(servers: string[]): number {
+    const names = new Set(listServers().map(s => s.name));
+    if (!servers.length || servers.some(s => !names.has(s))) throw new Error('Choose existing fleet servers.');
+    return db().transaction(() => {
+        const latest = release();
+        // An old cached release is still displayed, but never used as a fresh target.
+        const target = !latest.error && latest.checkedAt && Date.now() - Date.parse(latest.checkedAt) < HOUR
+            ? latest.revision : null;
+        let added = 0;
+        for (const server of new Set(servers)) {
+            added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at)
+                VALUES (?, ?, ?)`).run(server, target, nowIso()).changes;
+        }
+        return added;
+    })();
+}
+export function cancelMaintenance(id: number): boolean {
+    return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
+        WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0;
+}
+
+function conflictingWork(server: string): boolean {
+    return !!db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND state IN ('queued','running','awaiting-health')
+        UNION ALL SELECT 1 FROM deployment_servers WHERE server = ? AND restart_state IN ('pending','failed')
+        UNION ALL SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
+            WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server, server, server);
+}
+function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
+    db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?')
+        .run(state, detail, nowIso(), job.id);
+}
+function hold(job: MaintenanceJob, detail: string) {
+    db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ?').run(detail, job.id);
+}
+
+let ticking: Promise<void> | null = null;
+export function maintenanceTick(): Promise<void> {
+    if (ticking) return ticking;
+    ticking = runTick().finally(() => { ticking = null; });
+    return ticking;
+}
+async function runTick(): Promise<void> {
+    const latest = release();
+    if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR) await checkPlutonium();
+    db().transaction(() => {
+        const plan = schedule();
+        if (plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) {
+            const names = listServers().map(s => s.name);
+            if (names.length) queueMaintenance(names);
+            // Coalesce missed days after downtime into one run; no restart backlog.
+            writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time) });
+        }
+    })();
+    await refreshRestartRollout();
+    const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
+    for (const job of jobs) {
+        try { await processJob(job); }
+        catch { hold(job, 'Maintenance check failed; waiting for the next check.'); }
+    }
+    db().prepare(`DELETE FROM maintenance_jobs WHERE finished_at < ? AND state IN ('done','failed','cancelled')`)
+        .run(new Date(Date.now() - 30 * 24 * HOUR).toISOString());
+}
+async function processJob(job: MaintenanceJob): Promise<void> {
+    const server = getServer(job.server);
+    if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
+    if (job.state === 'restarting') {
+        const slot = db().prepare('SELECT requested_at FROM restart_rollout WHERE server = ?').get(job.server) as { requested_at: string } | undefined;
+        if (!slot || !job.restarted_at || slot.requested_at < job.restarted_at) {
+            return finish(job, 'failed', 'Restart submission was interrupted. Inspect the server before queuing another restart.');
+        }
+        const health = rolloutHealth(job.server);
+        if (!health.ok) return hold(job, health.reason);
+        const revision = await probeVersion(job.server);
+        if (job.target_revision && (revision === null || revision < job.target_revision)) {
+            return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
+        }
+        return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`);
+    }
+    if (!komodoConfigured() || !rconConfigured()) return hold(job, 'Configure Komodo and RCON before restarting.');
+    if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.');
+    const reason = rolloutWaitReason(job.server);
+    if (reason) return hold(job, reason);
+    const players = await rconGetPlayerCount(job.server);
+    if (players === null || players > 0) return hold(job, players === null ? 'Waiting for a fresh RCON player count.' : `Waiting for ${players} player(s) to leave.`);
+    // Recheck after async I/O; cancellation and lifecycle operations may have arrived.
+    const current = getServer(job.server);
+    if (!current || current.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
+    if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.');
+    const claimed = db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ?, detail = 'Restart requested'
+        WHERE id = ? AND state = 'queued'`).run(nowIso(), job.id);
+    if (!claimed.changes) return;
+    db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?')
+        .run('Waiting for restart recovery.', job.server);
+    // Persist intent before remote I/O. An ambiguous timeout must never cause an automatic second restart.
+    const result = await restartInRollout(job.server);
+    if (result.kind === 'queued') {
+        db().prepare(`UPDATE maintenance_jobs SET state = 'queued', restarted_at = NULL, detail = ? WHERE id = ?`).run(result.message, job.id);
+    } else if (result.kind === 'failed') finish(job, 'failed', result.message);
+}

src/lib/server/integrations/rcon.ts

@@ -3,6 +3,7 @@ import type { Socket } from 'node:dgram';
 import { config, komodoConfigured, rconConfigured } from '../config.js';
 import { getServer, setServerField } from '../fleet/servers.js';
 import { komodoResolveGamePort } from './komodo.js';
+import { parsePlutoniumRevision } from '../../plutonium.js';
 
 // Plutonium/CoD RCON is UDP with a 4-byte 0xff prefix. Requests look like
 //   \xff\xff\xff\xffrcon <password> <command>
@@ -451,3 +452,14 @@ function positiveCapacity(value: string | undefined): number | null {
     const count = Number(value);
     return Number.isInteger(count) && count > 0 ? count : null;
 }
+
+/** Query the running process, not the updater's on-disk files. */
+export async function rconPlutoniumVersion(server: string): Promise<{ revision: number | null; error: string | null }> {
+    const { endpoint } = await endpointFor(server);
+    if (!endpoint) return { revision: null, error: 'No RCON endpoint available.' };
+    const response = await sendRcon(endpoint.host, endpoint.port, 'version');
+    if (!response.ok) return { revision: null, error: response.detail };
+    const body = stripPrefix(response.buf);
+    const revision = body?.trimStart().startsWith('print') ? parsePlutoniumRevision(body) : null;
+    return { revision, error: revision === null ? 'RCON version reply did not identify a Plutonium revision.' : null };
+}

src/lib/server/pipeline/worker.ts

@@ -13,6 +13,7 @@ import { isProvisioning } from '../fleet/provisioning.js';
 import { STATUS_REFRESH_MS } from '../../server-freshness.js';
 import { recordMetrics } from '../fleet/metrics.js';
 import { selectedZombieMap } from '../../map-aliases.js';
+import { maintenanceTick } from '../fleet/plutonium.js';
 
 const TICK_MS = 15_000;
 // Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
@@ -46,6 +47,8 @@ async function runTick(): Promise<void> {
     } catch (err) {
         console.error('[worker] beta-runner error:', err);
     }
+    try { await maintenanceTick(); }
+    catch (err) { console.error('[worker] maintenance error:', err); }
 }
 
 /**

src/lib/ui/components/PlutoniumMaintenance.svelte

@@ -0,0 +1,108 @@
+<script lang="ts">
+    import { onMount } from 'svelte';
+    import type { PlutoniumStatus } from '$lib/plutonium.js';
+    import { timeFormat } from '$ui/stores/time-zone.js';
+    import Button from './primitives/Button.svelte';
+
+    let data = $state<PlutoniumStatus | null>(null);
+    let busy = $state(false);
+    let error = $state('');
+    let notice = $state('');
+    let enabled = $state(false);
+    let time = $state('06:00');
+    let initialized = false;
+    async function refresh(action?: Record<string, unknown>) {
+        if (busy) return;
+        busy = true;
+        try {
+            const response = await fetch('/api/plutonium', action ? {
+                method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(action)
+            } : undefined);
+            const result = await response.json();
+            if (!response.ok) throw new Error(result.error ?? 'Could not load Plutonium status.');
+            data = result;
+            error = '';
+            if (!initialized) {
+                enabled = result.schedule.enabled; time = result.schedule.time; initialized = true;
+            }
+            if (action) notice = action.action === 'schedule' ? 'Schedule saved.'
+                : action.action === 'restart' ? 'Restart queued. Occupied servers will wait until empty.'
+                : action.action === 'cancel' ? 'Queued restart cancelled.' : 'Version checks finished.';
+        } catch (cause) { error = (cause as Error).message; }
+        finally { busy = false; }
+    }
+    function pending(server: string) {
+        return data?.jobs.some(j => j.server === server && ['queued', 'restarting'].includes(j.state));
+    }
+    onMount(() => {
+        void refresh();
+        const timer = setInterval(() => { if (!document.hidden) void refresh(); }, 5000);
+        return () => clearInterval(timer);
+    });
+</script>
+
+<section id="plutonium" aria-labelledby="plutonium-heading">
+    <div class="heading"><h2 id="plutonium-heading">Plutonium updates & restarts</h2>
+        <Button disabled={busy} onclick={() => refresh({ action: 'check' })}>Check versions</Button>
+    </div>
+    <p>Plutainer applies updates on startup when auto-update is enabled. Restarts wait for a fresh player count of zero and share the fleet’s rolling restart limit.</p>
+    {#if error}<p class="ui-error" role="alert">{error}</p>{/if}
+    {#if notice}<p role="status">{notice}</p>{/if}
+    {#if data}
+        <p>Latest published revision: <strong>{data.release.revision ? `r${data.release.revision}` : 'Unknown'}</strong>
+            · <a href="https://plutonium.pw/docs/changelog/" target="_blank" rel="noreferrer">Changelog</a>
+            {#if data.release.checkedAt} · Checked {$timeFormat.dateTime(data.release.checkedAt)}{/if}
+        </p>
+        {#if data.release.error}<p role="alert">Update check failed: {data.release.error} Any displayed release is cached.</p>{/if}
+        <form onsubmit={(event) => { event.preventDefault(); void refresh({ action: 'schedule', enabled, time }); }}>
+            <label class="toggle"><input type="checkbox" bind:checked={enabled} /> Daily fleet restart</label>
+            <label>Queue time (UTC)<input type="time" bind:value={time} required /></label>
+            <Button type="submit" disabled={busy}>Save schedule</Button>
+        </form>
+        <p>This restarts all fleet servers daily, even without a new release. Busy servers may restart later. Disabling the schedule stops future runs; cancel existing queued restarts below.</p>
+        {#if data.schedule.nextAt}<p>Next run: {$timeFormat.dateTime(data.schedule.nextAt)}</p>{/if}
+        <div class="table-wrap"><table>
+            <thead><tr><th>Server</th><th>Running revision</th><th>Update status</th><th>Action</th></tr></thead>
+            <tbody>{#each data.servers as server (server.server)}
+                <tr><td><a href={`/servers/${encodeURIComponent(server.server)}`}>{server.server}</a></td>
+                    <td>{server.revision ? `r${server.revision}` : 'Unknown'}
+                        {#if server.checked_at}<small>Checked {$timeFormat.dateTime(server.checked_at)}</small>{/if}
+                        {#if server.error}<small>{server.error}</small>{/if}
+                    </td>
+                    <td>{server.revision === null || data.release.revision === null || data.release.error ? 'Unverified'
+                        : server.revision < data.release.revision ? 'Update available' : server.revision > data.release.revision ? 'Ahead of changelog' : 'Up to date'}</td>
+                    <td><Button size="sm" disabled={busy || pending(server.server)} onclick={() => refresh({ action: 'restart', servers: [server.server] })}>
+                        {pending(server.server) ? 'Restart pending' : 'Queue restart'}
+                    </Button></td></tr>
+            {:else}<tr><td colspan="4">No fleet servers yet.</td></tr>{/each}</tbody>
+        </table></div>
+        {#if data.servers.length}<Button disabled={busy || data.servers.every(s => pending(s.server))} onclick={() => refresh({ action: 'restart', servers: data!.servers.map(s => s.server) })}>Queue fleet restart</Button>{/if}
+        {#if data.jobs.length}
+            <h3>Restart activity</h3>
+            <ul>{#each data.jobs as job (job.id)}<li>
+                <strong>{job.server}</strong> · {job.state} · {$timeFormat.dateTime(job.requested_at)}
+                {#if job.target_revision} · Target r{job.target_revision}{/if}
+                {#if job.detail}<small>{job.detail}</small>{/if}
+                {#if job.state === 'queued'}<Button size="sm" disabled={busy} onclick={() => refresh({ action: 'cancel', id: job.id })}>Cancel</Button>{/if}
+            </li>{/each}</ul>
+        {/if}
+    {:else}<p role="status">{busy ? 'Loading maintenance status…' : 'Maintenance status unavailable.'}</p>{/if}
+</section>
+
+<style>
+    section { padding: 20px; border: 1px solid var(--border-strong); border-radius: var(--radius-md); background: var(--surface); scroll-margin-top: 90px; }
+    .heading, form { display: flex; flex-wrap: wrap; gap: 16px; align-items: center; justify-content: space-between; }
+    h2 { font-size: var(--text-md); margin: 0; }
+    h3 { font-size: var(--text-sm); margin-top: 24px; }
+    p, td, th, label, li { font-size: var(--text-sm); line-height: 1.6; }
+    p, small { color: var(--text-secondary); }
+    label { display: grid; gap: 6px; }
+    .toggle { display: flex; align-items: center; }
+    input[type='time'] { padding: 8px; color: var(--text-primary); background: var(--surface); border: 1px solid var(--border-strong); border-radius: var(--radius-sm); }
+    .table-wrap { overflow-x: auto; margin: 16px 0; }
+    table { width: 100%; border-collapse: collapse; text-align: left; }
+    th, td { padding: 10px; border-bottom: 1px solid var(--border-strong); }
+    small { display: block; }
+    ul { padding-left: 20px; }
+    li { margin-bottom: 12px; }
+</style>

src/routes/api/plutonium/+server.ts

@@ -0,0 +1,27 @@
+import { json } from '@sveltejs/kit';
+import { cancelMaintenance, checkPlutonium, plutoniumStatus, queueMaintenance, saveMaintenanceSchedule } from '$lib/server/fleet/plutonium.js';
+
+export function GET() { return json(plutoniumStatus()); }
+
+export async function POST({ request }: { request: Request }) {
+    let body;
+    try { body = await request.json(); } catch { return json({ error: 'Invalid JSON.' }, { status: 400 }); }
+    if (!body || typeof body !== 'object') return json({ error: 'Invalid request.' }, { status: 400 });
+    if (body.action === 'check') {
+        await checkPlutonium();
+    } else if (body.action === 'schedule') {
+        if (typeof body.enabled !== 'boolean' || typeof body.time !== 'string' || !/^([01]\d|2[0-3]):[0-5]\d$/.test(body.time)) {
+            return json({ error: 'Provide enabled and a valid UTC time (HH:mm).' }, { status: 400 });
+        }
+        saveMaintenanceSchedule(body.enabled, body.time);
+    } else if (body.action === 'restart') {
+        if (!Array.isArray(body.servers) || !body.servers.length || body.servers.some((s: unknown) => typeof s !== 'string')) {
+            return json({ error: 'Choose servers to restart.' }, { status: 400 });
+        }
+        try { queueMaintenance(body.servers); }
+        catch (err) { return json({ error: (err as Error).message }, { status: 400 }); }
+    } else if (body.action === 'cancel') {
+        if (!Number.isSafeInteger(body.id) || !cancelMaintenance(body.id)) return json({ error: 'Only queued restarts can be cancelled.' }, { status: 409 });
+    } else return json({ error: 'Unknown action.' }, { status: 400 });
+    return json(plutoniumStatus());
+}

src/routes/settings/+page.svelte

@@ -6,6 +6,7 @@
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
     import Badge from '$ui/components/primitives/Badge.svelte';
+    import PlutoniumMaintenance from '$ui/components/PlutoniumMaintenance.svelte';
     let data = $state<SystemStatus | null>(null);
     let loading = $state(false);
     let error = $state('');
@@ -38,6 +39,7 @@
         {#if data}<p>Example: {$timeFormat.dateTime(data.checkedAt)}</p>{/if}
         {#if $timeZonePersistenceError}<p role="status">Browser storage is unavailable. Your timezone applies for this visit but cannot be saved.</p>{/if}
     </section>
+    <PlutoniumMaintenance />
     <div class="heading">
         <div><h2>System status</h2><p>Check your connections and the prerequisites for a playable server.</p></div>
         <Button loading={loading} onclick={() => refresh(true)}>Check connections</Button>

tests/unit/plutonium.test.ts

@@ -0,0 +1,151 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { nextMaintenanceAt, parseChangelogRevision, parsePlutoniumRevision } from '../../src/lib/plutonium.js';
+
+vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() }));
+vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
+
+let fixture: Fixture;
+const servers = ['t5_zombie_theater_1', 't6_zombie_town_1'];
+beforeEach(async () => {
+    vi.resetModules(); vi.resetAllMocks();
+    vi.useFakeTimers({ toFake: ['Date'] });
+    vi.setSystemTime(new Date('2026-09-21T05:59:00Z'));
+    fixture = await newFixture(servers);
+    vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
+    vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test');
+    vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
+    vi.stubGlobal('fetch', vi.fn().mockImplementation(async () => new Response('<h2 id="r5354">r5354</h2><h2>r5300</h2>')));
+    const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
+    await seedServerRegistry();
+    const k = await import('../../src/lib/server/integrations/komodo.js');
+    vi.mocked(k.komodoRestart).mockResolvedValue({ ok: true });
+    vi.mocked(k.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+    const r = await import('../../src/lib/server/integrations/rcon.js');
+    vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0);
+    vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
+    vi.mocked(r.rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '', status: { players: 0, map: null, gametype: null, roster: [] } });
+});
+afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.useRealTimers(); });
+const api = () => import('../../src/lib/server/fleet/plutonium.js');
+
+describe('Plutonium versions', () => {
+    it('parses explicit revisions and release headings without interpreting engine numbers or links as releases', () => {
+        expect(parsePlutoniumRevision('print\nPlutonium T6 r5354 win-x86')).toBe(5354);
+        expect(parsePlutoniumRevision('version 1.0 build 20260921')).toBeNull();
+        expect(parseChangelogRevision('<a href="#r9999">r9999</a><h2><a>r5354</a></h2>')).toBe(5354);
+        expect(() => parseChangelogRevision('<h1>Service unavailable</h1>')).toThrow();
+    });
+    it('preserves the cached release with a visible error and clears failed running-version observations', async () => {
+        const m = await api(); await m.checkPlutonium();
+        vi.mocked(fetch).mockRejectedValue(new Error('Offline'));
+        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
+        vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Timeout' });
+        await m.checkPlutonium();
+        expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: 'Offline' });
+        expect(m.plutoniumStatus().servers.every(s => s.revision === null && s.error === 'Timeout')).toBe(true);
+    });
+});
+
+describe('persistent maintenance', () => {
+    it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => {
+        const m = await api(); await m.checkPlutonium();
+        expect(m.queueMaintenance([servers[0]])).toBe(1);
+        expect(m.queueMaintenance([servers[0]])).toBe(0);
+        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null);
+        await m.maintenanceTick(); await m.maintenanceTick();
+        expect(komodoRestart).not.toHaveBeenCalled();
+        await Promise.all([m.maintenanceTick(), m.maintenanceTick()]);
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+        expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
+        await m.maintenanceTick();
+        expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
+        vi.setSystemTime(Date.now() + 15_000);
+        await m.maintenanceTick();
+        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' });
+    });
+    it('shares rolling capacity across servers and survives closing the database', async () => {
+        const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers);
+        await m.maintenanceTick();
+        const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
+        expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']);
+        vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick();
+        expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']);
+    });
+    it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => {
+        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
+        await m.maintenanceTick();
+        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
+        vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
+        vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick(); await m.maintenanceTick();
+        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 });
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+    });
+    it('does not retry an ambiguous restart timeout', async () => {
+        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' });
+        await m.maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick();
+        expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+    });
+    it('honors cancellation arriving during the player check', async () => {
+        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
+        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+        vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
+            m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0;
+        });
+        await m.maintenanceTick();
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        expect(komodoRestart).not.toHaveBeenCalled();
+        expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
+    });
+    it('holds maintenance while lifecycle work is queued', async () => {
+        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
+            VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString());
+        await m.maintenanceTick();
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        expect(komodoRestart).not.toHaveBeenCalled();
+        expect(m.plutoniumStatus().jobs[0].detail).toContain('deployment or beta');
+    });
+    it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => {
+        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run();
+        await m.maintenanceTick();
+        expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        expect(komodoRestart).not.toHaveBeenCalled();
+    });
+    it('validates API requests without changing the schedule or queue', async () => {
+        const { POST } = await import('../../src/routes/api/plutonium/+server.js');
+        const post = (body: unknown) => POST({ request: new Request('http://localhost/api/plutonium', {
+            method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body)
+        }) });
+        expect((await post({ action: 'schedule', enabled: true, time: '26:99' })).status).toBe(400);
+        expect((await post({ action: 'restart', servers: ['missing'] })).status).toBe(400);
+        expect((await post({ action: 'cancel', id: 100 })).status).toBe(409);
+        expect((await post({ action: 'schedule', enabled: true, time: '06:00' })).status).toBe(200);
+        const m = await api();
+        expect(m.plutoniumStatus().jobs).toEqual([]);
+        expect(m.plutoniumStatus().schedule.enabled).toBe(true);
+    });
+    it('runs a daily UTC schedule once and coalesces missed days', async () => {
+        expect(nextMaintenanceAt('06:00', new Date('2026-09-21T06:00:00Z'))).toBe('2026-09-22T06:00:00.000Z');
+        expect(() => nextMaintenanceAt('24:00')).toThrow();
+        const m = await api(); await m.checkPlutonium();
+        expect(m.plutoniumStatus().schedule.enabled).toBe(false);
+        m.saveMaintenanceSchedule(true, '06:00');
+        vi.setSystemTime(new Date('2026-09-24T07:00:00Z'));
+        await m.maintenanceTick(); await m.maintenanceTick();
+        expect(m.plutoniumStatus().jobs).toHaveLength(2);
+        expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
+        m.saveMaintenanceSchedule(false, '06:00');
+        expect(m.plutoniumStatus().schedule.nextAt).toBeNull();
+    });
+});