feat: unify recovery, updates, and schedules under Maintenance

src/routes/api/maintenance/+server.ts

@@ -0,0 +1,47 @@
+import { json } from '@sveltejs/kit';
+import { maintenanceOverview } from '$lib/server/maintenance/index.js';
+import { pauseMaintenance, setMaintenancePolicy } from '$lib/server/maintenance/policies.js';
+import { watchdogTick } from '$lib/server/maintenance/recovery.js';
+import { cancelMaintenance, checkPlutonium, queueUpdates, retryMaintenance, saveMaintenanceSchedule } from '$lib/server/maintenance/runtime.js';
+import { validTimeZone } from '$lib/time-zone.js';
+import { kick } from '$lib/server/pipeline/worker.js';
+
+export function GET() { return json(maintenanceOverview()); }
+export async function POST({ request }: { request: Request }) {
+    const body = await request.json().catch(() => null);
+    if (!body || typeof body !== 'object' || Array.isArray(body)) return json({ error: 'Invalid request.' }, { status: 400 });
+    let notice = '';
+    try {
+        switch (body.action) {
+            case 'pause':
+                if (typeof body.paused !== 'boolean') throw new Error('Provide a paused boolean.');
+                pauseMaintenance(body.paused);
+                notice = body.paused ? 'Maintenance paused. Submitted operations continue recovery.' : 'Maintenance resumed.';
+                break;
+            case 'policy':
+                if (!['health', 'drift', 'updates'].includes(body.policy) || typeof body.enabled !== 'boolean') throw new Error('Choose a policy and an enabled boolean.');
+                setMaintenancePolicy(body.policy, body.enabled); notice = 'Policy saved.';
+                break;
+            case 'scan':
+                await Promise.all([watchdogTick(true), checkPlutonium()]); notice = 'Checks finished.';
+                break;
+            case 'update':
+                if (body.servers !== undefined && (!Array.isArray(body.servers) || !body.servers.length || body.servers.some((v: unknown) => typeof v !== 'string'))) throw new Error('Choose existing fleet servers.');
+                notice = `${queueUpdates(body.servers)} update restart(s) queued. Occupied servers wait until empty.`;
+                break;
+            case 'schedule':
+                if (typeof body.enabled !== 'boolean' || typeof body.time !== 'string' || !validTimeZone(body.timeZone)) throw new Error('Provide enabled, a time, and a valid timezone.');
+                saveMaintenanceSchedule(body.enabled, body.time, body.timeZone); notice = 'Schedule saved. Existing requests are retained.';
+                break;
+            case 'cancel':
+                if (!Number.isSafeInteger(body.id) || !cancelMaintenance(body.id)) return json({ error: 'Only unsubmitted restarts can be cancelled.' }, { status: 409 });
+                notice = 'Queued restart cancelled.'; break;
+            case 'retry':
+                if (!Number.isSafeInteger(body.id)) throw new Error('Choose a maintenance activity.');
+                notice = `${retryMaintenance(body.id)} restart(s) queued for verification.`; break;
+            default: throw new Error('Unknown maintenance action.');
+        }
+    } catch (error) { return json({ error: (error as Error).message }, { status: 400 }); }
+    void kick().catch(error => console.error('[maintenance] worker kick failed:', error));
+    return json({ ...maintenanceOverview(), notice });
+}

src/routes/api/maintenance/[id]/+server.ts

@@ -0,0 +1,16 @@
+import { json } from '@sveltejs/kit';
+import { watchdogDetail, updateWatchdogIncident } from '$lib/server/maintenance/recovery.js';
+import { kick } from '$lib/server/pipeline/worker.js';
+
+export function GET({ params }: { params: { id: string } }) {
+    const detail = watchdogDetail(params.id);
+    return detail ? json(detail) : json({ error: 'Maintenance finding not found.' }, { status: 404 });
+}
+export async function POST({ params, request }: { params: { id: string }; request: Request }) {
+    const body = await request.json().catch(() => null);
+    if (!body || !['retry','dismiss'].includes(body.action)) return json({ error: 'Choose retry or dismiss.' }, { status: 400 });
+    const result = updateWatchdogIncident(params.id, body.action);
+    if (!result.ok) return json({ error: result.error }, { status: result.status });
+    void kick().catch(err => console.error('[maintenance] worker kick failed:', err));
+    return json(watchdogDetail(params.id));
+}

src/routes/api/servers/+server.ts

@@ -6,12 +6,15 @@ import { serverObservation } from '$lib/server/fleet/status.js';
 import { queueCreateServer } from '$lib/server/provisioning/index.js';
 import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
 import type { GameMode, ZombieGame } from '$lib/server/fleet/map-aliases.js';
+import { runningVersions } from '$lib/server/integrations/plutonium.js';
 
 export async function GET() {
     const rconHost = config.rcon.host;
+    const versions = new Map(runningVersions().map(version => [version.server, version]));
     const servers = listServers().map((s) => {
         return {
             ...serverObservation(s),
+            plutonium: versions.get(s.name),
             // Derived endpoint — same logic as endpointFor() but without asking
             // Komodo. If we don't have a port yet, endpoint is null.
             endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,