Surface structured RCON failure reasons in the fleet UI

rconGetStatus previously collapsed at least five distinct failure modes
(no password, no port, socket error, timeout, bad reply) into a single
null return, so the UI could only say "rcon unreachable" — useless when
diagnosing a real Alcatraz outage.

Refactored to rconStatusResult() returning a discriminated union with a
RconFailure reason and human-facing detail. The worker persists these as
"<reason>:<detail>" in last_error (no schema change). /api/servers now
returns the derived endpoint (host:port) per server plus a top-level
integrations block (rcon/komodo configured, timeout, default host).

Server detail page gains an RCON diagnostic panel when health is not
online: shows the exact endpoint being dialed, integration status,
timeout, a color-coded reason tag, the detail, and a targeted fix hint.
Wrong-password replies are detected via "bad rcon" text in the print
body.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

src/lib/server/integrations/rcon.ts

@@ -1,6 +1,6 @@
 import { createSocket } from 'node:dgram';
 import type { Socket } from 'node:dgram';
-import { config, rconConfigured } from '../config.js';
+import { config, komodoConfigured, rconConfigured } from '../config.js';
 import { getServer, setServerField } from '../fleet/servers.js';
 import { komodoResolveGamePort } from './komodo.js';
 
@@ -16,18 +16,33 @@ import { komodoResolveGamePort } from './komodo.js';
 
 const PREFIX = Buffer.from([0xff, 0xff, 0xff, 0xff]);
 
-async function sendRcon(host: string, port: number, command: string): Promise<Buffer | null> {
-    if (!rconConfigured()) return null;
+// Distinct RCON failure modes. Surfaced to the UI so the operator can tell
+// "wrong password" from "wrong host" from "Plutonium isn't listening".
+export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
+
+export interface RconEndpoint {
+    host: string;
+    port: number;
+}
+
+type SendResult =
+    | { ok: true; buf: Buffer }
+    | { ok: false; reason: 'no-config' | 'send-error' | 'timeout'; detail: string };
+
+async function sendRcon(host: string, port: number, command: string): Promise<SendResult> {
+    if (!rconConfigured()) {
+        return { ok: false, reason: 'no-config', detail: 'RCON_PASSWORD is not set' };
+    }
     const payload = Buffer.concat([
         PREFIX,
         Buffer.from(`rcon ${config.rcon.password} ${command}\n`, 'utf8')
     ]);
 
-    return new Promise<Buffer | null>((resolve) => {
+    return new Promise<SendResult>((resolve) => {
         const sock: Socket = createSocket('udp4');
         let settled = false;
 
-        const finish = (val: Buffer | null): void => {
+        const finish = (val: SendResult): void => {
             if (settled) return;
             settled = true;
             try {
@@ -38,41 +53,52 @@ async function sendRcon(host: string, port: number, command: string): Promise<Bu
             resolve(val);
         };
 
-        const timer = setTimeout(() => finish(null), config.rcon.timeoutMs);
+        const timer = setTimeout(
+            () => finish({ ok: false, reason: 'timeout', detail: `no reply within ${config.rcon.timeoutMs}ms` }),
+            config.rcon.timeoutMs
+        );
 
         sock.on('message', (msg) => {
             clearTimeout(timer);
-            finish(msg);
+            finish({ ok: true, buf: msg });
         });
-        sock.on('error', () => {
+        sock.on('error', (err) => {
             clearTimeout(timer);
-            finish(null);
+            finish({ ok: false, reason: 'send-error', detail: err.message });
         });
 
         sock.send(payload, 0, payload.length, port, host, (err) => {
             if (err) {
                 clearTimeout(timer);
-                finish(null);
+                finish({ ok: false, reason: 'send-error', detail: err.message });
             }
         });
     });
 }
 
+interface EndpointResolution {
+    endpoint: RconEndpoint | null;
+    // Present when endpoint is null — tells the caller why we couldn't dial.
+    reason?: 'unknown-server' | 'no-port-no-komodo' | 'no-port-komodo-empty';
+}
+
 /**
  * Resolve the RCON endpoint (host + port) for a server, filling
- * `servers.game_port` from Komodo on first miss. Returns null when we
- * still can't determine a port after asking Komodo.
+ * `servers.game_port` from Komodo on first miss.
  */
-async function endpointFor(server: string): Promise<{ host: string; port: number } | null> {
+async function endpointFor(server: string): Promise<EndpointResolution> {
     const row = getServer(server);
-    if (!row) return null;
+    if (!row) return { endpoint: null, reason: 'unknown-server' };
     let port = row.game_port;
     if (!port) {
+        if (!komodoConfigured()) {
+            return { endpoint: null, reason: 'no-port-no-komodo' };
+        }
         port = await komodoResolveGamePort(server);
         if (port) setServerField(server, 'game_port', port);
     }
-    if (!port) return null;
-    return { host: row.rcon_host ?? config.rcon.host, port };
+    if (!port) return { endpoint: null, reason: 'no-port-komodo-empty' };
+    return { endpoint: { host: row.rcon_host ?? config.rcon.host, port } };
 }
 
 function stripPrefix(buf: Buffer): string | null {
@@ -187,11 +213,11 @@ export function parsePlayerRoster(text: string): RconPlayer[] {
 }
 
 export async function rconGetPlayerCount(server: string): Promise<number | null> {
-    const ep = await endpointFor(server);
-    if (!ep) return null;
-    const buf = await sendRcon(ep.host, ep.port, 'status');
-    if (!buf) return null;
-    const text = stripPrefix(buf);
+    const res = await endpointFor(server);
+    if (!res.endpoint) return null;
+    const send = await sendRcon(res.endpoint.host, res.endpoint.port, 'status');
+    if (!send.ok) return null;
+    const text = stripPrefix(send.buf);
     if (!text) return null;
     return parsePlayerCount(text);
 }
@@ -203,22 +229,73 @@ export interface RconStatus {
     roster: RconPlayer[];
 }
 
-export async function rconGetStatus(server: string): Promise<RconStatus | null> {
-    const ep = await endpointFor(server);
-    if (!ep) return null;
+export type RconStatusResult =
+    | { ok: true; endpoint: RconEndpoint; status: RconStatus }
+    | { ok: false; endpoint: RconEndpoint | null; reason: RconFailure; detail: string };
+
+/**
+ * Full RCON status probe with structured diagnostics. Callers get either
+ * (ok, endpoint, status) or (ok:false, endpoint?, reason, detail). This is
+ * the shape the fleet UI consumes to explain *why* a server is unreachable
+ * rather than just "unreachable".
+ */
+export async function rconStatusResult(server: string): Promise<RconStatusResult> {
+    const res = await endpointFor(server);
+    if (!res.endpoint) {
+        if (res.reason === 'unknown-server') {
+            return { ok: false, endpoint: null, reason: 'no-port', detail: `unknown server: ${server}` };
+        }
+        if (res.reason === 'no-port-no-komodo') {
+            return {
+                ok: false,
+                endpoint: null,
+                reason: 'no-port',
+                detail: 'no game_port set and Komodo is not configured'
+            };
+        }
+        return {
+            ok: false,
+            endpoint: null,
+            reason: 'no-port',
+            detail: 'no game_port — Komodo did not return one for this server'
+        };
+    }
+
+    const ep = res.endpoint;
     // `getstatus` is anonymous on most CoD builds but Plutonium requires the
     // authenticated rcon route for reliable results, so we ask via rcon.
-    const buf = await sendRcon(ep.host, ep.port, 'getstatus');
-    if (!buf) return null;
-    const text = stripPrefix(buf);
-    if (!text) return null;
+    const send = await sendRcon(ep.host, ep.port, 'getstatus');
+    if (!send.ok) {
+        return { ok: false, endpoint: ep, reason: send.reason, detail: send.detail };
+    }
+
+    const text = stripPrefix(send.buf);
+    if (!text) {
+        return {
+            ok: false,
+            endpoint: ep,
+            reason: 'bad-reply',
+            detail: 'reply missing 0xff prefix — wrong port or non-Plutonium listener?'
+        };
+    }
     const info = parseInfoLine(text);
     const players = parsePlayerCount(text);
-    if (players === null) return null;
+    if (players === null) {
+        // Reply arrived but neither the print separator nor an info line
+        // showed up — usually a wrong-password `print` body ("Bad rcon…").
+        const detail = text.trim().startsWith('print') && /bad\s+rcon/i.test(text)
+            ? 'wrong RCON password'
+            : `unparseable reply (${text.length} bytes)`;
+        return { ok: false, endpoint: ep, reason: 'bad-reply', detail };
+    }
     return {
-        players,
-        map: info.mapname ?? info.sv_mapname ?? null,
-        gametype: info.g_gametype ?? null,
-        roster: parsePlayerRoster(text)
+        ok: true,
+        endpoint: ep,
+        status: {
+            players,
+            map: info.mapname ?? info.sv_mapname ?? null,
+            gametype: info.g_gametype ?? null,
+            roster: parsePlayerRoster(text)
+        }
     };
 }

src/lib/server/pipeline/bus.ts

@@ -1,5 +1,5 @@
 import { EventEmitter } from 'node:events';
-import type { RconPlayer } from '../integrations/rcon.js';
+import type { RconEndpoint, RconFailure, RconPlayer } from '../integrations/rcon.js';
 import type { Health } from '../fleet/health.js';
 
 // One in-process event bus, multiplexed by the SSE endpoints. Kept separate
@@ -15,6 +15,8 @@ export interface ServerStatusEntry {
     health: Health;
     lastSeenAt: string | null;
     lastError: string | null;
+    endpoint: RconEndpoint | null;
+    reason: RconFailure | null;
 }
 
 export type PipelineEvent =

src/lib/server/pipeline/worker.ts

@@ -1,7 +1,8 @@
 import { sweep } from './sweep.js';
 import { emit } from './bus.js';
 import type { ServerStatusEntry } from './bus.js';
-import { rconGetStatus } from '../integrations/rcon.js';
+import { rconStatusResult } from '../integrations/rcon.js';
+import type { RconFailure } from '../integrations/rcon.js';
 import { listServers, seedServerRegistry, setServerField } from '../fleet/servers.js';
 import { classifyHealth } from '../fleet/health.js';
 
@@ -23,6 +24,15 @@ async function tick(): Promise<void> {
     }
 }
 
+/**
+ * Encode a structured RCON failure as `"reason:detail"` so `last_error`
+ * stays a single column but the UI can split it apart when rendering the
+ * diagnostic panel. Reasons are stable; details are human-facing.
+ */
+function encodeError(reason: RconFailure, detail: string): string {
+    return `${reason}:${detail}`;
+}
+
 async function fanoutStatus(): Promise<void> {
     const servers = listServers();
     if (servers.length === 0) return;
@@ -35,43 +45,52 @@ async function fanoutStatus(): Promise<void> {
             const i = cursor++;
             if (i >= servers.length) return;
             const srv = servers[i];
-            const status = await rconGetStatus(srv.name).catch(() => null);
+            const result = await rconStatusResult(srv.name).catch((err) => ({
+                ok: false as const,
+                endpoint: null,
+                reason: 'send-error' as RconFailure,
+                detail: (err as Error).message
+            }));
             const now = new Date().toISOString();
 
-            if (status) {
+            if (result.ok) {
                 // Persist last-observed state. last_seen_at is the health anchor.
-                setServerField(srv.name, 'last_players', status.players);
-                setServerField(srv.name, 'last_map', status.map);
-                setServerField(srv.name, 'last_gametype', status.gametype);
-                setServerField(srv.name, 'last_roster', JSON.stringify(status.roster));
+                setServerField(srv.name, 'last_players', result.status.players);
+                setServerField(srv.name, 'last_map', result.status.map);
+                setServerField(srv.name, 'last_gametype', result.status.gametype);
+                setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
                 setServerField(srv.name, 'last_seen_at', now);
                 setServerField(srv.name, 'last_checked_at', now);
                 setServerField(srv.name, 'last_error', null);
                 entries[i] = {
                     server: srv.name,
-                    players: status.players,
-                    map: status.map,
-                    gametype: status.gametype,
-                    roster: status.roster,
+                    players: result.status.players,
+                    map: result.status.map,
+                    gametype: result.status.gametype,
+                    roster: result.status.roster,
                     health: classifyHealth(now, null),
                     lastSeenAt: now,
-                    lastError: null
+                    lastError: null,
+                    endpoint: result.endpoint,
+                    reason: null
                 };
             } else {
                 // RCON unreachable — leave last_seen_at alone (health decays
-                // to offline naturally), record the failure reason.
-                const error = 'rcon unreachable';
+                // to offline naturally), record the structured failure.
+                const encoded = encodeError(result.reason, result.detail);
                 setServerField(srv.name, 'last_checked_at', now);
-                setServerField(srv.name, 'last_error', error);
+                setServerField(srv.name, 'last_error', encoded);
                 entries[i] = {
                     server: srv.name,
                     players: srv.last_players,
                     map: srv.last_map,
                     gametype: srv.last_gametype,
                     roster: srv.last_roster ? safeJson(srv.last_roster) : null,
-                    health: classifyHealth(srv.last_seen_at, error),
+                    health: classifyHealth(srv.last_seen_at, encoded),
                     lastSeenAt: srv.last_seen_at,
-                    lastError: error
+                    lastError: encoded,
+                    endpoint: result.endpoint,
+                    reason: result.reason
                 };
             }
         }

src/lib/ui/rconError.ts

@@ -0,0 +1,43 @@
+import type { RconFailure } from '$ui/stores/servers.js';
+
+export interface DecodedRconError {
+    reason: RconFailure;
+    detail: string;
+}
+
+const KNOWN: RconFailure[] = ['no-config', 'no-port', 'send-error', 'timeout', 'bad-reply', 'no-players'];
+
+/**
+ * Decode the "<reason>:<detail>" string the worker persists into
+ * `servers.last_error`. Returns null for empty/legacy strings so the UI can
+ * fall back to a generic message.
+ */
+export function decodeRconError(raw: string | null | undefined): DecodedRconError | null {
+    if (!raw) return null;
+    const idx = raw.indexOf(':');
+    if (idx <= 0) return null;
+    const reason = raw.slice(0, idx);
+    if (!KNOWN.includes(reason as RconFailure)) return null;
+    return { reason: reason as RconFailure, detail: raw.slice(idx + 1).trim() };
+}
+
+/**
+ * Human-facing hint about how to fix each RCON failure. Kept short — the
+ * detail already carries the specifics (host, port, byte counts, etc.).
+ */
+export function rconFixHint(reason: RconFailure): string {
+    switch (reason) {
+        case 'no-config':
+            return 'Set RCON_PASSWORD in the container env and restart.';
+        case 'no-port':
+            return 'Set game_port on the server row, or configure Komodo so it can resolve the port.';
+        case 'send-error':
+            return 'DNS or route problem — check the RCON host is reachable from this container.';
+        case 'timeout':
+            return 'Plutonium is not replying on the resolved UDP port. Verify the server is running and the port matches.';
+        case 'bad-reply':
+            return 'Something is listening but not speaking Plutonium RCON. Check the port and password.';
+        case 'no-players':
+            return 'Reply parsed but no player table was found — likely a server-side dialect mismatch.';
+    }
+}

src/lib/ui/stores/servers.ts

@@ -10,6 +10,12 @@ export interface RosterPlayer {
 }
 
 export type Health = 'online' | 'stale' | 'offline';
+export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
+
+export interface RconEndpoint {
+    host: string;
+    port: number;
+}
 
 export interface ServerStatus {
     server: string;
@@ -20,6 +26,8 @@ export interface ServerStatus {
     health: Health;
     lastSeenAt: string | null;
     lastError: string | null;
+    endpoint: RconEndpoint | null;
+    reason: RconFailure | null;
 }
 
 export const statusByServer = writable<Map<string, ServerStatus>>(new Map());