Fix transient health checks and avoid unnecessary recovery

src/lib/server/integrations/komodo.ts

@@ -320,7 +320,7 @@ const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
 async function fetchJsonWithSnippet(
     path: string,
     body: Record<string, string>
-): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string }> {
+): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string; unsupportedEndpoint?: boolean }> {
     const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
     let res: Response;
     try {
@@ -330,7 +330,11 @@ async function fetchJsonWithSnippet(
     }
     const text = await res.text().catch(() => '');
     if (!res.ok) {
-        return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text) };
+        const unsupportedEndpoint = res.status === 404 || (
+            [400, 405, 422, 500, 501].includes(res.status)
+            && /\b(?:unknown|unsupported|unrecognized)\s+(?:method|endpoint|variant)\b|\bmethod not found\b/i.test(text)
+        );
+        return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text), unsupportedEndpoint };
     }
     let json: unknown;
     try {
@@ -382,9 +386,10 @@ export async function komodoInspectHealthy(serverName: string): Promise<KomodoHe
     for (const { path, body } of attempts) {
         const r = await fetchJsonWithSnippet(path, body);
         if (!r.ok) {
+            // Alternate endpoints handle API-version differences. Retrying an
+            // outage against every endpoint delays all other server probes.
+            if (!r.unsupportedEndpoint) return { ok: false, reason: r.reason, snippet: r.snippet };
             lastSnippet = r.snippet ?? lastSnippet;
-            // Try the next endpoint on 404s or unknown-method-shaped errors; a
-            // real error (auth, timeout) will fail every attempt anyway.
             continue;
         }
         const state = extractContainerState(r.json);

src/lib/server/integrations/rcon.ts

@@ -2,6 +2,7 @@ import { createSocket } from 'node:dgram';
 import type { Socket } from 'node:dgram';
 import { config, komodoConfigured, rconConfigured } from '../config.js';
 import { getServer, setServerField } from '../fleet/servers.js';
+import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
 import { komodoResolveGamePort } from './komodo.js';
 import { parsePlutoniumRevision } from '../../plutonium.js';
 
@@ -300,13 +301,8 @@ export function parsePlayerRoster(text: string): RconPlayer[] {
 }
 
 export async function rconGetPlayerCount(server: string): Promise<number | null> {
-    const res = await endpointFor(server);
-    if (!res.endpoint) return null;
-    const send = await sendRcon(res.endpoint.host, res.endpoint.port, 'status');
-    if (!send.ok) return null;
-    const text = stripPrefix(send.buf);
-    if (!text) return null;
-    return parsePlayerCount(text);
+    const result = await rconStatusResult(server);
+    return result.ok ? result.status.players : null;
 }
 
 export interface RconStatus {
@@ -338,6 +334,18 @@ export type RconStatusResult =
       };
 
 const RAW_REPLY_LIMIT = 512;
+const statusInFlight = new Map<string, Promise<RconStatusResult>>();
+
+/** Retry only this read-only query: commands with effects must never be replayed. */
+async function sendStatus(endpoint: RconEndpoint): Promise<SendResult> {
+    for (let attempt = 0; ; attempt++) {
+        const result = await sendRcon(endpoint.host, endpoint.port, 'status');
+        if (result.ok || (result.reason !== 'timeout' && result.reason !== 'send-error') || attempt === 2) {
+            return result;
+        }
+        await new Promise<void>(resolve => setTimeout(resolve, 250 * 2 ** attempt));
+    }
+}
 
 /**
  * Render a Buffer as a short hex + printable-ASCII snippet — same layout as
@@ -365,6 +373,10 @@ export function formatRawReply(buf: Buffer): string {
  * rather than just "unreachable".
  */
 export async function rconStatusResult(server: string): Promise<RconStatusResult> {
+    // Snapshot before any network work. A probe begun after a restart or a
+    // replacement registration must not join one observing the old process.
+    const createdAt = getServer(server)?.created_at;
+    const epoch = createdAt ? latestGameSubmission(server) : null;
     const res = await endpointFor(server);
     if (!res.endpoint) {
         if (res.reason === 'unknown-server') {
@@ -387,13 +399,22 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
     }
 
     const ep = res.endpoint;
+    const key = JSON.stringify([server, createdAt, epoch, ep.host, ep.port]);
+    const pending = statusInFlight.get(key);
+    if (pending) return pending;
+    const request = probeStatus(ep).finally(() => { statusInFlight.delete(key); });
+    statusInFlight.set(key, request);
+    return request;
+}
+
+async function probeStatus(ep: RconEndpoint): Promise<RconStatusResult> {
     // Plutonium T5 doesn't recognize `getstatus` as an rcon subcommand
     // ("Unknown command"), so we use `status` — a print-body response with
     // a header table plus one row per player. Q3-derived engines that
     // *do* answer statusResponse (via anonymous `getstatus` query) would
     // give richer info-line dvars, but every engine we care about supports
     // `status` under the rcon wrapper.
-    const send = await sendRcon(ep.host, ep.port, 'status');
+    const send = await sendStatus(ep);
     if (!send.ok) {
         return { ok: false, endpoint: ep, reason: send.reason, detail: send.detail };
     }