Widen Komodo health probe: try container endpoints, snippet on failure

GetDeployment returns config, not runtime state, so the health check was
always coming back with 'no container state found in response'. For
deployment-type servers the probe now tries GetDeploymentContainer,
InspectDeploymentContainer, and GetDeploymentStatus first, falling back
to GetDeployment only if those aren't available. The extractor also
recognizes more shapes (Health.Status, container_state, deployment_state,
etc.) and accepts more healthy strings (running, healthy, ok, started,
up).

When nothing matches, the truncated raw response snippet rides along in
the error field so the deployment/beta UI shows exactly which field to
teach the extractor next time.

Currently-stuck deployment_server rows are still 'pending', so the sweep
will retry them on its next 15s tick — no manual reset needed. Once one
of the widened endpoints resolves, those rows advance to 'done'
automatically.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

src/lib/server/integrations/komodo.ts

@@ -255,14 +255,32 @@ export async function komodoResolveGamePort(serverName: string): Promise<number
 
 export type KomodoHealthResult =
     | { ok: true; state: string }
-    | { ok: false; reason: string };
+    | { ok: false; reason: string; snippet?: string };
+
+// Shape-agnostic scan for the container's running-state. Walks the entire
+// payload looking for anything that resembles a state field. Covered shapes:
+//   - Docker inspect:  { State: { Status: "running", Running: true, Health: { Status: "healthy" } } }
+//   - Komodo GetDeployment: top-level or nested `state`, `Status`, `container_state`
+//   - Komodo GetDeploymentStatus: { state: "Running" }
+//   - Anything with `Running: true` at any depth
+// When it can't find any of these, callers get the raw JSON in the error so
+// we can spot which field to add here.
+const STATE_KEYS = new Set([
+    'state',
+    'State',
+    'status',
+    'Status',
+    'container_state',
+    'containerState',
+    'deployment_state',
+    'deploymentState',
+    'docker_status'
+]);
 
-// Walks a Komodo read response looking for a container state string, matching
-// the shapes we've seen: docker-inspect ({ State: { Status, Running } }),
-// GetDeployment ({ state } / { status: { state } }), and GetStack similar.
 function extractContainerState(payload: unknown): string | null {
     const seen = new Set<unknown>();
     let found: string | null = null;
+
     const walk = (v: unknown): void => {
         if (found || v === null || typeof v !== 'object' || seen.has(v)) return;
         seen.add(v);
@@ -271,61 +289,43 @@ function extractContainerState(payload: unknown): string | null {
             return;
         }
         const obj = v as Record<string, unknown>;
-        // Prefer the explicit docker inspect shape.
+
-        const s = obj['State'];
+        // Docker inspect nested: { State: { Status, Running, Health: { Status } } }
-        if (s && typeof s === 'object') {
+        const stateObj = obj['State'];
-            const status = (s as Record<string, unknown>)['Status'];
+        if (stateObj && typeof stateObj === 'object') {
-            const running = (s as Record<string, unknown>)['Running'];
+            const so = stateObj as Record<string, unknown>;
-            if (typeof status === 'string') {
+            const status = so['Status'];
-                found = status;
+            const running = so['Running'];
-                return;
+            const health = so['Health'];
+            if (typeof status === 'string') { found = status; return; }
+            if (running === true) { found = 'running'; return; }
+            if (health && typeof health === 'object') {
+                const hs = (health as Record<string, unknown>)['Status'];
+                if (typeof hs === 'string') { found = hs; return; }
             }
-            if (running === true) {
+        }
-                found = 'running';
+
+        // Flat string-valued state under any of the recognized keys.
+        for (const key of STATE_KEYS) {
+            const val = obj[key];
+            if (typeof val === 'string' && val.length > 0) {
+                found = val;
                 return;
             }
         }
-        if (typeof obj['state'] === 'string') {
+
-            found = obj['state'] as string;
-            return;
-        }
         for (const val of Object.values(obj)) walk(val);
     };
     walk(payload);
     return found;
 }
 
-/**
+const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
- * Ask Komodo whether the server's container is currently running. Used by the
- * beta runner during the `awaiting-health` stage after a restart. A truthy
- * `ok` doesn't imply the game is fully ready — the runner also gates on RCON
- * before advancing.
- */
-export async function komodoInspectHealthy(serverName: string): Promise<KomodoHealthResult> {
-    if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' };
-    const row = getServer(serverName);
-    if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' };
-
-    const type = row.komodo_type ?? 'deployment';
-    let path: string;
-    let body: Record<string, string>;
-    switch (type) {
-        case 'stack':
-            path = 'read/GetStack';
-            body = { stack: row.komodo_name };
-            break;
-        case 'container':
-            if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' };
-            path = 'read/InspectContainer';
-            body = { server: row.komodo_server, container: row.komodo_name };
-            break;
-        case 'deployment':
-        default:
-            path = 'read/GetDeployment';
-            body = { deployment: row.komodo_name };
-            break;
-    }
 
+async function fetchJsonWithSnippet(
+    path: string,
+    body: Record<string, string>
+): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string }> {
     const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
     let res: Response;
     try {
@@ -333,20 +333,79 @@ export async function komodoInspectHealthy(serverName: string): Promise<KomodoHe
     } catch (err) {
         return { ok: false, reason: describeFetchError(err, url) };
     }
-    if (!res.ok) return { ok: false, reason: `HTTP ${res.status}` };
+    const text = await res.text().catch(() => '');
-
+    if (!res.ok) {
+        return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text) };
+    }
     let json: unknown;
     try {
-        json = await res.json();
+        json = JSON.parse(text);
     } catch (err) {
-        return { ok: false, reason: `unparseable json: ${(err as Error).message}` };
+        return { ok: false, reason: `unparseable json: ${(err as Error).message}`, snippet: truncate(text) };
     }
-    const state = extractContainerState(json);
+    return { ok: true, json, snippet: truncate(safeJsonStringify(json)) };
-    if (!state) return { ok: false, reason: 'no container state found in response' };
+}
-    const healthy = state.toLowerCase() === 'running' || state.toLowerCase() === 'healthy';
+
-    if (!healthy) return { ok: false, reason: `container state: ${state}` };
+/**
+ * Ask Komodo whether the server's container is currently running. Used by the
+ * sweep during the health phase (post-restart) and by the beta runner. For
+ * deployment-type servers we hit the container-inspect endpoint first because
+ * GetDeployment returns config, not runtime state — falling back only when
+ * the container endpoint isn't available.
+ */
+export async function komodoInspectHealthy(serverName: string): Promise<KomodoHealthResult> {
+    if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' };
+    const row = getServer(serverName);
+    if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' };
+
+    const type = row.komodo_type ?? 'deployment';
+    const attempts: Array<{ path: string; body: Record<string, string> }> = [];
+    switch (type) {
+        case 'stack':
+            attempts.push({ path: 'read/GetStack', body: { stack: row.komodo_name } });
+            break;
+        case 'container':
+            if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' };
+            attempts.push({
+                path: 'read/InspectContainer',
+                body: { server: row.komodo_server, container: row.komodo_name }
+            });
+            break;
+        case 'deployment':
+        default:
+            // Container-specific endpoints first — they return docker-inspect
+            // shape with State.Status. GetDeployment is the last-resort
+            // config-only fallback.
+            attempts.push({ path: 'read/GetDeploymentContainer', body: { deployment: row.komodo_name } });
+            attempts.push({ path: 'read/InspectDeploymentContainer', body: { deployment: row.komodo_name } });
+            attempts.push({ path: 'read/GetDeploymentStatus', body: { deployment: row.komodo_name } });
+            attempts.push({ path: 'read/GetDeployment', body: { deployment: row.komodo_name } });
+            break;
+    }
+
+    let lastSnippet: string | undefined;
+    for (const { path, body } of attempts) {
+        const r = await fetchJsonWithSnippet(path, body);
+        if (!r.ok) {
+            lastSnippet = r.snippet ?? lastSnippet;
+            // Try the next endpoint on 404s or unknown-method-shaped errors; a
+            // real error (auth, timeout) will fail every attempt anyway.
+            continue;
+        }
+        const state = extractContainerState(r.json);
+        if (state) {
+            const healthy = HEALTHY_STATES.has(state.toLowerCase());
+            if (!healthy) return { ok: false, reason: `container state: ${state}`, snippet: r.snippet };
             return { ok: true, state };
         }
+        lastSnippet = r.snippet;
+    }
+    return {
+        ok: false,
+        reason: 'no container state found in any known Komodo response shape',
+        snippet: lastSnippet
+    };
+}
 
 // ---------- Extractor ----------
 

src/lib/server/pipeline/beta-runner.ts

@@ -127,7 +127,27 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
         }
         case 'awaiting-health': {
             const h = await komodoInspectHealthy(row.server);
-            if (!h.ok) return; // hold — try next tick
+            if (!h.ok) {
+                // Record the failure reason (including any snippet) so the
+                // beta detail UI can surface why we're stuck.
+                const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
+                setStage().run(
+                    'awaiting-health',
+                    `unhealthy: ${h.reason}${snippet}`,
+                    nowIso(),
+                    'awaiting-health',
+                    'awaiting-health',
+                    nowIso(),
+                    'awaiting-health',
+                    'awaiting-health',
+                    nowIso(),
+                    'awaiting-health',
+                    nowIso(),
+                    row.beta_id,
+                    row.server
+                );
+                return;
+            }
             const rcon = await rconStatusResult(row.server);
             if (!rcon.ok) return; // container reports up but game not answering yet
             const next = row.wait_for_match ? 'awaiting-match' : 'ready';

src/lib/server/pipeline/sweep.ts

@@ -177,7 +177,10 @@ async function runRestartPhase(server: string): Promise<{ state: 'done' | 'faile
 async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
     const h = await komodoInspectHealthy(server);
     if (!h.ok) {
-        return { state: 'pending', players: null, error: `unhealthy: ${h.reason}`, restartedNow: false };
+        // Include a truncated response snippet (when the extractor is the
+        // failure) so the operator can spot which Komodo field to teach.
+        const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
+        return { state: 'pending', players: null, error: `unhealthy: ${h.reason}${snippet}`, restartedNow: false };
     }
     const rcon = await rconStatusResult(server);
     if (!rcon.ok) {