From 11e47b2a3f85b185faaa0b6d2b458c6b9f0868cd Mon Sep 17 00:00:00 2001 From: josh Date: Sat, 12 Sep 2026 13:51:04 -0400 Subject: [PATCH] Widen Komodo health probe: try container endpoints, snippet on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GetDeployment returns config, not runtime state, so the health check was always coming back with 'no container state found in response'. For deployment-type servers the probe now tries GetDeploymentContainer, InspectDeploymentContainer, and GetDeploymentStatus first, falling back to GetDeployment only if those aren't available. The extractor also recognizes more shapes (Health.Status, container_state, deployment_state, etc.) and accepts more healthy strings (running, healthy, ok, started, up). When nothing matches, the truncated raw response snippet rides along in the error field so the deployment/beta UI shows exactly which field to teach the extractor next time. Currently-stuck deployment_server rows are still 'pending', so the sweep will retry them on its next 15s tick — no manual reset needed. Once one of the widened endpoints resolves, those rows advance to 'done' automatically. Co-Authored-By: Claude Opus 4.7 --- src/lib/server/integrations/komodo.ts | 173 +++++++++++++++++-------- src/lib/server/pipeline/beta-runner.ts | 22 +++- src/lib/server/pipeline/sweep.ts | 5 +- 3 files changed, 141 insertions(+), 59 deletions(-) diff --git a/src/lib/server/integrations/komodo.ts b/src/lib/server/integrations/komodo.ts index df4f777..47d974a 100644 --- a/src/lib/server/integrations/komodo.ts +++ b/src/lib/server/integrations/komodo.ts @@ -255,14 +255,32 @@ export async function komodoResolveGamePort(serverName: string): Promise(); let found: string | null = null; + const walk = (v: unknown): void => { if (found || v === null || typeof v !== 'object' || seen.has(v)) return; seen.add(v); @@ -271,61 +289,43 @@ function extractContainerState(payload: unknown): string | null { return; } const obj = v as Record; - // Prefer the explicit docker inspect shape. - const s = obj['State']; - if (s && typeof s === 'object') { - const status = (s as Record)['Status']; - const running = (s as Record)['Running']; - if (typeof status === 'string') { - found = status; - return; + + // Docker inspect nested: { State: { Status, Running, Health: { Status } } } + const stateObj = obj['State']; + if (stateObj && typeof stateObj === 'object') { + const so = stateObj as Record; + const status = so['Status']; + const running = so['Running']; + const health = so['Health']; + if (typeof status === 'string') { found = status; return; } + if (running === true) { found = 'running'; return; } + if (health && typeof health === 'object') { + const hs = (health as Record)['Status']; + if (typeof hs === 'string') { found = hs; return; } } - if (running === true) { - found = 'running'; + } + + // Flat string-valued state under any of the recognized keys. + for (const key of STATE_KEYS) { + const val = obj[key]; + if (typeof val === 'string' && val.length > 0) { + found = val; return; } } - if (typeof obj['state'] === 'string') { - found = obj['state'] as string; - return; - } + for (const val of Object.values(obj)) walk(val); }; walk(payload); return found; } -/** - * Ask Komodo whether the server's container is currently running. Used by the - * beta runner during the `awaiting-health` stage after a restart. A truthy - * `ok` doesn't imply the game is fully ready — the runner also gates on RCON - * before advancing. - */ -export async function komodoInspectHealthy(serverName: string): Promise { - if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' }; - const row = getServer(serverName); - if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' }; - - const type = row.komodo_type ?? 'deployment'; - let path: string; - let body: Record; - switch (type) { - case 'stack': - path = 'read/GetStack'; - body = { stack: row.komodo_name }; - break; - case 'container': - if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' }; - path = 'read/InspectContainer'; - body = { server: row.komodo_server, container: row.komodo_name }; - break; - case 'deployment': - default: - path = 'read/GetDeployment'; - body = { deployment: row.komodo_name }; - break; - } +const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']); +async function fetchJsonWithSnippet( + path: string, + body: Record +): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string }> { const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`; let res: Response; try { @@ -333,19 +333,78 @@ export async function komodoInspectHealthy(serverName: string): Promise ''); + if (!res.ok) { + return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text) }; + } let json: unknown; try { - json = await res.json(); + json = JSON.parse(text); } catch (err) { - return { ok: false, reason: `unparseable json: ${(err as Error).message}` }; + return { ok: false, reason: `unparseable json: ${(err as Error).message}`, snippet: truncate(text) }; } - const state = extractContainerState(json); - if (!state) return { ok: false, reason: 'no container state found in response' }; - const healthy = state.toLowerCase() === 'running' || state.toLowerCase() === 'healthy'; - if (!healthy) return { ok: false, reason: `container state: ${state}` }; - return { ok: true, state }; + return { ok: true, json, snippet: truncate(safeJsonStringify(json)) }; +} + +/** + * Ask Komodo whether the server's container is currently running. Used by the + * sweep during the health phase (post-restart) and by the beta runner. For + * deployment-type servers we hit the container-inspect endpoint first because + * GetDeployment returns config, not runtime state — falling back only when + * the container endpoint isn't available. + */ +export async function komodoInspectHealthy(serverName: string): Promise { + if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' }; + const row = getServer(serverName); + if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' }; + + const type = row.komodo_type ?? 'deployment'; + const attempts: Array<{ path: string; body: Record }> = []; + switch (type) { + case 'stack': + attempts.push({ path: 'read/GetStack', body: { stack: row.komodo_name } }); + break; + case 'container': + if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' }; + attempts.push({ + path: 'read/InspectContainer', + body: { server: row.komodo_server, container: row.komodo_name } + }); + break; + case 'deployment': + default: + // Container-specific endpoints first — they return docker-inspect + // shape with State.Status. GetDeployment is the last-resort + // config-only fallback. + attempts.push({ path: 'read/GetDeploymentContainer', body: { deployment: row.komodo_name } }); + attempts.push({ path: 'read/InspectDeploymentContainer', body: { deployment: row.komodo_name } }); + attempts.push({ path: 'read/GetDeploymentStatus', body: { deployment: row.komodo_name } }); + attempts.push({ path: 'read/GetDeployment', body: { deployment: row.komodo_name } }); + break; + } + + let lastSnippet: string | undefined; + for (const { path, body } of attempts) { + const r = await fetchJsonWithSnippet(path, body); + if (!r.ok) { + lastSnippet = r.snippet ?? lastSnippet; + // Try the next endpoint on 404s or unknown-method-shaped errors; a + // real error (auth, timeout) will fail every attempt anyway. + continue; + } + const state = extractContainerState(r.json); + if (state) { + const healthy = HEALTHY_STATES.has(state.toLowerCase()); + if (!healthy) return { ok: false, reason: `container state: ${state}`, snippet: r.snippet }; + return { ok: true, state }; + } + lastSnippet = r.snippet; + } + return { + ok: false, + reason: 'no container state found in any known Komodo response shape', + snippet: lastSnippet + }; } // ---------- Extractor ---------- diff --git a/src/lib/server/pipeline/beta-runner.ts b/src/lib/server/pipeline/beta-runner.ts index 2a63506..867135e 100644 --- a/src/lib/server/pipeline/beta-runner.ts +++ b/src/lib/server/pipeline/beta-runner.ts @@ -127,7 +127,27 @@ async function advance(row: BetaServerActiveRow): Promise { } case 'awaiting-health': { const h = await komodoInspectHealthy(row.server); - if (!h.ok) return; // hold — try next tick + if (!h.ok) { + // Record the failure reason (including any snippet) so the + // beta detail UI can surface why we're stuck. + const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : ''; + setStage().run( + 'awaiting-health', + `unhealthy: ${h.reason}${snippet}`, + nowIso(), + 'awaiting-health', + 'awaiting-health', + nowIso(), + 'awaiting-health', + 'awaiting-health', + nowIso(), + 'awaiting-health', + nowIso(), + row.beta_id, + row.server + ); + return; + } const rcon = await rconStatusResult(row.server); if (!rcon.ok) return; // container reports up but game not answering yet const next = row.wait_for_match ? 'awaiting-match' : 'ready'; diff --git a/src/lib/server/pipeline/sweep.ts b/src/lib/server/pipeline/sweep.ts index 53945fb..53a2190 100644 --- a/src/lib/server/pipeline/sweep.ts +++ b/src/lib/server/pipeline/sweep.ts @@ -177,7 +177,10 @@ async function runRestartPhase(server: string): Promise<{ state: 'done' | 'faile async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> { const h = await komodoInspectHealthy(server); if (!h.ok) { - return { state: 'pending', players: null, error: `unhealthy: ${h.reason}`, restartedNow: false }; + // Include a truncated response snippet (when the extractor is the + // failure) so the operator can spot which Komodo field to teach. + const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : ''; + return { state: 'pending', players: null, error: `unhealthy: ${h.reason}${snippet}`, restartedNow: false }; } const rcon = await rconStatusResult(server); if (!rcon.ok) {