diff --git a/src/lib/server/integrations/komodo.ts b/src/lib/server/integrations/komodo.ts index df4f777..47d974a 100644 --- a/src/lib/server/integrations/komodo.ts +++ b/src/lib/server/integrations/komodo.ts @@ -255,14 +255,32 @@ export async function komodoResolveGamePort(serverName: string): Promise(); let found: string | null = null; + const walk = (v: unknown): void => { if (found || v === null || typeof v !== 'object' || seen.has(v)) return; seen.add(v); @@ -271,61 +289,43 @@ function extractContainerState(payload: unknown): string | null { return; } const obj = v as Record; - // Prefer the explicit docker inspect shape. - const s = obj['State']; - if (s && typeof s === 'object') { - const status = (s as Record)['Status']; - const running = (s as Record)['Running']; - if (typeof status === 'string') { - found = status; - return; + + // Docker inspect nested: { State: { Status, Running, Health: { Status } } } + const stateObj = obj['State']; + if (stateObj && typeof stateObj === 'object') { + const so = stateObj as Record; + const status = so['Status']; + const running = so['Running']; + const health = so['Health']; + if (typeof status === 'string') { found = status; return; } + if (running === true) { found = 'running'; return; } + if (health && typeof health === 'object') { + const hs = (health as Record)['Status']; + if (typeof hs === 'string') { found = hs; return; } } - if (running === true) { - found = 'running'; + } + + // Flat string-valued state under any of the recognized keys. + for (const key of STATE_KEYS) { + const val = obj[key]; + if (typeof val === 'string' && val.length > 0) { + found = val; return; } } - if (typeof obj['state'] === 'string') { - found = obj['state'] as string; - return; - } + for (const val of Object.values(obj)) walk(val); }; walk(payload); return found; } -/** - * Ask Komodo whether the server's container is currently running. Used by the - * beta runner during the `awaiting-health` stage after a restart. A truthy - * `ok` doesn't imply the game is fully ready — the runner also gates on RCON - * before advancing. - */ -export async function komodoInspectHealthy(serverName: string): Promise { - if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' }; - const row = getServer(serverName); - if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' }; - - const type = row.komodo_type ?? 'deployment'; - let path: string; - let body: Record; - switch (type) { - case 'stack': - path = 'read/GetStack'; - body = { stack: row.komodo_name }; - break; - case 'container': - if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' }; - path = 'read/InspectContainer'; - body = { server: row.komodo_server, container: row.komodo_name }; - break; - case 'deployment': - default: - path = 'read/GetDeployment'; - body = { deployment: row.komodo_name }; - break; - } +const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']); +async function fetchJsonWithSnippet( + path: string, + body: Record +): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string }> { const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`; let res: Response; try { @@ -333,19 +333,78 @@ export async function komodoInspectHealthy(serverName: string): Promise ''); + if (!res.ok) { + return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text) }; + } let json: unknown; try { - json = await res.json(); + json = JSON.parse(text); } catch (err) { - return { ok: false, reason: `unparseable json: ${(err as Error).message}` }; + return { ok: false, reason: `unparseable json: ${(err as Error).message}`, snippet: truncate(text) }; } - const state = extractContainerState(json); - if (!state) return { ok: false, reason: 'no container state found in response' }; - const healthy = state.toLowerCase() === 'running' || state.toLowerCase() === 'healthy'; - if (!healthy) return { ok: false, reason: `container state: ${state}` }; - return { ok: true, state }; + return { ok: true, json, snippet: truncate(safeJsonStringify(json)) }; +} + +/** + * Ask Komodo whether the server's container is currently running. Used by the + * sweep during the health phase (post-restart) and by the beta runner. For + * deployment-type servers we hit the container-inspect endpoint first because + * GetDeployment returns config, not runtime state — falling back only when + * the container endpoint isn't available. + */ +export async function komodoInspectHealthy(serverName: string): Promise { + if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' }; + const row = getServer(serverName); + if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' }; + + const type = row.komodo_type ?? 'deployment'; + const attempts: Array<{ path: string; body: Record }> = []; + switch (type) { + case 'stack': + attempts.push({ path: 'read/GetStack', body: { stack: row.komodo_name } }); + break; + case 'container': + if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' }; + attempts.push({ + path: 'read/InspectContainer', + body: { server: row.komodo_server, container: row.komodo_name } + }); + break; + case 'deployment': + default: + // Container-specific endpoints first — they return docker-inspect + // shape with State.Status. GetDeployment is the last-resort + // config-only fallback. + attempts.push({ path: 'read/GetDeploymentContainer', body: { deployment: row.komodo_name } }); + attempts.push({ path: 'read/InspectDeploymentContainer', body: { deployment: row.komodo_name } }); + attempts.push({ path: 'read/GetDeploymentStatus', body: { deployment: row.komodo_name } }); + attempts.push({ path: 'read/GetDeployment', body: { deployment: row.komodo_name } }); + break; + } + + let lastSnippet: string | undefined; + for (const { path, body } of attempts) { + const r = await fetchJsonWithSnippet(path, body); + if (!r.ok) { + lastSnippet = r.snippet ?? lastSnippet; + // Try the next endpoint on 404s or unknown-method-shaped errors; a + // real error (auth, timeout) will fail every attempt anyway. + continue; + } + const state = extractContainerState(r.json); + if (state) { + const healthy = HEALTHY_STATES.has(state.toLowerCase()); + if (!healthy) return { ok: false, reason: `container state: ${state}`, snippet: r.snippet }; + return { ok: true, state }; + } + lastSnippet = r.snippet; + } + return { + ok: false, + reason: 'no container state found in any known Komodo response shape', + snippet: lastSnippet + }; } // ---------- Extractor ---------- diff --git a/src/lib/server/pipeline/beta-runner.ts b/src/lib/server/pipeline/beta-runner.ts index 2a63506..867135e 100644 --- a/src/lib/server/pipeline/beta-runner.ts +++ b/src/lib/server/pipeline/beta-runner.ts @@ -127,7 +127,27 @@ async function advance(row: BetaServerActiveRow): Promise { } case 'awaiting-health': { const h = await komodoInspectHealthy(row.server); - if (!h.ok) return; // hold — try next tick + if (!h.ok) { + // Record the failure reason (including any snippet) so the + // beta detail UI can surface why we're stuck. + const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : ''; + setStage().run( + 'awaiting-health', + `unhealthy: ${h.reason}${snippet}`, + nowIso(), + 'awaiting-health', + 'awaiting-health', + nowIso(), + 'awaiting-health', + 'awaiting-health', + nowIso(), + 'awaiting-health', + nowIso(), + row.beta_id, + row.server + ); + return; + } const rcon = await rconStatusResult(row.server); if (!rcon.ok) return; // container reports up but game not answering yet const next = row.wait_for_match ? 'awaiting-match' : 'ready'; diff --git a/src/lib/server/pipeline/sweep.ts b/src/lib/server/pipeline/sweep.ts index 53945fb..53a2190 100644 --- a/src/lib/server/pipeline/sweep.ts +++ b/src/lib/server/pipeline/sweep.ts @@ -177,7 +177,10 @@ async function runRestartPhase(server: string): Promise<{ state: 'done' | 'faile async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> { const h = await komodoInspectHealthy(server); if (!h.ok) { - return { state: 'pending', players: null, error: `unhealthy: ${h.reason}`, restartedNow: false }; + // Include a truncated response snippet (when the extractor is the + // failure) so the operator can spot which Komodo field to teach. + const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : ''; + return { state: 'pending', players: null, error: `unhealthy: ${h.reason}${snippet}`, restartedNow: false }; } const rcon = await rconStatusResult(server); if (!rcon.ok) {