josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Widen Komodo health probe: try container endpoints, snippet on failure

Browse Source 
GetDeployment returns config, not runtime state, so the health check was
always coming back with 'no container state found in response'. For
deployment-type servers the probe now tries GetDeploymentContainer,
InspectDeploymentContainer, and GetDeploymentStatus first, falling back
to GetDeployment only if those aren't available. The extractor also
recognizes more shapes (Health.Status, container_state, deployment_state,
etc.) and accepts more healthy strings (running, healthy, ok, started,
up).

When nothing matches, the truncated raw response snippet rides along in
the error field so the deployment/beta UI shows exactly which field to
teach the extractor next time.

Currently-stuck deployment_server rows are still 'pending', so the sweep
will retry them on its next 15s tick — no manual reset needed. Once one
of the widened endpoints resolves, those rows advance to 'done'
automatically.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
josh
2026-09-12 13:51:04 -04:00
co-authored by Claude Opus 4.7
parent edfe4dfc09
commit 11e47b2a3f
3 changed files with 141 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/integrations/komodo.ts
+116 -57
View File
@@ -255,14 +255,32 @@ export async function komodoResolveGamePort(serverName: string): Promise<number
export type KomodoHealthResult =
| { ok: true; state: string }
| { ok: false; reason: string };
| { ok: false; reason: string; snippet?: string };
// Shape-agnostic scan for the container's running-state. Walks the entire
// payload looking for anything that resembles a state field. Covered shapes:
// - Docker inspect: { State: { Status: "running", Running: true, Health: { Status: "healthy" } } }
// - Komodo GetDeployment: top-level or nested `state`, `Status`, `container_state`
// - Komodo GetDeploymentStatus: { state: "Running" }
// - Anything with `Running: true` at any depth
// When it can't find any of these, callers get the raw JSON in the error so
// we can spot which field to add here.
const STATE_KEYS = new Set([
'state',
'State',
'status',
'Status',
'container_state',
'containerState',
'deployment_state',
'deploymentState',
'docker_status'
]);
// Walks a Komodo read response looking for a container state string, matching
// the shapes we've seen: docker-inspect ({ State: { Status, Running } }),
// GetDeployment ({ state } / { status: { state } }), and GetStack similar.
function extractContainerState(payload: unknown): string | null {
const seen = new Set<unknown>();
let found: string | null = null;
const walk = (v: unknown): void => {
if (found || v === null || typeof v !== 'object' || seen.has(v)) return;
seen.add(v);
@@ -271,61 +289,43 @@ function extractContainerState(payload: unknown): string | null {
return;
}
const obj = v as Record<string, unknown>;
// Prefer the explicit docker inspect shape.
const s = obj['State'];
if (s && typeof s === 'object') {
const status = (s as Record<string, unknown>)['Status'];
const running = (s as Record<string, unknown>)['Running'];
if (typeof status === 'string') {
found = status;
return;
// Docker inspect nested: { State: { Status, Running, Health: { Status } } }
const stateObj = obj['State'];
if (stateObj && typeof stateObj === 'object') {
const so = stateObj as Record<string, unknown>;
const status = so['Status'];
const running = so['Running'];
const health = so['Health'];
if (typeof status === 'string') { found = status; return; }
if (running === true) { found = 'running'; return; }
if (health && typeof health === 'object') {
const hs = (health as Record<string, unknown>)['Status'];
if (typeof hs === 'string') { found = hs; return; }
}
if (running === true) {
found = 'running';
}
// Flat string-valued state under any of the recognized keys.
for (const key of STATE_KEYS) {
const val = obj[key];
if (typeof val === 'string' && val.length > 0) {
found = val;
return;
}
}
if (typeof obj['state'] === 'string') {
found = obj['state'] as string;
return;
}
for (const val of Object.values(obj)) walk(val);
};
walk(payload);
return found;
}
/**
* Ask Komodo whether the server's container is currently running. Used by the
* beta runner during the `awaiting-health` stage after a restart. A truthy
* `ok` doesn't imply the game is fully ready — the runner also gates on RCON
* before advancing.
*/
export async function komodoInspectHealthy(serverName: string): Promise<KomodoHealthResult> {
if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' };
const row = getServer(serverName);
if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' };
const type = row.komodo_type ?? 'deployment';
let path: string;
let body: Record<string, string>;
switch (type) {
case 'stack':
path = 'read/GetStack';
body = { stack: row.komodo_name };
break;
case 'container':
if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' };
path = 'read/InspectContainer';
body = { server: row.komodo_server, container: row.komodo_name };
break;
case 'deployment':
default:
path = 'read/GetDeployment';
body = { deployment: row.komodo_name };
break;
}
const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
async function fetchJsonWithSnippet(
path: string,
body: Record<string, string>
): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string }> {
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
let res: Response;
try {
@@ -333,19 +333,78 @@ export async function komodoInspectHealthy(serverName: string): Promise<KomodoHe
} catch (err) {
return { ok: false, reason: describeFetchError(err, url) };
}
if (!res.ok) return { ok: false, reason: `HTTP ${res.status}` };
const text = await res.text().catch(() => '');
if (!res.ok) {
return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text) };
}
let json: unknown;
try {
json = await res.json();
json = JSON.parse(text);
} catch (err) {
return { ok: false, reason: `unparseable json: ${(err as Error).message}` };
return { ok: false, reason: `unparseable json: ${(err as Error).message}`, snippet: truncate(text) };
}
const state = extractContainerState(json);
if (!state) return { ok: false, reason: 'no container state found in response' };
const healthy = state.toLowerCase() === 'running' || state.toLowerCase() === 'healthy';
if (!healthy) return { ok: false, reason: `container state: ${state}` };
return { ok: true, state };
return { ok: true, json, snippet: truncate(safeJsonStringify(json)) };
}
/**
* Ask Komodo whether the server's container is currently running. Used by the
* sweep during the health phase (post-restart) and by the beta runner. For
* deployment-type servers we hit the container-inspect endpoint first because
* GetDeployment returns config, not runtime state — falling back only when
* the container endpoint isn't available.
*/
export async function komodoInspectHealthy(serverName: string): Promise<KomodoHealthResult> {
if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' };
const row = getServer(serverName);
if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' };
const type = row.komodo_type ?? 'deployment';
const attempts: Array<{ path: string; body: Record<string, string> }> = [];
switch (type) {
case 'stack':
attempts.push({ path: 'read/GetStack', body: { stack: row.komodo_name } });
break;
case 'container':
if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' };
attempts.push({
path: 'read/InspectContainer',
body: { server: row.komodo_server, container: row.komodo_name }
});
break;
case 'deployment':
default:
// Container-specific endpoints first — they return docker-inspect
// shape with State.Status. GetDeployment is the last-resort
// config-only fallback.
attempts.push({ path: 'read/GetDeploymentContainer', body: { deployment: row.komodo_name } });
attempts.push({ path: 'read/InspectDeploymentContainer', body: { deployment: row.komodo_name } });
attempts.push({ path: 'read/GetDeploymentStatus', body: { deployment: row.komodo_name } });
attempts.push({ path: 'read/GetDeployment', body: { deployment: row.komodo_name } });
break;
}
let lastSnippet: string | undefined;
for (const { path, body } of attempts) {
const r = await fetchJsonWithSnippet(path, body);
if (!r.ok) {
lastSnippet = r.snippet ?? lastSnippet;
// Try the next endpoint on 404s or unknown-method-shaped errors; a
// real error (auth, timeout) will fail every attempt anyway.
continue;
}
const state = extractContainerState(r.json);
if (state) {
const healthy = HEALTHY_STATES.has(state.toLowerCase());
if (!healthy) return { ok: false, reason: `container state: ${state}`, snippet: r.snippet };
return { ok: true, state };
}
lastSnippet = r.snippet;
}
return {
ok: false,
reason: 'no container state found in any known Komodo response shape',
snippet: lastSnippet
};
}
// ---------- Extractor ----------
src/lib/server/pipeline/beta-runner.ts
+21 -1
View File
@@ -127,7 +127,27 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
}
case 'awaiting-health': {
const h = await komodoInspectHealthy(row.server);
if (!h.ok) return; // hold — try next tick
if (!h.ok) {
// Record the failure reason (including any snippet) so the
// beta detail UI can surface why we're stuck.
const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
setStage().run(
'awaiting-health',
`unhealthy: ${h.reason}${snippet}`,
nowIso(),
'awaiting-health',
'awaiting-health',
nowIso(),
'awaiting-health',
'awaiting-health',
nowIso(),
'awaiting-health',
nowIso(),
row.beta_id,
row.server
);
return;
}
const rcon = await rconStatusResult(row.server);
if (!rcon.ok) return; // container reports up but game not answering yet
const next = row.wait_for_match ? 'awaiting-match' : 'ready';
src/lib/server/pipeline/sweep.ts
+4 -1
View File
@@ -177,7 +177,10 @@ async function runRestartPhase(server: string): Promise<{ state: 'done' | 'faile
async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
const h = await komodoInspectHealthy(server);
if (!h.ok) {
return { state: 'pending', players: null, error: `unhealthy: ${h.reason}`, restartedNow: false };
// Include a truncated response snippet (when the extractor is the
// failure) so the operator can spot which Komodo field to teach.
const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
return { state: 'pending', players: null, error: `unhealthy: ${h.reason}${snippet}`, restartedNow: false };
}
const rcon = await rconStatusResult(server);
if (!rcon.ok) {