Report stalled agent lanes and bound health lock waits

src/lib/hosts.ts

@@ -35,6 +35,21 @@ export interface HostAgentUpdate {
     checkedAt?: string;
 }
 
+export interface HostAgentLane {
+    name: 'health-1' | 'health-2' | 'read' | 'write';
+    alive: boolean; busy: boolean; lastPollAt: string | null; pollAgeSeconds: number;
+}
+export function hostAgentLaneLabel(name: HostAgentLane['name']): string {
+    return { 'health-1': 'Health check 1', 'health-2': 'Health check 2', read: 'File reads', write: 'Actions' }[name];
+}
+export function hostAgentLaneState(lane: HostAgentLane): 'stopped' | 'stalled' | 'busy' | 'starting' | 'polling' {
+    if (!lane.alive) return 'stopped';
+    // Reads and mutations can legitimately take minutes. Health work has a
+    // bounded budget; an old poll there indicates a blocked worker even if busy.
+    if (lane.pollAgeSeconds > 90 && (!lane.busy || lane.name.startsWith('health-'))) return 'stalled';
+    return lane.busy ? 'busy' : lane.lastPollAt === null ? 'starting' : 'polling';
+}
+
 export function hostAgentUpdatesEnabled(inventory: HostInventory): boolean {
     return inventory.agentUpdate?.enabled === true && inventory.capabilities?.includes('agent.update') === true;
 }
@@ -68,6 +83,7 @@ export interface HostInventory {
     currentOperation?: string | null; [key: string]: unknown;
     capabilities?: string[];
     agentUpdate?: HostAgentUpdate;
+    agentLanes?: HostAgentLane[];
     instanceResources?: InstanceResources[];
     instanceResourcesError?: string;
     instanceStates?: InstanceState[];

src/lib/server/hosts/index.ts

@@ -11,6 +11,7 @@ import { parseHostInventory } from './request-context.js';
 import { fleetSettings } from '../settings.js';
 import { recordInstanceResources, validateInstanceResources } from './calibration.js';
 import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
+import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js';
 import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationFilter, HostOperationHistoryPage, HostOperationScope, HostOperationState } from '../../hosts.js';
 
 export const AGENT_PROTOCOL_VERSION = 1;
@@ -46,6 +47,10 @@ function hostFromRow(row: HostRow): Host {
     if (!row.enrolled_at) reasons.push('Agent has not enrolled.');
     else if (!row.credential_active) reasons.push('Agent credential has been revoked.');
     if (!connected) reasons.push('Agent heartbeat is unavailable or stale.');
+    if (connected) for (const lane of inventory.agentLanes ?? []) {
+        const state = hostAgentLaneState(lane);
+        if (state === 'stopped' || state === 'stalled') reasons.push(`Agent ${hostAgentLaneLabel(lane.name).toLowerCase()} worker is ${state}. Check the host agent service.`);
+    }
     if (inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) reasons.push('A compatible agent protocol is required.');
     if (!inventory.dockerRunning) reasons.push('Docker is not ready.');
     if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.');
@@ -287,6 +292,20 @@ export function validateInventory(value: unknown): HostInventory {
             ...(update.error === undefined ? {} : { error: update.error }),
             ...(update.checkedAt === undefined ? {} : { checkedAt: update.checkedAt }) };
     }
+    if (inventory.agentLanes !== undefined) {
+        if (!Array.isArray(inventory.agentLanes) || inventory.agentLanes.length > 4) throw new Error('Invalid agent lane inventory.');
+        // Keep valid lane diagnostics without discarding a host heartbeat over
+        // one malformed sample. Do not persist thread details or raw errors.
+        const names = new Set<string>();
+        inventory.agentLanes = inventory.agentLanes.filter(lane => {
+            if (!lane || !['health-1','health-2','read','write'].includes(lane.name) || names.has(lane.name)
+                || typeof lane.alive !== 'boolean' || typeof lane.busy !== 'boolean'
+                || typeof lane.pollAgeSeconds !== 'number' || !Number.isFinite(lane.pollAgeSeconds) || lane.pollAgeSeconds < 0
+                || (lane.lastPollAt !== null && (typeof lane.lastPollAt !== 'string' || lane.lastPollAt.length > 40 || !Number.isFinite(Date.parse(lane.lastPollAt))))) return false;
+            names.add(lane.name);
+            return true;
+        }).map(({name,alive,busy,lastPollAt,pollAgeSeconds}) => ({name,alive,busy,lastPollAt,pollAgeSeconds}));
+    }
     if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library =>
         !library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.');
     return inventory;