Axis2 version: 1.7.1 with WLS 12.1.3
Under heavy load many threads are found in BLOCKED state doing the following:
[ACTIVE] ExecuteThread: '10' for queue: 'weblogic.kernel.Default (self-tuning)'" daemon prio=3 tid=0x0000000109b4b800 nid=0x72 waiting for monitor entry [0xffffffff4b0fc000]
java.lang.Thread.State: BLOCKED (on object monitor)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:278)
at com.sun.beans.finder.ClassFinder.findClass(ClassFinder.java:67)
at com.sun.beans.finder.ClassFinder.findClass(ClassFinder.java:110)
at java.beans.Introspector.findCustomizerClass(Introspector.java:1245)
at java.beans.Introspector.getTargetBeanDescriptor(Introspector.java:1239)
at java.beans.Introspector.getBeanInfo(Introspector.java:415)
at java.beans.Introspector.getBeanInfo(Introspector.java:252)
at java.beans.Introspector.getBeanInfo(Introspector.java:214)
at org.apache.axis2.classloader.BeanInfoCache.getCachedBeanInfo(BeanInfoCache.java:127)
at org.apache.axis2.databinding.utils.BeanUtil.getBeanInfo(BeanUtil.java:116)
at org.apache.axis2.databinding.utils.BeanUtil.getPropertyQnameList(BeanUtil.java:162)
at org.apache.axis2.databinding.utils.BeanUtil.getPullParser(BeanUtil.java:88)
at org.apache.axis2.databinding.utils.reader.ADBXMLStreamReaderImpl.processProperties(ADBXMLStreamReaderImpl.java:989)
at org.apache.axis2.databinding.utils.reader.ADBXMLStreamReaderImpl.next(ADBXMLStreamReaderImpl.java:848)
at org.apache.axis2.databinding.utils.reader.ADBXMLStreamReaderImpl.next(ADBXMLStreamReaderImpl.java:851)
at org.apache.axis2.util.StreamWrapper.next(StreamWrapper.java:71)
at org.apache.axiom.om.impl.builder.StAXOMBuilder.parserNext(StAXOMBuilder.java:663)
at org.apache.axiom.om.impl.builder.StAXOMBuilder.next(StAXOMBuilder.java:181)
at org.apache.axiom.om.impl.common.AxiomContainerSupport.ajc$interMethod$org_apache_axiom_om_impl_common_AxiomContainerSupport$org_apache_axiom_om_impl_intf_AxiomContainer$build(AxiomContainerSupport.aj:176)
at org.apache.axiom.om.impl.llom.OMElementImpl.build(OMElementImpl.java:1)
at org.apache.axiom.core.CoreElementSupport.ajc$interMethod$org_apache_axiom_core_CoreElementSupport$org_apache_axiom_core_CoreElement$beforeDetach(CoreElementSupport.aj:28)
at org.apache.axiom.om.impl.llom.OMElementImpl.ajc$interMethodDispatch2$org_apache_axiom_core$beforeDetach(OMElementImpl.java:1)
at org.apache.axiom.core.CoreChildNodeSupport.ajc$interMethodDispatch1$org_apache_axiom_core_CoreChildNodeSupport$org_apache_axiom_core_CoreChildNode$beforeDetach(CoreChildNodeSupport.aj)
at org.apache.axiom.core.CoreChildNodeSupport.ajc$interMethod$org_apache_axiom_core_CoreChildNodeSupport$org_apache_axiom_core_CoreChildNode$internalDetach(CoreChildNodeSupport.aj:205)
at org.apache.axiom.om.impl.llom.OMNodeImpl.ajc$interMethodDispatch2$org_apache_axiom_core$internalDetach(OMNodeImpl.java:1)
at org.apache.axiom.core.CoreChildNodeSupport.ajc$interMethodDispatch1$org_apache_axiom_core_CoreChildNodeSupport$org_apache_axiom_core_CoreChildNode$internalDetach(CoreChildNodeSupport.aj)
at org.apache.axiom.core.CoreParentNodeSupport.ajc$interMethod$org_apache_axiom_core_CoreParentNodeSupport$org_apache_axiom_core_CoreParentNode$coreAppendChild(CoreParentNodeSupport.aj:154)
at org.apache.axiom.om.impl.llom.OMElementImpl.coreAppendChild(OMElementImpl.java:1)
at org.apache.axiom.om.impl.common.AxiomContainerSupport.ajc$interMethod$org_apache_axiom_om_impl_common_AxiomContainerSupport$org_apache_axiom_om_impl_intf_AxiomContainer$addChild(AxiomContainerSupport.aj:140)
at org.apache.axiom.om.impl.llom.OMElementImpl.addChild(OMElementImpl.java:1)
at org.apache.axiom.om.impl.common.AxiomContainerSupport.ajc$interMethodDispatch1$org_apache_axiom_om_impl_common_AxiomContainerSupport$org_apache_axiom_om_impl_intf_AxiomContainer$addChild(AxiomContainerSupport.aj)
at org.apache.axiom.om.impl.common.AxiomContainerSupport.ajc$interMethod$org_apache_axiom_om_impl_common_AxiomContainerSupport$org_apache_axiom_om_impl_intf_AxiomContainer$addChild(AxiomContainerSupport.aj:114)
at org.apache.axiom.om.impl.llom.OMElementImpl.addChild(OMElementImpl.java:1)
at org.apache.axis2.rpc.receivers.RPCUtil.processResponse(RPCUtil.java:134)
at org.apache.axis2.rpc.receivers.RPCUtil.processResponseAsDocLitWrapped(RPCUtil.java:533)
at org.apache.axis2.rpc.receivers.RPCMessageReceiver.invokeBusinessLogic(RPCMessageReceiver.java:159)
at org.apache.axis2.receivers.AbstractInOutMessageReceiver.invokeBusinessLogic(AbstractInOutMessageReceiver.java:40)
at org.apache.axis2.receivers.AbstractMessageReceiver.receive(AbstractMessageReceiver.java:106)
at org.apache.axis2.engine.AxisEngine.receive(AxisEngine.java:169)
at org.apache.axis2.transport.http.HTTPTransportUtils.processHTTPPostRequest(HTTPTransportUtils.java:176)
at org.apache.axis2.transport.http.AxisServlet.doPost(AxisServlet.java:163)
at ws.monitoring.servlet.MonitoringServlet.doPost(MonitoringServlet.java:220)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:751)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:844)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:280)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:254)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:136)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:346)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:243)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3447)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3417)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2280)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2196)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2174)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1632)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:256)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:311)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:263)
Looking a request using JFR, we can see how many times the thread goes to BLOCKED state waiting for a monitor lock (each red square). So the thread is getting blocked 92 times waiting for the same monitor lock!. And this can be different in each request.
This is generating high contention between threads resulting in high response times. I'm not familiar with axis2 implementation. Can anyone explain why is this occurring and how it can be fixed?.