activemq-users mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmachina <jmach...@millenniumadvisors.com>
Subject Deadlock on broker (thread dumps included)
Date Fri, 28 Feb 2014 18:30:37 GMT
Experiencing a recurring issue with our activeMQ broker, where it will stop
servicing new connections (any new connection from the client side hangs on
establishing a connection). Thread dump attached, and a small selection of
the dump pasted below. Our configuration is also listed below (and
attached).


<beans
  xmlns="http://www.springframework.org/schema/beans"
  xmlns:amq="http://activemq.apache.org/schema/core"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
  http://activemq.apache.org/schema/core
http://activemq.apache.org/schema/core/activemq-core.xsd">

    <bean
class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
        <property name="locations">
            <value>file:${activemq.conf}/credentials.properties</value>
        </property>
    </bean>

    <broker xmlns="http://activemq.apache.org/schema/core" brokerName="main"
dataDirectory="${activemq.data}" offlineDurableSubscriberTimeout="86400000"
offlineDurableSubscriberTaskSchedule="21600000">

        <destinationPolicy>
            <policyMap>
              <policyEntries>
                <policyEntry topic=">" producerFlowControl="false"
					advisoryForSlowConsumers="true" 
					advisoryForFastProducers="true" 
					advisoryWhenFull="true" 
					memoryLimit="4096mb">
                
					<dispatchPolicy>
						<simpleDispatchPolicy/>
					</dispatchPolicy>					

					<pendingMessageLimitStrategy>
						<constantPendingMessageLimitStrategy limit="10000"/> 
					</pendingMessageLimitStrategy>
					
					<messageEvictionStrategy>
                        <oldestMessageEvictionStrategy/>
                    </messageEvictionStrategy>					
                </policyEntry>

                <policyEntry queue=">" 
					producerFlowControl="false" 
					optimizedDispatch="true" 
					useCache="false" 
					memoryLimit="4096mb">
	                <pendingMessageLimitStrategy>
						<constantPendingMessageLimitStrategy limit="10000"/> 
					</pendingMessageLimitStrategy>
			
					
					<messageEvictionStrategy>
                        <oldestMessageEvictionStrategy/>
                    </messageEvictionStrategy>				    
                </policyEntry>	
				
				<policyEntry topic=">" tempTopic="true" producerFlowControl="false"	
					advisoryForSlowConsumers="true" 
					advisoryForFastProducers="true" 
					advisoryWhenFull="true" 
					memoryLimit="4096mb">
				</policyEntry>
				
				<policyEntry queue=">" tempQueue="true" producerFlowControl="false"	
				    optimizedDispatch="true" 
					useCache="false" 
			        advisoryForSlowConsumers="true" 
					advisoryForFastProducers="true" 
					advisoryWhenFull="true" 
					memoryLimit="4096mb">
				</policyEntry>	
              </policyEntries>
            </policyMap>
        </destinationPolicy>

        <managementContext>
            <managementContext createConnector="true" connectorPort="1099"
rmiServerPort="1199"/>
        </managementContext>

        <persistenceAdapter>
            <kahaDB directory="${activemq.data}/kahadb-main"/>
        </persistenceAdapter>

        <systemUsage>
            <systemUsage>
                <memoryUsage>
                    <memoryUsage limit="8 gb"/>
                </memoryUsage>
                <storeUsage>
                    <storeUsage limit="20 gb"/>
                </storeUsage>
                <tempUsage>
                    <tempUsage limit="50 gb"/>
                </tempUsage>
            </systemUsage>
        </systemUsage>

        <transportConnectors>
			      <transportConnector name="main-openwire"
uri="tcp://0.0.0.0:61616"/>
        </transportConnectors>

        <shutdownHooks>
            <bean xmlns="http://www.springframework.org/schema/beans"
class="org.apache.activemq.hooks.SpringContextHook" />
        </shutdownHooks>

    </broker>
    <import resource="jetty-main.xml"/>
</beans>


Selection of thread dumps:
 "ActiveMQ Transport: tcp:///10.0.1.229:51552@61616" - Thread t@1545
   java.lang.Thread.State: RUNNABLE
	at java.net.SocketOutputStream.socketWrite0(Native Method)
	at java.net.SocketOutputStream.socketWrite(Unknown Source)
	at java.net.SocketOutputStream.write(Unknown Source)
	at
org.apache.activemq.transport.tcp.TcpBufferedOutputStream.flush(TcpBufferedOutputStream.java:115)
	at java.io.DataOutputStream.flush(Unknown Source)
	at
org.apache.activemq.transport.tcp.TcpTransport.oneway(TcpTransport.java:176)
	at
org.apache.activemq.transport.AbstractInactivityMonitor.doOnewaySend(AbstractInactivityMonitor.java:304)
	at
org.apache.activemq.transport.AbstractInactivityMonitor.oneway(AbstractInactivityMonitor.java:286)
	at
org.apache.activemq.transport.TransportFilter.oneway(TransportFilter.java:85)
	at
org.apache.activemq.transport.WireFormatNegotiator.oneway(WireFormatNegotiator.java:104)
	at
org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68)
	at
org.apache.activemq.broker.TransportConnection.dispatch(TransportConnection.java:1339)
	at
org.apache.activemq.broker.TransportConnection.processDispatch(TransportConnection.java:858)
	at
org.apache.activemq.broker.TransportConnection.dispatchSync(TransportConnection.java:818)
	at
org.apache.activemq.broker.region.TopicSubscription.dispatch(TopicSubscription.java:611)
	at
org.apache.activemq.broker.region.TopicSubscription.add(TopicSubscription.java:112)
	at
org.apache.activemq.broker.region.policy.SimpleDispatchPolicy.dispatch(SimpleDispatchPolicy.java:48)
	at org.apache.activemq.broker.region.Topic.dispatch(Topic.java:705)
	at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:504)
	- locked <6f8e78f0> (a org.apache.activemq.broker.region.Topic)
	at org.apache.activemq.broker.region.Topic.send(Topic.java:440)
	at
org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:394)
	at
org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:442)
	at
org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:283)
	at
org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:585)
	at
org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:516)
	at
org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:511)
	at
org.apache.activemq.advisory.AdvisoryBroker.addDestinationInfo(AdvisoryBroker.java:206)
	at
org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257)
	at
org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257)
	at
org.apache.activemq.broker.MutableBrokerFilter.addDestinationInfo(MutableBrokerFilter.java:262)
	at
org.apache.activemq.broker.TransportConnection.processAddDestination(TransportConnection.java:495)
	at
org.apache.activemq.command.DestinationInfo.visit(DestinationInfo.java:122)
	at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:292)
	at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:149)
	at
org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50)
	at
org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
	at
org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:270)
	at
org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
	at
org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214)
	at
org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196)
	at java.lang.Thread.run(Unknown Source)

   Locked ownable synchronizers:
	- locked <6c833519> (a
java.util.concurrent.locks.ReentrantLock$NonfairSync)
	
"ActiveMQ Transport: tcp:///10.0.1.229:51551@61616" - Thread t@1544
   java.lang.Thread.State: BLOCKED
	at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:459)
	- waiting to lock <6f8e78f0> (a org.apache.activemq.broker.region.Topic)
owned by "ActiveMQ Transport: tcp:///10.0.1.229:51552@61616" t@1545
	at org.apache.activemq.broker.region.Topic.send(Topic.java:440)
	at
org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:394)
	at
org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:442)
	at
org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:283)
	at
org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:585)
	at
org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:516)
	at
org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:511)
	at
org.apache.activemq.advisory.AdvisoryBroker.addDestinationInfo(AdvisoryBroker.java:206)
	at
org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257)
	at
org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257)
	at
org.apache.activemq.broker.MutableBrokerFilter.addDestinationInfo(MutableBrokerFilter.java:262)
	at
org.apache.activemq.broker.TransportConnection.processAddDestination(TransportConnection.java:495)
	at
org.apache.activemq.command.DestinationInfo.visit(DestinationInfo.java:122)
	at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:292)
	at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:149)
	at
org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50)
	at
org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
	at
org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:270)
	at
org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
	at
org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214)
	at
org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196)
	at java.lang.Thread.run(Unknown Source)

   Locked ownable synchronizers:
	- None
	
"ActiveMQ BrokerService[main] Task-374" - Thread t@1906
   java.lang.Thread.State: BLOCKED
	at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:459)
	- waiting to lock <6f8e78f0> (a org.apache.activemq.broker.region.Topic)
owned by "ActiveMQ Transport: tcp:///10.0.1.229:51552@61616" t@1545
	at org.apache.activemq.broker.region.Topic.send(Topic.java:440)
	at
org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:394)
	at
org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:442)
	at
org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:283)
	at
org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:585)
	at
org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:516)
	at
org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:511)
	at
org.apache.activemq.advisory.AdvisoryBroker.removeDestination(AdvisoryBroker.java:221)
	at
org.apache.activemq.broker.BrokerFilter.removeDestination(BrokerFilter.java:172)
	at
org.apache.activemq.broker.BrokerFilter.removeDestination(BrokerFilter.java:172)
	at
org.apache.activemq.broker.MutableBrokerFilter.removeDestination(MutableBrokerFilter.java:177)
	at
org.apache.activemq.broker.TransportConnection.processRemoveConnection(TransportConnection.java:779)
	- locked <6f39a3be> (a
org.apache.activemq.broker.jmx.ManagedTransportConnection)
	at
org.apache.activemq.broker.TransportConnection.doStop(TransportConnection.java:1107)
	at
org.apache.activemq.broker.TransportConnection$4.run(TransportConnection.java:1037)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
	at java.lang.Thread.run(Unknown Source)

   Locked ownable synchronizers:
	- locked <312d47fe> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)

	- locked <1556e994> (a java.util.concurrent.ThreadPoolExecutor$Worker)
	
"ActiveMQ BrokerService[main] Task-396" - Thread t@1959
   java.lang.Thread.State: TIMED_WAITING
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for <6eb8ca36> (a
java.util.concurrent.SynchronousQueue$TransferStack)
	at java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source)
	at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(Unknown
Source)
	at java.util.concurrent.SynchronousQueue$TransferStack.transfer(Unknown
Source)
	at java.util.concurrent.SynchronousQueue.poll(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
	at java.lang.Thread.run(Unknown Source)

   Locked ownable synchronizers:
	- None
	
"ActiveMQ Transport: tcp:///10.0.1.202:59370@61616" - Thread t@1498
   java.lang.Thread.State: RUNNABLE
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(Unknown Source)
	at java.net.SocketInputStream.read(Unknown Source)
	at
org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill(TcpBufferedInputStream.java:50)
	at
org.apache.activemq.transport.tcp.TcpTransport$2.fill(TcpTransport.java:604)
	at
org.apache.activemq.transport.tcp.TcpBufferedInputStream.read(TcpBufferedInputStream.java:58)
	at
org.apache.activemq.transport.tcp.TcpTransport$2.read(TcpTransport.java:589)
	at java.io.DataInputStream.readInt(Unknown Source)
	at
org.apache.activemq.openwire.OpenWireFormat.unmarshal(OpenWireFormat.java:275)
	at
org.apache.activemq.transport.tcp.TcpTransport.readCommand(TcpTransport.java:221)
	at
org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:213)
	at
org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196)
	at java.lang.Thread.run(Unknown Source)

   Locked ownable synchronizers:
	- None



threaddump-1393608740161.tdump
<http://activemq.2283324.n4.nabble.com/file/n4678423/threaddump-1393608740161.tdump>
 
activemq.xml
<http://activemq.2283324.n4.nabble.com/file/n4678423/activemq.xml>  



--
View this message in context: http://activemq.2283324.n4.nabble.com/Deadlock-on-broker-thread-dumps-included-tp4678423.html
Sent from the ActiveMQ - User mailing list archive at Nabble.com.

Mime
View raw message