activemq-users mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From immunojeff <jfisc...@jfischer.com>
Subject ActiveMQ performance deteriorates over time
Date Tue, 04 Mar 2008 20:18:23 GMT

PROBLEM:

We currently have ActiveMQ 4.x setup on 3 Linux servers in a Master/Slave
arrangement with Postgres 8.1 acting as the backing database for jdbc only
persistence. Our message load requirements are quite small (around 500
messages total in a 24 hour period). There are other Postgres databases
setup on the database box, but the load we've monitored on these databases
is never out of hand. After about a 24 hour period of usage, we start to see
long wait times to enqueue and dequeue (perhaps as long as 5 minutes).
Additionally, we've noticed on several occasions that our SpringProducer
appeared to successful enqueue a message, but we found the message was not
actually persisted to the database. Additionally, we observe that if we
restart the brokers, performance resumes at the expected level for a period
of time, until it begins to deteriorate again.

Are there known compatibility issues with ActiveMQ 4.x and Postgres when
using Master/Slave for clustering? Or, is this behavior perhaps attributable
to some other cause? I was not able to find out anything definitive through
my web searches.

ENVIRONMENT AND SOFTWARE:

ActiveMQ 4.1-SNAPSHOT (as of several weeks ago)
Postgres 8.1
Tomcat 5
JDK 1.4

We have web modules deployed on Tomcat using Spring to enqueue and dequeue
(SpringProducer to enqueue and DefaultMessageListenerContainer for MDP and
dequeue)

CONFIGURATION:

(This is the ActiveMQ configuration)
<beans
  xmlns="http://www.springframework.org/schema/beans"
  xmlns:amq="http://activemq.org/config/1.0"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
  http://activemq.org/config/1.0
http://activemq.apache.org/schema/activemq-core.xsd
  http://activemq.apache.org/camel/schema/spring
http://activemq.apache.org/camel/schema/spring/camel-spring.xsd">

  <!-- Allows us to use system properties as variables in this configuration
file -->
  <bean class="com....PropertyPlaceholderConfigurer">
  	  <property name="configurationAgent" ref="configurationAgent" />
  </bean>
  
  <broker xmlns="http://activemq.org/config/1.0" useJmx="true"
brokerName="localhost" persistent="true">
  
    <destinations>
      <queue physicalName="..." />
      <queue physicalName="..." />
      <queue physicalName="..." />
    </destinations>

    <!-- The transport connectors ActiveMQ will listen to -->
    <transportConnectors>
       <transportConnector name="openwire" uri="${openwireUrl}"/>
    </transportConnectors>

    <persistenceAdapter>
        <jdbcPersistenceAdapter dataSource="#activemq-ds"/>
    </persistenceAdapter>
    
    <managementContext>
       <managementContext createConnector="true"
connectorPort="${JMXConnectorPort}" useMBeanServer="false"/>
    </managementContext>
   
  </broker>
  
  <!--  This xbean configuration file supports all the standard spring xml
configuration options -->
	
  <!-- Define a Spring Bean for managing application configurations -->
  <bean id="configurationAgent" class="....ConfigurationAgentImpl"
init-method="initialize" singleton="true">
      <property name="applicationConfigurationDAO"
ref="applicationConfigurationDAO"/>
  </bean>
    
  <!-- ApplicationConfigurationDAO: Hibernate implementation -->
  <bean id="applicationConfigurationDAO"
class="....ApplicationConfigurationDAOHibernate">
    <property name="sessionFactory" ref="sessionFactory" />
  </bean>
  
  <!-- Hibernate SessionFactory -->
  <bean id="sessionFactory"
class="org.springframework.orm.hibernate3.LocalSessionFactoryBean">
    <property name="dataSource">
      <ref local="appconfig-ds" />
    </property>
    <property name="mappingResources">
      <list>
        <!-- production definitions -->
        <value>
          .../ApplicationConfiguration.hbm.xml
        </value>
      </list>
    </property>
    <property name="hibernateProperties">
      <props>
        <prop key="hibernate.dialect">
          org.hibernate.dialect.PostgreSQLDialect
        </prop>
        <!--prop key="hibernate.connection.pool_size">3</prop-->
        <prop key="hibernate.show_sql">false</prop>
        <prop key="hibernate.jdbc.batch_size">20</prop>
      </props>
    </property>
  </bean>
  
  <bean id="activemq-ds"
class="org.springframework.jndi.JndiObjectFactoryBean">
      <property name="jndiName">
          <value>java:jdbc/ActiveMQRepositoryDataSource</value>
      </property>
  </bean>
  
  <bean id="appconfig-ds"
class="org.springframework.jndi.JndiObjectFactoryBean">
      <property name="jndiName">
          <value>java:jdbc/AppConfigDataSource</value>
      </property>
  </bean>

</beans>

(DataSource setup for Postgres)
<Resource auth="Container" name="jdbc/ActiveMQRepositoryDataSource"
type="javax.sql.DataSource"/>
    <ResourceParams name="jdbc/ActiveMQRepositoryDataSource">
        <parameter>
            <name>factory</name>
            <value>org.apache.commons.dbcp.BasicDataSourceFactory</value>
        </parameter>
        <parameter>
            <name>driverClassName</name>
            <value>org.postgresql.Driver</value>
        </parameter>
        <parameter>
            <name>url</name>
            <value>...</value>
        </parameter>
        <parameter>
            <name>username</name>
            <value>...</value>
        </parameter>
        <parameter>
            <name>password</name>
            <value>...</value>
        </parameter>
        <parameter>
            <name>maxActive</name>
            <value>20</value>
        </parameter>
        <parameter>
            <name>maxIdle</name>
            <value>0</value>
        </parameter>
        <parameter>
            <name>maxWait</name>
            <value>60000</value>
        </parameter>
        <parameter>
            <name>removeAbandoned</name>
            <value>true</value>
        </parameter>
        <parameter>
            <name>logAbandoned</name>
            <value>true</value>
        </parameter>
        <parameter>
            <name>removeAbandonedTimeout</name>
            <value>300</value>
        </parameter>
    </ResourceParams>

(Consumer Configuration)
<?xml version="1.0" encoding="UTF-8"?>

<!DOCTYPE beans PUBLIC "-//SPRING//DTD BEAN//EN"
"http://www.springframework.org/dtd/spring-beans.dtd">
<beans>
	<bean id="queueList" class="....QueueList" singleton="true">
		<property name="jmxPort" value="${JMXConnectorPort}"/>
	</bean>
	
	<bean id="WorkOrderTranslationWriter"
		class="....WorkOrderTranslationWriter"
		singleton="true">
		<property name="configurationAgent" ref="configurationAgent" />
		<property name="loggingAgent" ref="loggingAgent" />
		<property name="productionDAO" ref="productionDAO" />
		<property name="mailingSystemDAO" ref="mailingSystemDAO" />
		<property name="translationEngine" ref="translationEngine" />
		<property name="workOrderTranslationWriterDAO" ref="transactionalService"
/>
		<property name="jobTicketSpringProducer" ref="jobTicketSpringProducer" />
	</bean>

	<bean id="TranslationValidator"
		class="....TranslationValidator"
		singleton="true">
		<property name="productionRequestDAO"
			ref="productionRequestDAO" />
		<property name="loggingAgent" ref="loggingAgent" />
		<property name="translationEngine" ref="translationEngine" />
	</bean>

	<bean id="jmsFactory"
		class="org.apache.activemq.pool.PooledConnectionFactory"
		destroy-method="stop">
		<property name="connectionFactory">
			<bean
				class="org.apache.activemq.ActiveMQConnectionFactory">
				<property name="brokerURL">
					<value>${failoverBrokerRange}</value>
				</property>
			</bean>
		</property>
	</bean>

	<bean id="jmsQueue"
		class="org.springframework.jndi.JndiObjectFactoryBean">
		<property name="jndiName" value="workorder.id" />
		<property name="jndiEnvironment">
			<props>
				<prop key="java.naming.factory.initial">
					org.apache.activemq.jndi.ActiveMQInitialContextFactory
				</prop>
				<prop key="java.naming.provider.url">
					${failoverBrokerRange}
				</prop>
				<prop key="queue.queue">
					...
				</prop>
			</props>
		</property>
	</bean>
	
	<bean id="jmsQueueAccudata"
		class="org.springframework.jndi.JndiObjectFactoryBean">
		<property name="jndiName" value="..." />
		<property name="jndiEnvironment">
			<props>
				<prop key="java.naming.factory.initial">
					org.apache.activemq.jndi.ActiveMQInitialContextFactory
				</prop>
				<prop key="java.naming.provider.url">
					${failoverBrokerRange}
				</prop>
				<prop key="queue.queue">
					...
				</prop>
			</props>
		</property>
	</bean>
	
	<bean id="jmsQueueJobTicket"
		class="org.springframework.jndi.JndiObjectFactoryBean">
		<property name="jndiName" value="..." />
		<property name="jndiEnvironment">
			<props>
				<prop key="java.naming.factory.initial">
					org.apache.activemq.jndi.ActiveMQInitialContextFactory
				</prop>
				<prop key="java.naming.provider.url">
					${failoverBrokerRange}
				</prop>
				<prop key="queue.queue">
					...
				</prop>
			</props>
		</property>
	</bean>

	<bean id="jmsTemplate"
		class="org.springframework.jms.core.JmsTemplate">
		<property name="connectionFactory" ref="jmsFactory" />
	</bean>

	<bean id="producer"
		class="...SpringProducer">
		<property name="template" ref="jmsTemplate" />
		<property name="destination" ref="jmsQueue" />
	</bean>
	
	<bean id="producerAccudata"
		class="....AccudataSpringProducer">
		<property name="template" ref="jmsTemplate" />
		<property name="destination" ref="jmsQueueAccudata" />
	</bean>
	
	<bean id="jobTicketSpringProducer"
		class="....JobTicketSpringProducer">
		<property name="template" ref="jmsTemplate" />
		<property name="destination" ref="jmsQueueJobTicket" />
	</bean>

	<bean id="translationManager"
		class="....TranslationManager"
		singleton="true" />

	<bean id="messageValidator"
		class="....MessageValidator"
		singleton="true">
		<property name="maxAccudataAttempts" value="6" />
	</bean>

	<bean id="messageListener"
		class="....TranslateOrderMDP">
		<property name="translationManager" ref="translationManager" />
		<property name="loggingAgent" ref="loggingAgent" />
		<property name="notificationAgent" ref="notificationAgent" />
		<property name="messageValidator" ref="messageValidator" />
		<property name="producer" ref="producerAccudata" />
		<property name="maximumRedeliveries" value="${queueRedeliveryAttempts}" />
	</bean>

	<bean id="redeliveryPolicy"
		class="org.apache.activemq.RedeliveryPolicy">
		<property name="maximumRedeliveries" value="${queueRedeliveryAttempts}" />
		<property name="initialRedeliveryDelay" value="1000" />
	</bean>

	<bean id="consumerConnection1"
		class="org.springframework.jms.connection.SingleConnectionFactory">
		<property name="targetConnectionFactory">
			<bean
				class="org.apache.activemq.ActiveMQConnectionFactory">
				<property name="brokerURL">
					<value>${failoverBrokerRange}</value>
				</property>
				<property name="redeliveryPolicy"
					ref="redeliveryPolicy" />
			</bean>
		</property>
	</bean>
	
	<bean id="consumerConnection2"
		class="org.springframework.jms.connection.SingleConnectionFactory">
		<property name="targetConnectionFactory">
			<bean
				class="org.apache.activemq.ActiveMQConnectionFactory">
				<property name="brokerURL">
					<value>${failoverBrokerRange}</value>
				</property>
				<property name="redeliveryPolicy"
					ref="redeliveryPolicy" />
			</bean>
		</property>
	</bean>

	<bean id="jmsContainer" class="...DefaultMessageListenerContainer">
		<!-- This delay value allow us to create a JMS message selector that
		will only return messages from the given queue that are at least the
		specified number of milliseconds in the past -->
		<property name="delay" value="${normalDelay}"/>
		<property name="repeatDelay" value="${initialRedeliveryDelay}"/>
		<property name="connectionFactory" ref="consumerConnection1" />
		<property name="destination" ref="jmsQueue" />
		<property name="messageListener" ref="messageListener" />
		<property name="pubSubNoLocal" value="true" />
		<property name="sessionTransacted" value="true" />
		<property name="concurrentConsumers" value="${concurrentConsumers}" />
		<property name="maxConcurrentConsumers" value="${maxConcurrentConsumers}"
/>
		<!-- The cache level must be set to 2 to force the recreation of the
consumer
		on each attempt to communicate with the broker. This allows us to recreate
		the message selector as we proceed through time for each consumer
instance. -->
		<property name="cacheLevel" value="2"/>
	</bean>
	
	<bean id="jmsContainer2" class="...DefaultMessageListenerContainer">
		<!-- This delay value allow us to create a JMS message selector that
		will only return messages from the given queue that are at least the
		specified number of milliseconds in the past -->
		<property name="delay" value="${accudataDelay}"/>
		<property name="repeatDelay" value="${initialRedeliveryDelay}"/>
		<property name="connectionFactory" ref="consumerConnection2" />
		<property name="destination" ref="jmsQueueAccudata" />
		<property name="messageListener" ref="messageListener" />
		<property name="pubSubNoLocal" value="true" />
		<property name="sessionTransacted" value="true" />
		<property name="concurrentConsumers" value="${concurrentConsumers}" />
		<property name="maxConcurrentConsumers" value="${maxConcurrentConsumers}"
/>
		<!-- The cache level must be set to 2 to force the recreation of the
consumer
		on each attempt to communicate with the broker. This allows us to recreate
		the message selector as we proceed through time for each consumer
instance. -->
		<property name="cacheLevel" value="2"/>
	</bean>
</beans>
-- 
View this message in context: http://www.nabble.com/ActiveMQ-performance-deteriorates-over-time-tp15836200s2354p15836200.html
Sent from the ActiveMQ - User mailing list archive at Nabble.com.


Mime
View raw message