activemq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chir...@apache.org
Subject svn commit: r1205104 - in /activemq/activemq-apollo/trunk: apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/ apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/ apollo-distro/ apollo-distro/src/main/descriptors/ ...
Date Tue, 22 Nov 2011 17:46:17 GMT
Author: chirino
Date: Tue Nov 22 17:46:15 2011
New Revision: 1205104

URL: http://svn.apache.org/viewvc?rev=1205104&view=rev
Log:
Package the leveldb store and use it by default.

Removed:
    activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/HALevelDBStoreDTO.java
Modified:
    activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java
    activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala
    activemq/activemq-apollo/trunk/apollo-distro/pom.xml
    activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml
    activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml
    activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index
    activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java
    activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
    activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml
    activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md

Modified: activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java
(original)
+++ activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java
Tue Nov 22 17:46:15 2011
@@ -17,7 +17,6 @@
 
 /**
  * The JAXB POJOs for the
- * The JAXB POJOs for the
  * <a href="http://activemq.apache.org/schema/activemq/apollo/xml-configuration.html">XML
Configuration</a>
  * of the ActiveMQ Broker.
  */

Modified: activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala
(original)
+++ activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala
Tue Nov 22 17:46:15 2011
@@ -251,7 +251,11 @@ class Create extends Action {
   }
 
   def store_config = {
-    if( can_load("com.sleepycat.je.Environment") ) {
+    if( can_load("org.fusesource.leveldbjni.JniDBFactory") ) {
+    """<!-- You can delete this element if you want to disable persistence for this virtual
host -->
+    <leveldb_store directory="${apollo.base}/data"/>
+    """
+    } else if( can_load("com.sleepycat.je.Environment") ) {
     """<!-- You can delete this element if you want to disable persistence for this virtual
host -->
     <bdb_store directory="${apollo.base}/data"/>
     """

Modified: activemq/activemq-apollo/trunk/apollo-distro/pom.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-distro/pom.xml?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-distro/pom.xml (original)
+++ activemq/activemq-apollo/trunk/apollo-distro/pom.xml Tue Nov 22 17:46:15 2011
@@ -42,7 +42,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.activemq</groupId>
-      <artifactId>apollo-bdb</artifactId>
+      <artifactId>apollo-leveldb</artifactId>
       <version>1.0-SNAPSHOT</version>
     </dependency>
     <dependency>
@@ -52,6 +52,11 @@
     </dependency>
     <dependency>
       <groupId>org.apache.activemq</groupId>
+      <artifactId>apollo-bdb</artifactId>
+      <version>1.0-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.activemq</groupId>
       <artifactId>apollo-tcp</artifactId>
       <version>1.0-SNAPSHOT</version>
     </dependency>

Modified: activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml (original)
+++ activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml Tue Nov
22 17:46:15 2011
@@ -29,10 +29,9 @@
         <include>org.scala-lang:scala-library</include>
         <include>org.fusesource.hawtbuf:*</include>
         <include>org.fusesource.hawtdispatch:*</include>
-        <include>org.fusesource.hawtdb:*</include>
-        <include>org.fusesource.jdbm:*</include>
         <include>org.codehaus.jackson:jackson-mapper-asl</include>
         <include>org.codehaus.jackson:jackson-core-asl</include>
+        <include>org.fusesource.leveldbjni:leveldbjni-all</include>
         
         <!-- security stuff -->
         <include>org.jasypt:jasypt</include>

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml Tue Nov 22 17:46:15 2011
@@ -116,26 +116,6 @@
   <build>
     <plugins>
       
-      <!-- include all the dependencies into the jar so it can run standalone -->
-      <plugin>
-        <groupId>org.fusesource.mvnplugins</groupId>
-        <artifactId>maven-uberize-plugin</artifactId>
-        <version>1.15</version>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals><goal>uberize</goal></goals>
-            <configuration>
-              <artifactSet>
-                <includes>
-                  <include>org.fusesource.leveldbjni:leveldbjni-all</include>
-                </includes>
-              </artifactSet>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      
       <!-- <plugin>
         <groupId>org.fusesource.mvnplugins</groupId>
         <artifactId>maven-fab-plugin</artifactId>

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index
(original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index
Tue Nov 22 17:46:15 2011
@@ -15,4 +15,4 @@
 ## limitations under the License.
 ## ---------------------------------------------------------------------------
 LevelDBStoreDTO
-LevelDBStoreStatusDTO
+LevelDBStoreStatusDTO
\ No newline at end of file

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java
(original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java
Tue Nov 22 17:46:15 2011
@@ -21,7 +21,7 @@
  * of the ActiveMQ Broker.
  */
 @javax.xml.bind.annotation.XmlSchema(
-        namespace = "http://fabric.fusesource.org/apollo",
+        namespace = "http://activemq.apache.org/schema/activemq/apollo",
         elementFormDefault = javax.xml.bind.annotation.XmlNsForm.QUALIFIED)
 package org.apache.activemq.apollo.broker.store.leveldb.dto;
 

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
(original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
Tue Nov 22 17:46:15 2011
@@ -33,7 +33,6 @@ import org.fusesource.hawtdispatch._
 import org.apache.activemq.apollo.util.{TreeMap=>ApolloTreeMap}
 import collection.immutable.TreeMap
 import org.iq80.leveldb._
-import org.fusesource.leveldbjni.JniDBFactory._
 import org.fusesource.leveldbjni.internal.Util
 import org.fusesource.hawtbuf.{Buffer, AbstractVarIntSupport}
 import java.util.concurrent.atomic.{AtomicReference, AtomicLong}
@@ -54,7 +53,10 @@ object LevelDBClient extends Log {
   final val queue_prefix_array = Array(queue_prefix)
   final val map_prefix_array = Array(map_prefix)
   final val queue_entry_prefix_array = Array(queue_entry_prefix)
+
   final val dirty_index_key = bytes(":dirty")
+  final val TRUE = bytes("true")
+  final val FALSE = bytes("false")
 
   final val LOG_ADD_QUEUE           = 1.toByte
   final val LOG_REMOVE_QUEUE        = 2.toByte
@@ -67,6 +69,8 @@ object LevelDBClient extends Log {
   final val LOG_SUFFIX  = ".log"
   final val INDEX_SUFFIX  = ".index"
 
+  def bytes(value:String) = value.getBytes("UTF-8")
+
   import FileSupport._
   def create_sequence_file(directory:File, id:Long, suffix:String) = directory / ("%016x%s".format(id,
suffix))
 
@@ -139,6 +143,8 @@ class LevelDBClient(store: LevelDBStore)
   var last_gc_duration = 0L
   var in_gc = false
   var gc_detected_log_usage = Map[Long, UsageCounter]()
+  var factory:DBFactory = org.fusesource.leveldbjni.JniDBFactory.factory
+
 
   def dirty_index_file = directory / ("dirty"+INDEX_SUFFIX)
   def temp_index_file = directory / ("temp"+INDEX_SUFFIX)
@@ -224,7 +230,7 @@ class LevelDBClient(store: LevelDBStore)
 
       index = new RichDB(factory.open(dirty_index_file, index_options));
       try {
-        index.put(dirty_index_key, bytes("true"))
+        index.put(dirty_index_key, TRUE)
         // Update the index /w what was stored on the logs..
         var pos = last_index_snapshot_pos;
 
@@ -318,7 +324,7 @@ class LevelDBClient(store: LevelDBStore)
     snapshot_rw_lock.writeLock().lock()
 
     // Close the index so that it's files are not changed async on us.
-    index.put(dirty_index_key, bytes("false"), new WriteOptions().sync(true))
+    index.put(dirty_index_key, FALSE, new WriteOptions().sync(true))
     index.close
   }
 
@@ -330,7 +336,7 @@ class LevelDBClient(store: LevelDBStore)
     // re=open it..
     retry {
       index = new RichDB(factory.open(dirty_index_file, index_options));
-      index.put(dirty_index_key, bytes("true"))
+      index.put(dirty_index_key, TRUE)
     }
     snapshot_rw_lock.writeLock().unlock()
   }

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml
(original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml
Tue Nov 22 17:46:15 2011
@@ -15,10 +15,9 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
-<broker xmlns="http://activemq.apache.org/schema/activemq/apollo"
-        xmlns:fabric="http://fabric.fusesource.org/apollo">
+<broker xmlns="http://activemq.apache.org/schema/activemq/apollo">
     <virtual_host enabled="true" id="vh-local">
         <host_name>localhost</host_name>
-        <fabric:leveldb_store directory="activemq-data"/>
+        <leveldb_store directory="activemq-data"/>
     </virtual_host>
 </broker>

Modified: activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md (original)
+++ activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md Tue Nov
22 17:46:15 2011
@@ -342,12 +342,14 @@ cannot get swapped to disk.
 ${project_name} supports multiple message store implementations.  The 
 implementations currently supported are:
 
+* [LevelDB Store](#LevelDB_Store) : is a file based message store implemented using the 
+  [Google's LevelDB](http://en.wikipedia.org/wiki/LevelDB) library to maintain indexes into

+  log files holding the messages.
 * [BDB Store](#BDB_Store) : is a file based message store implemented using the 
   [Sleepycat BDB](http://en.wikipedia.org/wiki/Berkeley_DB) library.
   This is the most stable implementation.
 * [JDBM2 Store](#JDBM2_Store) : is a file based message store implemented using the 
   [JDBM2](http://code.google.com/p/jdbm2/) library.
-  This is the most stable implementation.
 
 <!-- 
 * [HawtDB Store](#HawtDB_Store) : is a file based message store implemented using the 
@@ -356,6 +358,61 @@ implementations currently supported are:
   debugger. 
 -->
 
+
+###### LevelDB Store
+
+The LevelDB store is the default store which a newly created Broker instance
+will use.
+
+It is enabled when your `virtual_host` element contains a `leveldb_store` element.
+
+{pygmentize:: xml}
+  ...
+  <virtual_host id="default">
+    ...
+    <leveldb_store directory="${apollo.base}/data"/>
+    ..
+  </virtual_host>
+  ...
+{pygmentize}
+
+A `leveldb_store` element may be configured with the following attributes:
+
+* `directory` : The directory which the store will use to hold it's data
+  files. The store will create the directory if it does not already
+  exist.
+* `flush_delay` : The flush delay is the amount of time in milliseconds
+  that a store will delay persisting a messaging unit of work in hopes
+  that it will be invalidated shortly thereafter by another unit of work
+  which would negate the operation.
+* `gc_interval` : How often to check to find log files which can be discarded 
+   in seconds. The value defaults to 1800 (30 minutes).
+* `read_threads` : The number of concurrent IO reads to allow. The value 
+   defaults to 10.
+* `sync` : If set to `false`, then the store does not sync logging operations to 
+  disk. The value defaults to `true`.
+* `log_size` : The max size (in bytes) of each data log file before log file rotation
+   occurs. The value defaults to 104857600 (100 MB).
+* `log_write_buffer_size`: That maximum amount of log data to build up before writing 
+   to the file system. The value defaults to 4194304 (4 MB).
+* `verify_checksums` :  If set to `true` to force checksum verification of all 
+   data that is read from the file system on behalf of a particular read. By 
+   default, no such verification is done.
+* `paranoid_checks` : Make the store error out as soon as possible if it detects 
+   internal corruption.  The value defaults to false.
+* `index_max_open_files` : Number of open files that can be used by the index. The 
+   value defaults to 1000.
+* `index_block_restart_interval` : Number keys between restart points for delta 
+   encoding of keys. The value defaults to 16.
+* `index_write_buffer_size` : Amount of index data to build up in memory before 
+   converting to a sorted on-disk file. The value defaults to 4194304 (4 MB).
+* `index_block_size` : The size of index data packed per block. The value defaults 
+   to 4096 (4 K).
+* `index_cache_size` : The maximum amount of memory to use to cache index blocks. 
+   The value defaults to 268435456 (256 MB).
+* `index_compression` : The type of compression to apply to the index blocks.  
+   Can be `snappy` or `none`. The value defaults to `snappy`.
+
 ###### BDB Store
 
 Apache cannot redistribute the BDB library due to the terms of it's



Mime
View raw message