activemq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chir...@apache.org
Subject svn commit: r1448654 - in /activemq/activemq-apollo/trunk: apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/ apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/ apollo-website/src/documentation/
Date Thu, 21 Feb 2013 14:36:22 GMT
Author: chirino
Date: Thu Feb 21 14:36:22 2013
New Revision: 1448654

URL: http://svn.apache.org/r1448654
Log:
Fixes APLO-296: Support configuring the LevelDB auto compaction frequency

Modified:
    activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala
    activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java
    activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala?rev=1448654&r1=1448653&r2=1448654&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala
(original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala
Thu Feb 21 14:36:22 2013
@@ -71,6 +71,7 @@ object LevelDBClient extends Log {
 
   final val LOG_SUFFIX = ".log"
   final val INDEX_SUFFIX = ".index"
+  var auto_compaction_ratio = 100
 
   def bytes(value: String) = value.getBytes("UTF-8")
 
@@ -251,6 +252,7 @@ class LevelDBClient(store: LevelDBStore)
     index_options.createIfMissing(true);
     val paranoid_checks = config.paranoid_checks.getOrElse(false)
 
+    auto_compaction_ratio = OptionSupport(config.auto_compaction_ratio).getOrElse(100)
     config.index_max_open_files.foreach(index_options.maxOpenFiles(_))
     config.index_block_restart_interval.foreach(index_options.blockRestartInterval(_))
     index_options.paranoidChecks(paranoid_checks)
@@ -1151,9 +1153,13 @@ class LevelDBClient(store: LevelDBStore)
     }
   }
 
+  // APLO-245: lets try to detect when leveldb needs a compaction..
+  def detect_if_compact_needed:Unit = {
 
-  def detect_if_compact_needed = {
-    // APLO-245: lets try to detect when leveldb needs a compaction..
+    // auto compaction might be disabled...
+    if ( auto_compaction_ratio <= 0 ) {
+      return
+    }
 
     // How much space is the dirty index using??
     var index_usage = 0L
@@ -1174,7 +1180,7 @@ class LevelDBClient(store: LevelDBStore)
 
       // After running some load we empirically found that a healthy ratio is between 12
and 25 bytes per entry.
       // lets compact if we go way over the healthy ratio.
-      if( ratio > 100 ) {
+      if( ratio > auto_compaction_ratio ) {
         index.compact_needed = true
       }
     } else if( index_usage > 1024*1024*5 )  {
@@ -1195,9 +1201,11 @@ class LevelDBClient(store: LevelDBStore)
 
     // Lets compact the leveldb index if it looks like we need to.
     if( index.compact_needed ) {
-      info("Compacting the leveldb index at: "+dirty_index_file)
+      debug("Compacting the leveldb index at: %s", dirty_index_file)
+      val start = System.nanoTime()
       index.compact
-      info("Compaction completed")
+      val duration = System.nanoTime() - start;
+      info("Compacted the leveldb index at: %s in %.2f ms", dirty_index_file, (duration /
1000000.0))
     }
     val empty_journals = log.log_infos.keySet.toSet -- log_refs.keySet
 

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java?rev=1448654&r1=1448653&r2=1448654&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java
(original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java
Thu Feb 21 14:36:22 2013
@@ -78,6 +78,9 @@ public class LevelDBStoreDTO extends Sto
     @XmlAttribute(name="index_factory")
     public String index_factory;
 
+    @XmlAttribute(name="auto_compaction_ratio")
+    public Integer auto_compaction_ratio;
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

Modified: activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md?rev=1448654&r1=1448653&r2=1448654&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md (original)
+++ activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md Thu Feb
21 14:36:22 2013
@@ -639,6 +639,11 @@ A `leveldb_store` element may be configu
    Can be `snappy` or `none`. The value defaults to `snappy`.
 * `log_compression` : The type of compression to apply to the log records.  
    Can be `snappy` or `none`. The value defaults to `snappy`.
+* `auto_compaction_ratio`: This ratio is used to determine when to compact 
+   the leveldb indexes.  When you take the ratio of disk space used by the leveldb 
+   indexes to the number queue entries and it exceeds the configured 
+   `auto_compaction_ratio` then the leveldb index will be scheduled for compaction.
+   If set to 0, then auto compactions are disabled.  The value defaults to 100.
 
 ### Support Platforms
 



Mime
View raw message