accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject git commit: ACCUMULO-2458 make fewer queries over more blocks to emphasize bloom filter performance
Date Fri, 21 Mar 2014 17:01:08 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.6.0-SNAPSHOT bb8bdc066 -> ff0693cc5


ACCUMULO-2458 make fewer queries over more blocks to emphasize bloom filter performance


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ff0693cc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ff0693cc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ff0693cc

Branch: refs/heads/1.6.0-SNAPSHOT
Commit: ff0693cc5713a34355f5f8864631bdc5dff546b5
Parents: bb8bdc0
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Fri Mar 21 13:00:26 2014 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Fri Mar 21 13:00:26 2014 -0400

----------------------------------------------------------------------
 .../accumulo/test/functional/BloomFilterIT.java   | 18 +++++++++---------
 test/src/test/resources/log4j.properties          |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff0693cc/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
index 50ca776..9ba713d 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
@@ -58,7 +58,7 @@ public class BloomFilterIT extends ConfigurableMacIT {
     siteConfig.put(Property.TABLE_BLOOM_ERRORRATE.getKey(), "1%");
     siteConfig.put(Property.TABLE_BLOOM_LOAD_THRESHOLD.getKey(), "0");
     siteConfig.put(Property.TSERV_MUTATION_QUEUE_MAX.getKey(), "10M");
-    siteConfig.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1G");
+    siteConfig.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "64K");
     cfg.setSiteConfig(siteConfig );
   }
   
@@ -96,9 +96,9 @@ public class BloomFilterIT extends ConfigurableMacIT {
     
     // these queries should only run quickly if bloom filters are working, so lets get a
base
     log.info("Base query");
-    long t1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500);
-    long t2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500);
-    long t3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500);
+    long t1 = query(c, "bt1", 1, 0, 2000000000, 5000, 500);
+    long t2 = query(c, "bt2", 2, 0, 2000000000, 5000, 500);
+    long t3 = query(c, "bt3", 3, 0, 2000000000, 5000, 500);
     log.info("Base query complete");
     
     log.info("Rewriting with bloom filters");
@@ -114,7 +114,7 @@ public class BloomFilterIT extends ConfigurableMacIT {
     c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
     c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName());
     
-    // ensure the updates to zookeeper propogate
+    // ensure the updates to zookeeper propagate
     UtilWaitThread.sleep(500);
     
     c.tableOperations().compact("bt4", null, null, false, true);
@@ -126,9 +126,9 @@ public class BloomFilterIT extends ConfigurableMacIT {
     // these queries should only run quickly if bloom
     // filters are working
     log.info("Bloom query");
-    long tb1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500);
-    long tb2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500);
-    long tb3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500);
+    long tb1 = query(c, "bt1", 1, 0, 2000000000, 5000, 500);
+    long tb2 = query(c, "bt2", 2, 0, 2000000000, 5000, 500);
+    long tb3 = query(c, "bt3", 3, 0, 2000000000, 5000, 500);
     log.info("Bloom query complete");
     timeCheck(t1 + t2 + t3, tb1 + tb2 + tb3);
     
@@ -147,7 +147,7 @@ public class BloomFilterIT extends ConfigurableMacIT {
     if (improvement < .1) {
       throw new Exception("Queries had less than 10% improvement (old: " + t1 + " new: "
+ t2 + " improvement: " + (improvement*100) + "%)");
     }
-    log.info("Improvement: " + (improvement * 100) + "%");
+    log.info(String.format("Improvement: %.2f%% (%d vs %d)", (improvement * 100), t1, t2));
   }
   
   private long query(Connector c, String table, int depth, long start, long end, int num,
int step) throws Exception {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff0693cc/test/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/test/src/test/resources/log4j.properties b/test/src/test/resources/log4j.properties
index 27e4db2..3fa1983 100644
--- a/test/src/test/resources/log4j.properties
+++ b/test/src/test/resources/log4j.properties
@@ -18,7 +18,7 @@ log4j.appender.CA=org.apache.log4j.ConsoleAppender
 log4j.appender.CA.layout=org.apache.log4j.PatternLayout
 log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c{2}] %-5p: %m%n
 
-log4j.logger.org.apache.accumulo.core=TRACE
+log4j.logger.org.apache.accumulo.core=DEBUG
 log4j.logger.org.apache.accumulo.core.client.impl.ServerClient=ERROR
 log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=off
 log4j.logger.org.apache.accumulo.core.util.shell.Shell=FATAL


Mime
View raw message