accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [12/14] git commit: ACCUMULO-2455 small work arounds to increase the chance that zookeeper updates are in place before tservers need to see them
Date Wed, 12 Mar 2014 05:12:26 GMT
ACCUMULO-2455 small work arounds to increase the chance that zookeeper updates are in place
before tservers need to see them


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/09307322
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/09307322
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/09307322

Branch: refs/heads/ACCUMULO-2061
Commit: 09307322a3e9f45cd37bbd4c63fa66ccbe2cc0a2
Parents: 46bf772
Author: Eric Newton <eric.newton@gmail.com>
Authored: Tue Mar 11 16:30:45 2014 -0400
Committer: Eric Newton <eric.newton@gmail.com>
Committed: Tue Mar 11 16:30:45 2014 -0400

----------------------------------------------------------------------
 .../apache/accumulo/proxy/SimpleProxyIT.java    | 13 ++++-----
 .../org/apache/accumulo/test/NamespacesIT.java  |  8 +++---
 .../accumulo/test/functional/BloomFilterIT.java | 30 ++++++++++++--------
 3 files changed, 28 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
----------------------------------------------------------------------
diff --git a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
index efe36b9..f554d63 100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
@@ -772,15 +772,14 @@ public class SimpleProxyIT {
 
     // create a table that's very slow, so we can look for scans/compactions
     client.createTable(creds, "slow", true, TimeType.MILLIS);
+    IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(),
Collections.singletonMap("sleepTime", "250"));
+    client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
 
-    // Should take 5 seconds to read every record
-    for (int i = 0; i < 20; i++) {
+    // Should take 10 seconds to read every record
+    for (int i = 0; i < 40; i++) {
       client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
     }
 
-    IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(),
Collections.singletonMap("sleepTime", "250"));
-    client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
-
     // scan
     Thread t = new Thread() {
       @Override
@@ -798,7 +797,7 @@ public class SimpleProxyIT {
     };
     t.start();
 
-    // look for the scan
+    // look for the scan many times 
     List<ActiveScan> scans = new ArrayList<ActiveScan>();
     for (int i = 0; i < 100 && scans.isEmpty(); i++) {
       for (String tserver : client.getTabletServers(creds)) {
@@ -811,7 +810,7 @@ public class SimpleProxyIT {
 
         if (!scans.isEmpty())
           break;
-        UtilWaitThread.sleep(10);
+        UtilWaitThread.sleep(100);
       }
     }
     t.join();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
index a8d5f7f..af9274b 100644
--- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
@@ -321,17 +321,17 @@ public class NamespacesIT extends SimpleMacIT {
     }
     IteratorSetting setting2 = c.namespaceOperations().getIteratorSetting(namespace, setting.getName(),
IteratorScope.scan);
     assertEquals(setting, setting2);
-    s = c.createScanner(t1, Authorizations.EMPTY);
-    assertFalse(s.iterator().hasNext());
     assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
     assertTrue(c.tableOperations().listIterators(t1).containsKey(iterName));
+    s = c.createScanner(t1, Authorizations.EMPTY);
+    assertFalse(s.iterator().hasNext());
 
     // verify can see inserted entry again
     c.namespaceOperations().removeIterator(namespace, setting.getName(), EnumSet.allOf(IteratorScope.class));
-    s = c.createScanner(t1, Authorizations.EMPTY);
-    assertTrue(s.iterator().hasNext());
     assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
     assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
+    s = c.createScanner(t1, Authorizations.EMPTY);
+    assertTrue(s.iterator().hasNext());
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
index 27aab91..50ca776 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
@@ -39,6 +39,7 @@ import org.apache.accumulo.core.file.keyfunctor.ColumnFamilyFunctor;
 import org.apache.accumulo.core.file.keyfunctor.ColumnQualifierFunctor;
 import org.apache.accumulo.core.file.keyfunctor.RowFunctor;
 import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.fate.util.UtilWaitThread;
 import org.apache.accumulo.minicluster.MemoryUnit;
 import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 import org.apache.hadoop.conf.Configuration;
@@ -56,6 +57,7 @@ public class BloomFilterIT extends ConfigurableMacIT {
     siteConfig.put(Property.TABLE_BLOOM_SIZE.getKey(), "2000000");
     siteConfig.put(Property.TABLE_BLOOM_ERRORRATE.getKey(), "1%");
     siteConfig.put(Property.TABLE_BLOOM_LOAD_THRESHOLD.getKey(), "0");
+    siteConfig.put(Property.TSERV_MUTATION_QUEUE_MAX.getKey(), "10M");
     siteConfig.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1G");
     cfg.setSiteConfig(siteConfig );
   }
@@ -69,9 +71,9 @@ public class BloomFilterIT extends ConfigurableMacIT {
       c.tableOperations().setProperty(table, Property.TABLE_BLOCKCACHE_ENABLED.getKey(),
"false");
     }
     log.info("Writing");
-    write(c, "bt1", 1, 0, 2000000000, 1000);
-    write(c, "bt2", 2, 0, 2000000000, 1000);
-    write(c, "bt3", 3, 0, 2000000000, 1000);
+    write(c, "bt1", 1, 0, 2000000000, 500);
+    write(c, "bt2", 2, 0, 2000000000, 500);
+    write(c, "bt3", 3, 0, 2000000000, 500);
     log.info("Writing complete");
     
     // test inserting an empty key
@@ -94,35 +96,39 @@ public class BloomFilterIT extends ConfigurableMacIT {
     
     // these queries should only run quickly if bloom filters are working, so lets get a
base
     log.info("Base query");
-    long t1 = query(c, "bt1", 1, 0, 2000000000, 100000, 1000);
-    long t2 = query(c, "bt2", 2, 0, 2000000000, 100000, 1000);
-    long t3 = query(c, "bt3", 3, 0, 2000000000, 100000, 1000);
+    long t1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500);
+    long t2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500);
+    long t3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500);
     log.info("Base query complete");
     
     log.info("Rewriting with bloom filters");
     c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
     c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName());
-    c.tableOperations().compact("bt1", null, null, false, true);
     
     c.tableOperations().setProperty("bt2", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
     c.tableOperations().setProperty("bt2", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ColumnFamilyFunctor.class.getName());
-    c.tableOperations().compact("bt2", null, null, false, true);
     
     c.tableOperations().setProperty("bt3", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
     c.tableOperations().setProperty("bt3", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ColumnQualifierFunctor.class.getName());
-    c.tableOperations().compact("bt3", null, null, false, true);
     
     c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
     c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName());
+    
+    // ensure the updates to zookeeper propogate
+    UtilWaitThread.sleep(500);
+    
     c.tableOperations().compact("bt4", null, null, false, true);
+    c.tableOperations().compact("bt1", null, null, false, true);
+    c.tableOperations().compact("bt2", null, null, false, true);
+    c.tableOperations().compact("bt3", null, null, false, true);
     log.info("Rewriting with bloom filters complete");
     
     // these queries should only run quickly if bloom
     // filters are working
     log.info("Bloom query");
-    long tb1 = query(c, "bt1", 1, 0, 2000000000, 100000, 1000);
-    long tb2 = query(c, "bt2", 2, 0, 2000000000, 100000, 1000);
-    long tb3 = query(c, "bt3", 3, 0, 2000000000, 100000, 1000);
+    long tb1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500);
+    long tb2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500);
+    long tb3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500);
     log.info("Bloom query complete");
     timeCheck(t1 + t2 + t3, tb1 + tb2 + tb3);
     


Mime
View raw message