incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [5/5] git commit: Refactoring all integration tests to run as a test suite in their own project. This is an effort to get the tests to run faster.
Date Mon, 08 Jun 2015 12:41:53 GMT
Refactoring all integration tests to run as a test suite in their own project.  This is an effort to get the tests to run faster.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d8756092
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d8756092
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d8756092

Branch: refs/heads/blur-it-tests
Commit: d8756092198bd608e2e4e489560add2dd93fca8a
Parents: 9243fe0
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Jun 8 08:41:24 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Jun 8 08:41:24 2015 -0400

----------------------------------------------------------------------
 .../blur/command/IntegrationTestSuite.java      |  23 +-
 .../clusterstatus/ZookeeperClusterStatus.java   |  14 -
 .../indexserver/DistributedIndexServer.java     |   2 +-
 .../IndexSearcherCloseableSecureBase.java       |   2 +
 .../java/org/apache/blur/thrift/TableAdmin.java |  38 +-
 .../test/java/org/apache/blur/MiniCluster.java  |  32 +-
 .../java/org/apache/blur/MiniClusterServer.java |   2 +
 .../blur/analysis/ThriftFieldManagerTest.java   |  81 ---
 .../blur/analysis/ThriftFieldManagerTestIT.java |  75 +++
 .../ReconnectWhileCommandIsRunningIntTests.java |  39 --
 ...econnectWhileCommandIsRunningIntTestsIT.java |  44 ++
 .../ZookeeperClusterStatusTest.java             | 305 ----------
 .../ZookeeperClusterStatusTestIT.java           | 307 ++++++++++
 ...MasterBasedDistributedLayoutFactoryTest.java | 175 ------
 ...sterBasedDistributedLayoutFactoryTestIT.java | 174 ++++++
 .../blur/manager/indexserver/SafeModeTest.java  | 261 ---------
 .../manager/indexserver/SafeModeTestIT.java     | 257 ++++++++
 .../SharedMergeSchedulerThroughputTest.java     |  83 ---
 .../SharedMergeSchedulerThroughputTestIT.java   |  87 +++
 .../org/apache/blur/thrift/BaseClusterTest.java |  13 +-
 .../org/apache/blur/thrift/BlurClientTest.java  |  77 ---
 .../thrift/BlurClientTestMultipleQuorumsIT.java |  94 +++
 .../apache/blur/thrift/BlurClusterTestBase.java |  80 ++-
 .../blur/thrift/BlurClusterTestNoSecurity.java  |  54 --
 .../thrift/BlurClusterTestNoSecurityIT.java     |  54 ++
 .../blur/thrift/BlurClusterTestSecurity.java    | 185 ------
 .../blur/thrift/BlurClusterTestSecurityIT.java  | 185 ++++++
 .../java/org/apache/blur/thrift/FacetTests.java |  72 ---
 .../org/apache/blur/thrift/FacetTestsIT.java    |  72 +++
 .../blur/thrift/IntegrationTestSuite.java       |  28 +-
 .../org/apache/blur/thrift/SuiteCluster.java    | 132 ++++-
 .../java/org/apache/blur/thrift/TableGen.java   |   3 +-
 .../java/org/apache/blur/thrift/TermsTests.java |  99 ----
 .../org/apache/blur/thrift/TermsTestsIT.java    |  99 ++++
 .../apache/blur/thrift/ThriftServerTest.java    |  78 ---
 .../apache/blur/thrift/ThriftServerTestIT.java  |  78 +++
 blur-hive/pom.xml                               |  12 +
 .../org/apache/blur/hive/BlurSerDeTest.java     | 579 -------------------
 .../org/apache/blur/hive/BlurSerDeTestIT.java   | 526 +++++++++++++++++
 blur-integration-tests/pom.xml                  | 331 +++++++++++
 .../apache/blur/tests/RunIntegrationTests.java  |  79 +++
 .../blur/mapreduce/lib/BlurInputFormatTest.java | 297 ----------
 .../mapreduce/lib/BlurInputFormatTestIT.java    | 296 ++++++++++
 .../lib/BlurOutputFormatMiniClusterTest.java    | 235 --------
 .../lib/BlurOutputFormatMiniClusterTestIT.java  | 230 ++++++++
 .../mapreduce/lib/BlurOutputFormatTest.java     | 478 ---------------
 .../mapreduce/lib/BlurOutputFormatTestIT.java   | 484 ++++++++++++++++
 .../blur/mapreduce/lib/update/DriverTest.java   | 357 ------------
 .../blur/mapreduce/lib/update/DriverTestIT.java | 322 +++++++++++
 .../search/IndexSearcherCloseableBase.java      |   2 +
 blur-shell/pom.xml                              |  12 +
 .../blur/shell/RemoveShardServerCommand.java    |   3 +-
 blur-status/pom.xml                             |  12 +
 blur-util/pom.xml                               |  26 +-
 .../org/apache/blur/concurrent/Executors.java   |   2 +-
 .../apache/blur/zookeeper/ZkMiniCluster.java    |   2 +-
 .../org/apache/blur/zookeeper/ZkUtilsTest.java  |   2 +-
 pom.xml                                         |   6 +
 58 files changed, 4100 insertions(+), 3597 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java b/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java
index e619bec..2412a61 100644
--- a/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java
+++ b/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java
@@ -36,28 +36,21 @@ public class IntegrationTestSuite {
   @ClassRule
   public static ExternalResource testCluster = new ExternalResource() {
 
-    private boolean _managing;
-
     @Override
     protected void after() {
-      if (_managing) {
-        try {
-          SuiteCluster.shutdownMiniCluster();
-        } catch (IOException e) {
-          throw new RuntimeException(e);
-        }
+      try {
+        SuiteCluster.shutdownMiniCluster(IntegrationTestSuite.class);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
       }
     }
 
     @Override
     protected void before() throws Throwable {
-      if (!SuiteCluster.isClusterSetup()) {
-        _managing = true;
-        try {
-          SuiteCluster.setupMiniCluster();
-        } catch (IOException e) {
-          throw new RuntimeException(e);
-        }
+      try {
+        SuiteCluster.setupMiniCluster(IntegrationTestSuite.class);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
       }
     }
   };

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
index 7ae5761..421477b 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
@@ -100,24 +100,10 @@ public class ZookeeperClusterStatus extends ClusterStatus {
     }
   }
 
-  public ZookeeperClusterStatus(String connectionStr, BlurConfiguration configuration, Configuration config)
-      throws IOException {
-    this(new ZooKeeper(connectionStr, 30000, new Watcher() {
-      @Override
-      public void process(WatchedEvent event) {
-
-      }
-    }), configuration, config);
-  }
-
   public ZookeeperClusterStatus(ZooKeeper zooKeeper) throws IOException {
     this(zooKeeper, new BlurConfiguration(), new Configuration());
   }
 
-  public ZookeeperClusterStatus(String connectionStr) throws IOException {
-    this(connectionStr, new BlurConfiguration(), new Configuration());
-  }
-
   class Clusters extends OnChange {
     @Override
     public void action(List<String> clusters) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index 6972abb..d995c80 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -75,7 +75,7 @@ import com.google.common.io.Closer;
 public class DistributedIndexServer extends AbstractDistributedIndexServer {
 
   private static final Log LOG = LogFactory.getLog(DistributedIndexServer.class);
-  private static final long _delay = TimeUnit.SECONDS.toMillis(10);
+  private static final long _delay = TimeUnit.SECONDS.toMillis(1);
   private static final AtomicLong _pauseWarmup = new AtomicLong();
   private static final Set<String> EMPTY = new HashSet<String>();
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java
index ac58229..cd3e0b1 100644
--- a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java
+++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java
@@ -85,6 +85,8 @@ public abstract class IndexSearcherCloseableSecureBase extends BlurSecureIndexSe
             Throwable cause = e.getCause();
             if (cause instanceof IOException) {
               throw (IOException) cause;
+            } else if (cause instanceof RuntimeException) {
+              throw (RuntimeException) cause;
             } else {
               throw new RuntimeException(cause);
             }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
index c55ce0a..56c5aeb 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
@@ -71,6 +71,7 @@ import org.apache.zookeeper.ZooKeeper;
 
 public abstract class TableAdmin implements Iface {
 
+  private static final long WAIT_BETWEEN_POLLS = 500;
   private static final Log LOG = LogFactory.getLog(TableAdmin.class);
   protected ZooKeeper _zookeeper;
   protected ClusterStatus _clusterStatus;
@@ -204,7 +205,7 @@ public abstract class TableAdmin implements Iface {
         return;
       }
       try {
-        Thread.sleep(3000);
+        Thread.sleep(WAIT_BETWEEN_POLLS);
       } catch (InterruptedException e) {
         LOG.error("Unknown error while enabling table [" + table + "]", e);
         throw new BException("Unknown error while enabling table [" + table + "]", e);
@@ -222,14 +223,7 @@ public abstract class TableAdmin implements Iface {
     LOG.info("Waiting for shards to engage on table [" + table + "]");
     while (true) {
       try {
-        Thread.sleep(3000);
-      } catch (InterruptedException e) {
-        LOG.error("Unknown error while engaging table [" + table + "]", e);
-        throw new BException("Unknown error while engaging table [" + table + "]", e);
-      }
-      try {
         Map<String, Map<String, ShardState>> shardServerLayoutState = shardServerLayoutState(table);
-
         int countNumberOfOpen = 0;
         int countNumberOfOpening = 0;
         for (Entry<String, Map<String, ShardState>> shardEntry : shardServerLayoutState.entrySet()) {
@@ -249,6 +243,12 @@ public abstract class TableAdmin implements Iface {
         if (countNumberOfOpen == shardCount && countNumberOfOpening == 0) {
           return;
         }
+        try {
+          Thread.sleep(WAIT_BETWEEN_POLLS);
+        } catch (InterruptedException e) {
+          LOG.error("Unknown error while engaging table [" + table + "]", e);
+          throw new BException("Unknown error while engaging table [" + table + "]", e);
+        }
       } catch (BlurException e) {
         LOG.info("Stilling waiting", e);
       } catch (TException e) {
@@ -264,12 +264,7 @@ public abstract class TableAdmin implements Iface {
   private void waitForTheTableToDisengage(String cluster, String table) throws BlurException, TException {
     LOG.info("Waiting for shards to disengage on table [" + table + "]");
     while (true) {
-      try {
-        Thread.sleep(3000);
-      } catch (InterruptedException e) {
-        LOG.error("Unknown error while disengaging table [" + table + "]", e);
-        throw new BException("Unknown error while disengaging table [" + table + "]", e);
-      }
+
       try {
         Map<String, Map<String, ShardState>> shardServerLayoutState = shardServerLayoutState(table);
 
@@ -277,13 +272,16 @@ public abstract class TableAdmin implements Iface {
         int countNumberOfClosing = 0;
         for (Entry<String, Map<String, ShardState>> shardEntry : shardServerLayoutState.entrySet()) {
           Map<String, ShardState> value = shardEntry.getValue();
-          for (ShardState state : value.values()) {
+          for (Entry<String, ShardState> e : value.entrySet()) {
+            String server = e.getKey();
+            ShardState state = e.getValue();
             if (state == ShardState.OPEN) {
               countNumberOfOpen++;
+              LOG.info("Shard [{0}] of table [{1}] from server [{2}] still reporting open.", shardEntry.getKey(),
+                  table, server);
             } else if (state == ShardState.CLOSING) {
               countNumberOfClosing++;
             } else if (state == ShardState.CLOSED) {
-              LOG.info("Shard [{0}] of table [{1}] now reporting closed.", shardEntry.getKey(), table);
             } else {
               LOG.warn("Unexpected state of [{0}] for shard [{1}].", state, shardEntry.getKey());
             }
@@ -294,6 +292,12 @@ public abstract class TableAdmin implements Iface {
         if (countNumberOfOpen == 0 && countNumberOfClosing == 0) {
           return;
         }
+        try {
+          Thread.sleep(WAIT_BETWEEN_POLLS);
+        } catch (InterruptedException e) {
+          LOG.error("Unknown error while disengaging table [" + table + "]", e);
+          throw new BException("Unknown error while disengaging table [" + table + "]", e);
+        }
       } catch (BlurException e) {
         LOG.info("Stilling waiting", e);
       } catch (TException e) {
@@ -309,7 +313,7 @@ public abstract class TableAdmin implements Iface {
         return;
       }
       try {
-        Thread.sleep(3000);
+        Thread.sleep(WAIT_BETWEEN_POLLS);
       } catch (InterruptedException e) {
         LOG.error("Unknown error while enabling table [" + table + "]", e);
         throw new BException("Unknown error while enabling table [" + table + "]", e);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
index 2cc805a..68f2718 100644
--- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
@@ -43,6 +43,7 @@ import java.util.Enumeration;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
@@ -182,7 +183,7 @@ public class MiniCluster {
     }
   }
 
-  public void stopMrMiniCluster() throws IOException {
+  public void shutdownMrMiniCluster() throws IOException {
     ScriptEngineManager manager = new ScriptEngineManager();
     ScriptEngine engine = manager.getEngineByName("js");
     if (useYarn()) {
@@ -219,7 +220,7 @@ public class MiniCluster {
     ScriptEngine engine = manager.getEngineByName("js");
 
     if (useYarn()) {
-      int nodeManagers = 2;
+      int nodeManagers = 4;
       Class<?> c = getClass();
       engine.put("c", c);
       engine.put("nodeManagers", nodeManagers);
@@ -503,6 +504,7 @@ public class MiniCluster {
 
         private Process _process;
         private AtomicBoolean _online = new AtomicBoolean();
+        private boolean _dead;
 
         @Override
         public void close() throws IOException {
@@ -555,8 +557,14 @@ public class MiniCluster {
         public void kill() {
           if (_process != null) {
             _process.destroy();
+            _dead = true;
           }
         }
+
+        @Override
+        public boolean isDead() {
+          return _dead;
+        }
       };
       Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
         @Override
@@ -581,6 +589,13 @@ public class MiniCluster {
     }
     return new MiniClusterServer() {
 
+      private boolean _dead;
+
+      @Override
+      public boolean isDead() {
+        return _dead;
+      }
+
       @Override
       public void close() throws IOException {
         thriftServer.close();
@@ -604,6 +619,7 @@ public class MiniCluster {
           String path = onlineShardsPath + "/" + nodeName;
           zk.delete(path, -1);
           zk.close();
+          _dead = true;
         } catch (Exception e) {
           throw new RuntimeException(e);
         } finally {
@@ -900,4 +916,16 @@ public class MiniCluster {
       throw new RuntimeException(e);
     }
   }
+
+  public void killRandomShardServer() {
+    Random random = new Random();
+    for (int i = 0; i < shards.size(); i++) {
+      int index = random.nextInt(shards.size());
+      MiniClusterServer miniClusterServer = shards.get(index);
+      if (!miniClusterServer.isDead()) {
+        miniClusterServer.kill();
+        return;
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/MiniClusterServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/MiniClusterServer.java b/blur-core/src/test/java/org/apache/blur/MiniClusterServer.java
index 99ccf49..d085774 100644
--- a/blur-core/src/test/java/org/apache/blur/MiniClusterServer.java
+++ b/blur-core/src/test/java/org/apache/blur/MiniClusterServer.java
@@ -20,6 +20,8 @@ import java.io.Closeable;
 
 public interface MiniClusterServer extends Closeable {
 
+  boolean isDead();
+  
   void kill();
 
   void start();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/analysis/ThriftFieldManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/analysis/ThriftFieldManagerTest.java b/blur-core/src/test/java/org/apache/blur/analysis/ThriftFieldManagerTest.java
deleted file mode 100644
index 3243006..0000000
--- a/blur-core/src/test/java/org/apache/blur/analysis/ThriftFieldManagerTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.blur.analysis;
-
-import java.io.IOException;
-
-import org.apache.blur.server.TableContext;
-import org.apache.blur.thirdparty.thrift_0_9_0.TException;
-import org.apache.blur.thrift.SuiteCluster;
-import org.apache.blur.thrift.generated.Blur.Iface;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-public class ThriftFieldManagerTest extends BaseFieldManagerTest {
-
-  private static boolean _managing;
-  private static TableDescriptor _tableDescriptor;
-
-  @BeforeClass
-  public static void startup() throws IOException, BlurException, TException {
-    if (!SuiteCluster.isClusterSetup()) {
-      SuiteCluster.setupMiniCluster();
-      _managing = true;
-    }
-  }
-
-  @AfterClass
-  public static void shutdown() throws IOException {
-    if (_managing) {
-      SuiteCluster.shutdownMiniCluster();
-    }
-  }
-
-  public Iface getClient() throws IOException {
-    return SuiteCluster.getClient();
-  }
-
-  @Before
-  public void setup() throws BlurException, TException, IOException {
-    _tableDescriptor = new TableDescriptor();
-    _tableDescriptor.setName("ThriftFieldManagerTest");
-    _tableDescriptor.setShardCount(1);
-    String fileSystemUri = SuiteCluster.getFileSystemUri();
-    _tableDescriptor.setTableUri(fileSystemUri + "/tables/ThriftFieldManagerTest");
-    Iface client = getClient();
-    client.createTable(_tableDescriptor);
-  }
-
-  @After
-  public void teardown() throws BlurException, TException, IOException {
-    Iface client = getClient();
-    String table = _tableDescriptor.getName();
-    client.disableTable(table);
-    client.removeTable(table, true);
-  }
-
-  @Override
-  protected FieldManager newFieldManager(boolean create) throws IOException {
-    TableContext tableContext = TableContext.create(_tableDescriptor, true, getClient());
-    return tableContext.getFieldManager();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/analysis/ThriftFieldManagerTestIT.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/analysis/ThriftFieldManagerTestIT.java b/blur-core/src/test/java/org/apache/blur/analysis/ThriftFieldManagerTestIT.java
new file mode 100644
index 0000000..0c4b959
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/analysis/ThriftFieldManagerTestIT.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.analysis;
+
+import java.io.IOException;
+
+import org.apache.blur.server.TableContext;
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.SuiteCluster;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+public class ThriftFieldManagerTestIT extends BaseFieldManagerTest {
+
+  private static TableDescriptor _tableDescriptor;
+
+  @BeforeClass
+  public static void startup() throws IOException, BlurException, TException {
+    SuiteCluster.setupMiniCluster(ThriftFieldManagerTestIT.class);
+  }
+
+  @AfterClass
+  public static void shutdown() throws IOException {
+    SuiteCluster.shutdownMiniCluster(ThriftFieldManagerTestIT.class);
+  }
+
+  public Iface getClient() throws IOException {
+    return SuiteCluster.getClient();
+  }
+
+  @Before
+  public void setup() throws BlurException, TException, IOException {
+    _tableDescriptor = new TableDescriptor();
+    _tableDescriptor.setName("ThriftFieldManagerTest");
+    _tableDescriptor.setShardCount(1);
+    String fileSystemUri = SuiteCluster.getFileSystemUri();
+    _tableDescriptor.setTableUri(fileSystemUri + "/tables/ThriftFieldManagerTest");
+    Iface client = getClient();
+    client.createTable(_tableDescriptor);
+  }
+
+  @After
+  public void teardown() throws BlurException, TException, IOException {
+    Iface client = getClient();
+    String table = _tableDescriptor.getName();
+    client.disableTable(table);
+    client.removeTable(table, true);
+  }
+
+  @Override
+  protected FieldManager newFieldManager(boolean create) throws IOException {
+    TableContext tableContext = TableContext.create(_tableDescriptor, true, getClient());
+    return tableContext.getFieldManager();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/command/ReconnectWhileCommandIsRunningIntTests.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/command/ReconnectWhileCommandIsRunningIntTests.java b/blur-core/src/test/java/org/apache/blur/command/ReconnectWhileCommandIsRunningIntTests.java
deleted file mode 100644
index a38c5ab..0000000
--- a/blur-core/src/test/java/org/apache/blur/command/ReconnectWhileCommandIsRunningIntTests.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.apache.blur.command;
-
-import static org.junit.Assert.assertNotNull;
-
-import java.io.IOException;
-
-import org.apache.blur.thirdparty.thrift_0_9_0.TException;
-import org.apache.blur.thrift.BaseClusterTest;
-import org.apache.blur.thrift.TableGen;
-import org.apache.blur.thrift.generated.BlurException;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-public class ReconnectWhileCommandIsRunningIntTests extends BaseClusterTest {
-
-//  @Test  Slow test
-  public void testWaitForSecondsIntTest() throws BlurException, TException, IOException, InterruptedException {
-    final String tableName = "testWaitForSecondsIntTest";
-    TableGen.define(tableName).cols("test", "col1").addRows(100, 20, "r1", "rec-###", "value").build(getClient());
-    WaitForSeconds command = new WaitForSeconds();
-    command.setTable(tableName);
-    assertNotNull(command.run(getClient()));
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/command/ReconnectWhileCommandIsRunningIntTestsIT.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/command/ReconnectWhileCommandIsRunningIntTestsIT.java b/blur-core/src/test/java/org/apache/blur/command/ReconnectWhileCommandIsRunningIntTestsIT.java
new file mode 100644
index 0000000..00238c7
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/command/ReconnectWhileCommandIsRunningIntTestsIT.java
@@ -0,0 +1,44 @@
+package org.apache.blur.command;
+
+import static org.junit.Assert.assertNotNull;
+
+import java.io.IOException;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.BaseClusterTest;
+import org.apache.blur.thrift.TableGen;
+import org.apache.blur.thrift.generated.BlurException;
+import org.junit.Test;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+public class ReconnectWhileCommandIsRunningIntTestsIT extends BaseClusterTest {
+  @Test
+  public void voidTest() {
+
+  }
+
+  // @Test Slow test
+  public void testWaitForSecondsIntTest() throws BlurException, TException, IOException, InterruptedException {
+    final String tableName = "testWaitForSecondsIntTest";
+    TableGen.define(tableName).cols("test", "col1").addRows(100, 20, "r1", "rec-###", "value").build(getClient());
+    WaitForSeconds command = new WaitForSeconds();
+    command.setTable(tableName);
+    assertNotNull(command.run(getClient()));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java b/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java
deleted file mode 100644
index deb930d..0000000
--- a/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java
+++ /dev/null
@@ -1,305 +0,0 @@
-package org.apache.blur.manager.clusterstatus;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.blur.MiniCluster;
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.utils.BlurUtil;
-import org.apache.blur.zookeeper.ZooKeeperClient;
-import org.apache.blur.zookeeper.ZooKeeperLockManager;
-import org.apache.blur.zookeeper.ZookeeperPathConstants;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
-import org.apache.zookeeper.server.quorum.QuorumPeerMain;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class ZookeeperClusterStatusTest {
-
-  private static final String TEST = "test";
-  private static final String DEFAULT = "default";
-
-  private static final Log LOG = LogFactory.getLog(ZookeeperClusterStatusTest.class);
-  private static MiniCluster miniCluster;
-  private ZooKeeper zooKeeper1;
-  private ZooKeeper zooKeeper2;
-  private ZookeeperClusterStatus clusterStatus1;
-  private ZookeeperClusterStatus clusterStatus2;
-
-  public static class QuorumPeerMainRun extends QuorumPeerMain {
-    @Override
-    public void initializeAndRun(String[] args) throws ConfigException, IOException {
-      super.initializeAndRun(args);
-    }
-  }
-
-  @BeforeClass
-  public static void setupOnce() throws InterruptedException, IOException, KeeperException {
-    miniCluster = new MiniCluster();
-    miniCluster.startZooKeeper("./target/zk_test", true);
-  }
-
-  @AfterClass
-  public static void teardownOnce() {
-    miniCluster.shutdownZooKeeper();
-  }
-
-  @Before
-  public void setup() throws KeeperException, InterruptedException, IOException {
-    zooKeeper1 = new ZooKeeperClient(miniCluster.getZkConnectionString(), 30000, new Watcher() {
-      @Override
-      public void process(WatchedEvent event) {
-
-      }
-    });
-    BlurUtil.setupZookeeper(zooKeeper1, DEFAULT);
-    zooKeeper2 = new ZooKeeperClient(miniCluster.getZkConnectionString(), 30000, new Watcher() {
-      @Override
-      public void process(WatchedEvent event) {
-
-      }
-    });
-    BlurUtil.setupZookeeper(zooKeeper1, DEFAULT);
-    BlurUtil.setupZookeeper(zooKeeper2, DEFAULT);
-    clusterStatus1 = new ZookeeperClusterStatus(zooKeeper1);
-    clusterStatus2 = new ZookeeperClusterStatus(zooKeeper2);
-  }
-
-  @After
-  public void teardown() throws InterruptedException, KeeperException {
-    clusterStatus1.close();
-    clusterStatus2.close();
-    rmr(zooKeeper1, "/blur");
-    zooKeeper1.close();
-    zooKeeper2.close();
-  }
-
-  private static void rmr(ZooKeeper zooKeeper, String path) throws KeeperException, InterruptedException {
-    List<String> children = zooKeeper.getChildren(path, false);
-    for (String c : children) {
-      rmr(zooKeeper, path + "/" + c);
-    }
-    zooKeeper.delete(path, -1);
-  }
-
-  @Test
-  public void testGetClusterList() {
-    LOG.warn("testGetClusterList");
-    List<String> clusterList = clusterStatus2.getClusterList(false);
-    assertEquals(Arrays.asList(DEFAULT), clusterList);
-  }
-
-  @Test
-  public void testSafeModeNoCache() throws KeeperException, InterruptedException {
-    String safemodePath = ZookeeperPathConstants.getSafemodePath(DEFAULT);
-    ZooKeeperLockManager zooKeeperLockManager = new ZooKeeperLockManager(zooKeeper1, safemodePath);
-    zooKeeperLockManager.lock(DEFAULT);
-    assertTrue(clusterStatus2.isInSafeMode(false, DEFAULT));
-    zooKeeperLockManager.unlock(DEFAULT);
-    assertFalse(clusterStatus2.isInSafeMode(false, DEFAULT));
-  }
-
-  @Test
-  public void testSafeModeCache() throws KeeperException, InterruptedException {
-    String safemodePath = ZookeeperPathConstants.getSafemodePath(DEFAULT);
-    ZooKeeperLockManager zooKeeperLockManager = new ZooKeeperLockManager(zooKeeper1, safemodePath);
-    zooKeeperLockManager.lock(DEFAULT);
-    new WaitForAnswerToBeCorrect(20L) {
-      @Override
-      public Object run() {
-        return clusterStatus2.isInSafeMode(false, DEFAULT);
-      }
-    }.test(true);
-    zooKeeperLockManager.unlock(DEFAULT);
-    new WaitForAnswerToBeCorrect(20L) {
-      @Override
-      public Object run() {
-        return clusterStatus2.isInSafeMode(false, DEFAULT);
-      }
-    }.test(false);
-  }
-
-  @Test
-  public void testGetClusterNoTable() {
-    LOG.warn("testGetCluster");
-    assertNull(clusterStatus2.getCluster(false, TEST));
-    assertNull(clusterStatus2.getCluster(true, TEST));
-  }
-
-  @Test
-  public void testGetClusterTable() throws KeeperException, InterruptedException {
-    LOG.warn("testGetCluster");
-    createTable(TEST);
-    assertEquals(DEFAULT, clusterStatus2.getCluster(false, TEST));
-    new WaitForAnswerToBeCorrect(20L) {
-      @Override
-      public Object run() {
-        return clusterStatus2.getCluster(true, TEST);
-      }
-    }.test(DEFAULT);
-  }
-
-  @Test
-  public void testGetTableList() throws KeeperException, InterruptedException {
-    testGetClusterTable();
-    assertEquals(Arrays.asList(TEST), clusterStatus2.getTableList(false));
-  }
-
-  @Test
-  public void testIsEnabledNoTable() {
-    try {
-      clusterStatus1.isEnabled(false, DEFAULT, "notable");
-      fail("should throw exception.");
-    } catch (RuntimeException e) {
-
-    }
-    try {
-      clusterStatus1.isEnabled(true, DEFAULT, "notable");
-      fail("should throw exception.");
-    } catch (RuntimeException e) {
-
-    }
-  }
-
-  @Test
-  public void testIsEnabledDisabledTable() throws KeeperException, InterruptedException {
-    createTable("disabledtable", false);
-    assertFalse(clusterStatus2.isEnabled(false, DEFAULT, "disabledtable"));
-    assertFalse(clusterStatus2.isEnabled(true, DEFAULT, "disabledtable"));
-  }
-
-  @Test
-  public void testIsEnabledEnabledTable() throws KeeperException, InterruptedException {
-    createTable("enabledtable", true);
-    assertTrue(clusterStatus2.isEnabled(false, DEFAULT, "enabledtable"));
-
-    new WaitForAnswerToBeCorrect(20L) {
-      @Override
-      public Object run() {
-        return clusterStatus2.isEnabled(true, DEFAULT, "enabledtable");
-      }
-    }.test(true);
-  }
-
-  @Test
-  public void testDisablingTableNoCache() throws KeeperException, InterruptedException {
-    createTable(TEST);
-    assertTrue(clusterStatus2.isEnabled(false, DEFAULT, TEST));
-    clusterStatus1.disableTable(DEFAULT, TEST);
-    assertFalse(clusterStatus2.isEnabled(false, DEFAULT, TEST));
-  }
-
-  @Test
-  public void testDisablingTableCache() throws KeeperException, InterruptedException {
-    createTable(TEST);
-    assertTrue(clusterStatus2.isEnabled(true, DEFAULT, TEST));
-    clusterStatus1.disableTable(DEFAULT, TEST);
-    new WaitForAnswerToBeCorrect(20L) {
-      @Override
-      public Object run() {
-        return clusterStatus2.isEnabled(true, DEFAULT, TEST);
-      }
-    }.test(false);
-  }
-
-  @Test
-  public void testEnablingTableNoCache() throws KeeperException, InterruptedException {
-    createTable(TEST, false);
-    assertFalse(clusterStatus2.isEnabled(false, DEFAULT, TEST));
-    clusterStatus1.enableTable(DEFAULT, TEST);
-    assertTrue(clusterStatus2.isEnabled(false, DEFAULT, TEST));
-  }
-
-  @Test
-  public void testEnablingTableCache() throws KeeperException, InterruptedException {
-    createTable(TEST, false);
-    assertFalse(clusterStatus2.isEnabled(true, DEFAULT, TEST));
-    clusterStatus1.enableTable(DEFAULT, TEST);
-    new WaitForAnswerToBeCorrect(20L) {
-      @Override
-      public Object run() {
-        return clusterStatus2.isEnabled(true, DEFAULT, TEST);
-      }
-    }.test(true);
-  }
-
-  private void createTable(String name) throws KeeperException, InterruptedException {
-    createTable(name, true);
-  }
-
-  private void createTable(String name, boolean enabled) throws KeeperException, InterruptedException {
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setName(name);
-    tableDescriptor.setTableUri("./target/tmp/zk_test_hdfs");
-    tableDescriptor.setEnabled(enabled);
-    clusterStatus1.createTable(tableDescriptor);
-    if (enabled) {
-      clusterStatus1.enableTable(tableDescriptor.getCluster(), name);
-    }
-  }
-
-  public abstract class WaitForAnswerToBeCorrect {
-
-    private long totalWaitTimeNanos;
-
-    public WaitForAnswerToBeCorrect(long totalWaitTimeMs) {
-      this.totalWaitTimeNanos = TimeUnit.MILLISECONDS.toNanos(totalWaitTimeMs);
-    }
-
-    public abstract Object run();
-
-    public void test(Object o) {
-      long start = System.nanoTime();
-      while (true) {
-        Object object = run();
-        if (object.equals(o) || object == o) {
-          return;
-        }
-        long now = System.nanoTime();
-        if (now - start > totalWaitTimeNanos) {
-          fail();
-        }
-        try {
-          Thread.sleep(1);
-        } catch (InterruptedException e) {
-          fail(e.getMessage());
-        }
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTestIT.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTestIT.java b/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTestIT.java
new file mode 100644
index 0000000..e5e7514
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTestIT.java
@@ -0,0 +1,307 @@
+package org.apache.blur.manager.clusterstatus;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.SuiteCluster;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.zookeeper.ZooKeeperClient;
+import org.apache.blur.zookeeper.ZooKeeperLockManager;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ZookeeperClusterStatusTestIT {
+
+  private static final String TEST = "test";
+  private static final String DEFAULT = "default";
+
+  private static final Log LOG = LogFactory.getLog(ZookeeperClusterStatusTestIT.class);
+  private ZooKeeper zooKeeper1;
+  private ZooKeeper zooKeeper2;
+  private ZookeeperClusterStatus clusterStatus1;
+  private ZookeeperClusterStatus clusterStatus2;
+
+  public static class QuorumPeerMainRun extends QuorumPeerMain {
+    @Override
+    public void initializeAndRun(String[] args) throws ConfigException, IOException {
+      super.initializeAndRun(args);
+    }
+  }
+
+  @BeforeClass
+  public static void startup() throws IOException, BlurException, TException {
+    SuiteCluster.setupMiniCluster(ZookeeperClusterStatusTestIT.class);
+  }
+
+  @AfterClass
+  public static void shutdown() throws IOException {
+    SuiteCluster.shutdownMiniCluster(ZookeeperClusterStatusTestIT.class);
+  }
+
+  @Before
+  public void setup() throws KeeperException, InterruptedException, IOException {
+    zooKeeper1 = new ZooKeeperClient(SuiteCluster.getZooKeeperConnStr(ZookeeperClusterStatusTestIT.class), 30000,
+        new Watcher() {
+          @Override
+          public void process(WatchedEvent event) {
+
+          }
+        });
+    BlurUtil.setupZookeeper(zooKeeper1, DEFAULT);
+    zooKeeper2 = new ZooKeeperClient(SuiteCluster.getZooKeeperConnStr(ZookeeperClusterStatusTestIT.class), 30000,
+        new Watcher() {
+          @Override
+          public void process(WatchedEvent event) {
+
+          }
+        });
+    BlurUtil.setupZookeeper(zooKeeper1, DEFAULT);
+    BlurUtil.setupZookeeper(zooKeeper2, DEFAULT);
+    clusterStatus1 = new ZookeeperClusterStatus(zooKeeper1);
+    clusterStatus2 = new ZookeeperClusterStatus(zooKeeper2);
+  }
+
+  @After
+  public void teardown() throws InterruptedException, KeeperException {
+    clusterStatus1.close();
+    clusterStatus2.close();
+    rmr(zooKeeper1, "/blur");
+    zooKeeper1.close();
+    zooKeeper2.close();
+  }
+
+  private static void rmr(ZooKeeper zooKeeper, String path) throws KeeperException, InterruptedException {
+    List<String> children = zooKeeper.getChildren(path, false);
+    for (String c : children) {
+      rmr(zooKeeper, path + "/" + c);
+    }
+    zooKeeper.delete(path, -1);
+  }
+
+  @Test
+  public void testGetClusterList() {
+    LOG.warn("testGetClusterList");
+    List<String> clusterList = clusterStatus2.getClusterList(false);
+    assertEquals(Arrays.asList(DEFAULT), clusterList);
+  }
+
+  @Test
+  public void testSafeModeNoCache() throws KeeperException, InterruptedException {
+    String safemodePath = ZookeeperPathConstants.getSafemodePath(DEFAULT);
+    ZooKeeperLockManager zooKeeperLockManager = new ZooKeeperLockManager(zooKeeper1, safemodePath);
+    zooKeeperLockManager.lock(DEFAULT);
+    assertTrue(clusterStatus2.isInSafeMode(false, DEFAULT));
+    zooKeeperLockManager.unlock(DEFAULT);
+    assertFalse(clusterStatus2.isInSafeMode(false, DEFAULT));
+  }
+
+  @Test
+  public void testSafeModeCache() throws KeeperException, InterruptedException {
+    String safemodePath = ZookeeperPathConstants.getSafemodePath(DEFAULT);
+    ZooKeeperLockManager zooKeeperLockManager = new ZooKeeperLockManager(zooKeeper1, safemodePath);
+    zooKeeperLockManager.lock(DEFAULT);
+    new WaitForAnswerToBeCorrect(20L) {
+      @Override
+      public Object run() {
+        return clusterStatus2.isInSafeMode(false, DEFAULT);
+      }
+    }.test(true);
+    zooKeeperLockManager.unlock(DEFAULT);
+    new WaitForAnswerToBeCorrect(20L) {
+      @Override
+      public Object run() {
+        return clusterStatus2.isInSafeMode(false, DEFAULT);
+      }
+    }.test(false);
+  }
+
+  @Test
+  public void testGetClusterNoTable() {
+    LOG.warn("testGetCluster");
+    assertNull(clusterStatus2.getCluster(false, TEST));
+    assertNull(clusterStatus2.getCluster(true, TEST));
+  }
+
+  @Test
+  public void testGetClusterTable() throws KeeperException, InterruptedException {
+    LOG.warn("testGetCluster");
+    createTable(TEST);
+    assertEquals(DEFAULT, clusterStatus2.getCluster(false, TEST));
+    new WaitForAnswerToBeCorrect(20L) {
+      @Override
+      public Object run() {
+        return clusterStatus2.getCluster(true, TEST);
+      }
+    }.test(DEFAULT);
+  }
+
+  @Test
+  public void testGetTableList() throws KeeperException, InterruptedException {
+    testGetClusterTable();
+    assertEquals(Arrays.asList(TEST), clusterStatus2.getTableList(false));
+  }
+
+  @Test
+  public void testIsEnabledNoTable() {
+    try {
+      clusterStatus1.isEnabled(false, DEFAULT, "notable");
+      fail("should throw exception.");
+    } catch (RuntimeException e) {
+
+    }
+    try {
+      clusterStatus1.isEnabled(true, DEFAULT, "notable");
+      fail("should throw exception.");
+    } catch (RuntimeException e) {
+
+    }
+  }
+
+  @Test
+  public void testIsEnabledDisabledTable() throws KeeperException, InterruptedException {
+    createTable("disabledtable", false);
+    assertFalse(clusterStatus2.isEnabled(false, DEFAULT, "disabledtable"));
+    assertFalse(clusterStatus2.isEnabled(true, DEFAULT, "disabledtable"));
+  }
+
+  @Test
+  public void testIsEnabledEnabledTable() throws KeeperException, InterruptedException {
+    createTable("enabledtable", true);
+    assertTrue(clusterStatus2.isEnabled(false, DEFAULT, "enabledtable"));
+
+    new WaitForAnswerToBeCorrect(20L) {
+      @Override
+      public Object run() {
+        return clusterStatus2.isEnabled(true, DEFAULT, "enabledtable");
+      }
+    }.test(true);
+  }
+
+  @Test
+  public void testDisablingTableNoCache() throws KeeperException, InterruptedException {
+    createTable(TEST);
+    assertTrue(clusterStatus2.isEnabled(false, DEFAULT, TEST));
+    clusterStatus1.disableTable(DEFAULT, TEST);
+    assertFalse(clusterStatus2.isEnabled(false, DEFAULT, TEST));
+  }
+
+  @Test
+  public void testDisablingTableCache() throws KeeperException, InterruptedException {
+    createTable(TEST);
+    assertTrue(clusterStatus2.isEnabled(true, DEFAULT, TEST));
+    clusterStatus1.disableTable(DEFAULT, TEST);
+    new WaitForAnswerToBeCorrect(20L) {
+      @Override
+      public Object run() {
+        return clusterStatus2.isEnabled(true, DEFAULT, TEST);
+      }
+    }.test(false);
+  }
+
+  @Test
+  public void testEnablingTableNoCache() throws KeeperException, InterruptedException {
+    createTable(TEST, false);
+    assertFalse(clusterStatus2.isEnabled(false, DEFAULT, TEST));
+    clusterStatus1.enableTable(DEFAULT, TEST);
+    assertTrue(clusterStatus2.isEnabled(false, DEFAULT, TEST));
+  }
+
+  @Test
+  public void testEnablingTableCache() throws KeeperException, InterruptedException {
+    createTable(TEST, false);
+    assertFalse(clusterStatus2.isEnabled(true, DEFAULT, TEST));
+    clusterStatus1.enableTable(DEFAULT, TEST);
+    new WaitForAnswerToBeCorrect(20L) {
+      @Override
+      public Object run() {
+        return clusterStatus2.isEnabled(true, DEFAULT, TEST);
+      }
+    }.test(true);
+  }
+
+  private void createTable(String name) throws KeeperException, InterruptedException {
+    createTable(name, true);
+  }
+
+  private void createTable(String name, boolean enabled) throws KeeperException, InterruptedException {
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName(name);
+    tableDescriptor.setTableUri("./target/tmp/zk_test_hdfs");
+    tableDescriptor.setEnabled(enabled);
+    clusterStatus1.createTable(tableDescriptor);
+    if (enabled) {
+      clusterStatus1.enableTable(tableDescriptor.getCluster(), name);
+    }
+  }
+
+  public abstract class WaitForAnswerToBeCorrect {
+
+    private long totalWaitTimeNanos;
+
+    public WaitForAnswerToBeCorrect(long totalWaitTimeMs) {
+      this.totalWaitTimeNanos = TimeUnit.MILLISECONDS.toNanos(totalWaitTimeMs);
+    }
+
+    public abstract Object run();
+
+    public void test(Object o) {
+      long start = System.nanoTime();
+      while (true) {
+        Object object = run();
+        if (object.equals(o) || object == o) {
+          return;
+        }
+        long now = System.nanoTime();
+        if (now - start > totalWaitTimeNanos) {
+          fail();
+        }
+        try {
+          Thread.sleep(1);
+        } catch (InterruptedException e) {
+          fail(e.getMessage());
+        }
+      }
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java b/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java
deleted file mode 100644
index 1f834d3..0000000
--- a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.blur.manager.indexserver;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-
-import org.apache.blur.MiniCluster;
-import org.apache.blur.zookeeper.ZooKeeperClient;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.data.Stat;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-@SuppressWarnings("unchecked")
-public class MasterBasedDistributedLayoutFactoryTest {
-  private static String path = "./target/test-zk-MasterBasedDistributedLayoutFactoryTest";
-  private static MiniCluster miniCluster;
-
-  private ZooKeeper _zooKeeper;
-  private String cluster = "default";
-
-  @BeforeClass
-  public static void startZooKeeper() throws IOException {
-    new File(path).mkdirs();
-    miniCluster = new MiniCluster();
-    miniCluster.startZooKeeper(path, true);
-  }
-
-  @AfterClass
-  public static void stopZooKeeper() throws InterruptedException {
-    miniCluster.shutdownZooKeeper();
-  }
-
-  @Before
-  public void setup() throws IOException, KeeperException, InterruptedException {
-    _zooKeeper = new ZooKeeperClient(miniCluster.getZkConnectionString(), 50000, new Watcher() {
-      @Override
-      public void process(WatchedEvent event) {
-
-      }
-    });
-    rmr(_zooKeeper, "/blur");
-  }
-
-  @After
-  public void teardown() throws InterruptedException {
-    _zooKeeper.close();
-  }
-
-  @Test
-  public void testDecreaseInServers() throws IOException, KeeperException, InterruptedException {
-    MasterBasedDistributedLayoutFactory factory = new MasterBasedDistributedLayoutFactory(_zooKeeper, cluster);
-
-    List<String> shardList = Lists.newArrayList("shard-0", "shard-1", "shard-2", "shard-3", "shard-4", "shard-5");
-    List<String> shardServerList = Lists.newArrayList("server-0", "server-1", "server-2", "server-3", "server-4", "server-5");
-    
-    String table = "t1";
-
-    DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList);
-    Map<String, String> expected1 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"),
-        e("shard-3", "server-3"), e("shard-4", "server-4"), e("shard-5", "server-5"));
-
-    Map<String, String> actual1 = new TreeMap<String, String>(layout1.getLayout());
-
-    assertEquals(expected1, actual1);
-
-    List<String> newShardServerList = Lists.newArrayList("server-0", "server-1", "server-2", "server-3");
-    
-    DistributedLayout layout2 = factory.createDistributedLayout(table, shardList, newShardServerList);
-
-    Map<String, String> expected2 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"),
-        e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1"));
-    Map<String, String> actual2 = new TreeMap<String, String>(layout2.getLayout());
-    assertEquals(expected2, actual2);
-  }
-
-  @Test
-  public void testIncreaseInServers() throws IOException, KeeperException, InterruptedException {
-    MasterBasedDistributedLayoutFactory factory = new MasterBasedDistributedLayoutFactory(_zooKeeper, cluster);
-
-    List<String> shardList = Lists.newArrayList("shard-0", "shard-1", "shard-2", "shard-3", "shard-4", "shard-5");
-    List<String> shardServerList = Lists.newArrayList("server-0", "server-1", "server-2", "server-3");
-    
-    String table = "t1";
-
-    DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList);
-    Map<String, String> expected1 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"),
-        e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1"));
-
-    Map<String, String> actual1 = new TreeMap<String, String>(layout1.getLayout());
-
-    assertEquals(expected1, actual1);
-
-    List<String> newShardServerList = Lists.newArrayList("server-0", "server-1", "server-2", "server-3", "server-4", "server-5");
-    
-    DistributedLayout layout2 = factory.createDistributedLayout(table, shardList, newShardServerList);
-
-    Map<String, String> expected2 = map(e("shard-0", "server-4"), e("shard-1", "server-5"), e("shard-2", "server-2"),
-        e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1"));
-
-    Map<String, String> actual2 = new TreeMap<String, String>(layout2.getLayout());
-    assertEquals(expected2, actual2);
-  }
-
-  private void rmr(ZooKeeper zooKeeper, String storagePath) throws KeeperException, InterruptedException {
-    Stat stat = zooKeeper.exists(storagePath, false);
-    if (stat == null) {
-      return;
-    }
-    List<String> children = zooKeeper.getChildren(storagePath, false);
-    for (String s : children) {
-      rmr(zooKeeper, storagePath + "/" + s);
-    }
-    zooKeeper.delete(storagePath, -1);
-  }
-
-  private static Map<String, String> map(Entry<String, String>... entries) {
-    Map<String, String> map = new TreeMap<String, String>();
-    for (Entry<String, String> e : entries) {
-      map.put(e.getKey(), e.getValue());
-    }
-    return map;
-  }
-
-  private static Entry<String, String> e(final String key, final String value) {
-    return new Entry<String, String>() {
-
-      @Override
-      public String getKey() {
-        return key;
-      }
-
-      @Override
-      public String getValue() {
-        return value;
-      }
-
-      @Override
-      public String setValue(String value) {
-        throw new RuntimeException("Not Supported");
-      }
-
-    };
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTestIT.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTestIT.java b/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTestIT.java
new file mode 100644
index 0000000..be9f011
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTestIT.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.manager.indexserver;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.SuiteCluster;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.zookeeper.ZooKeeperClient;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("unchecked")
+public class MasterBasedDistributedLayoutFactoryTestIT {
+
+  private ZooKeeper _zooKeeper;
+  private String cluster = "default";
+
+  @BeforeClass
+  public static void startup() throws IOException, BlurException, TException {
+    SuiteCluster.setupMiniCluster(MasterBasedDistributedLayoutFactoryTestIT.class);
+  }
+
+  @AfterClass
+  public static void shutdown() throws IOException {
+    SuiteCluster.shutdownMiniCluster(MasterBasedDistributedLayoutFactoryTestIT.class);
+  }
+
+  @Before
+  public void setup() throws IOException, KeeperException, InterruptedException {
+    _zooKeeper = new ZooKeeperClient(SuiteCluster.getZooKeeperConnStr(MasterBasedDistributedLayoutFactoryTestIT.class),
+        50000, new Watcher() {
+          @Override
+          public void process(WatchedEvent event) {
+
+          }
+        });
+    rmr(_zooKeeper, "/blur");
+  }
+
+  @After
+  public void teardown() throws InterruptedException {
+    _zooKeeper.close();
+  }
+
+  @Test
+  public void testDecreaseInServers() throws IOException, KeeperException, InterruptedException {
+    MasterBasedDistributedLayoutFactory factory = new MasterBasedDistributedLayoutFactory(_zooKeeper, cluster);
+
+    List<String> shardList = Lists.newArrayList("shard-0", "shard-1", "shard-2", "shard-3", "shard-4", "shard-5");
+    List<String> shardServerList = Lists.newArrayList("server-0", "server-1", "server-2", "server-3", "server-4",
+        "server-5");
+
+    String table = "t1";
+
+    DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList);
+    Map<String, String> expected1 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"),
+        e("shard-3", "server-3"), e("shard-4", "server-4"), e("shard-5", "server-5"));
+
+    Map<String, String> actual1 = new TreeMap<String, String>(layout1.getLayout());
+
+    assertEquals(expected1, actual1);
+
+    List<String> newShardServerList = Lists.newArrayList("server-0", "server-1", "server-2", "server-3");
+
+    DistributedLayout layout2 = factory.createDistributedLayout(table, shardList, newShardServerList);
+
+    Map<String, String> expected2 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"),
+        e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1"));
+    Map<String, String> actual2 = new TreeMap<String, String>(layout2.getLayout());
+    assertEquals(expected2, actual2);
+  }
+
+  @Test
+  public void testIncreaseInServers() throws IOException, KeeperException, InterruptedException {
+    MasterBasedDistributedLayoutFactory factory = new MasterBasedDistributedLayoutFactory(_zooKeeper, cluster);
+
+    List<String> shardList = Lists.newArrayList("shard-0", "shard-1", "shard-2", "shard-3", "shard-4", "shard-5");
+    List<String> shardServerList = Lists.newArrayList("server-0", "server-1", "server-2", "server-3");
+
+    String table = "t1";
+
+    DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList);
+    Map<String, String> expected1 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"),
+        e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1"));
+
+    Map<String, String> actual1 = new TreeMap<String, String>(layout1.getLayout());
+
+    assertEquals(expected1, actual1);
+
+    List<String> newShardServerList = Lists.newArrayList("server-0", "server-1", "server-2", "server-3", "server-4",
+        "server-5");
+
+    DistributedLayout layout2 = factory.createDistributedLayout(table, shardList, newShardServerList);
+
+    Map<String, String> expected2 = map(e("shard-0", "server-4"), e("shard-1", "server-5"), e("shard-2", "server-2"),
+        e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1"));
+
+    Map<String, String> actual2 = new TreeMap<String, String>(layout2.getLayout());
+    assertEquals(expected2, actual2);
+  }
+
+  private void rmr(ZooKeeper zooKeeper, String storagePath) throws KeeperException, InterruptedException {
+    Stat stat = zooKeeper.exists(storagePath, false);
+    if (stat == null) {
+      return;
+    }
+    List<String> children = zooKeeper.getChildren(storagePath, false);
+    for (String s : children) {
+      rmr(zooKeeper, storagePath + "/" + s);
+    }
+    zooKeeper.delete(storagePath, -1);
+  }
+
+  private static Map<String, String> map(Entry<String, String>... entries) {
+    Map<String, String> map = new TreeMap<String, String>();
+    for (Entry<String, String> e : entries) {
+      map.put(e.getKey(), e.getValue());
+    }
+    return map;
+  }
+
+  private static Entry<String, String> e(final String key, final String value) {
+    return new Entry<String, String>() {
+
+      @Override
+      public String getKey() {
+        return key;
+      }
+
+      @Override
+      public String getValue() {
+        return value;
+      }
+
+      @Override
+      public String setValue(String value) {
+        throw new RuntimeException("Not Supported");
+      }
+
+    };
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java b/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java
deleted file mode 100644
index ed5a301..0000000
--- a/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java
+++ /dev/null
@@ -1,261 +0,0 @@
-package org.apache.blur.manager.indexserver;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.blur.MiniCluster;
-import org.apache.blur.zookeeper.ZooKeeperClient;
-import org.apache.blur.zookeeper.ZooKeeperLockManager;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class SafeModeTest {
-
-  private static String path = "./target/test-zk";
-  private static MiniCluster miniCluster;
-
-  @BeforeClass
-  public static void startZooKeeper() throws IOException {
-    new File(path).mkdirs();
-    miniCluster = new MiniCluster();
-    miniCluster.startZooKeeper(path, true);
-  }
-
-  @AfterClass
-  public static void stopZooKeeper() throws InterruptedException {
-    miniCluster.shutdownZooKeeper();
-  }
-
-  private ZooKeeper zk;
-
-  @Before
-  public void setup() throws IOException {
-    zk = new ZooKeeperClient(miniCluster.getZkConnectionString(), 20000, new Watcher() {
-      @Override
-      public void process(WatchedEvent event) {
-
-      }
-    });
-  }
-
-  @After
-  public void teardown() throws KeeperException, InterruptedException {
-    rm(zk, "/testing");
-    zk.close();
-  }
-
-  @Test
-  public void testBasicStartup() throws IOException, InterruptedException, KeeperException {
-    List<AtomicReference<Throwable>> errors = new ArrayList<AtomicReference<Throwable>>();
-    List<AtomicLong> timeRegisteredLst = new ArrayList<AtomicLong>();
-    List<Thread> threads = new ArrayList<Thread>();
-    for (int i = 0; i < 32; i++) {
-      AtomicReference<Throwable> ref = new AtomicReference<Throwable>();
-      AtomicLong timeRegistered = new AtomicLong();
-      errors.add(ref);
-      timeRegisteredLst.add(timeRegistered);
-      threads.add(startThread(zk, "node" + i, ref, timeRegistered));
-      Thread.sleep(100);
-    }
-
-    boolean alive = true;
-    while (alive) {
-      alive = false;
-      for (Thread t : threads) {
-        t.join(1000);
-        if (t.isAlive()) {
-          System.out.println("Thread [" + t + "] has not finished.");
-          alive = true;
-        }
-      }
-    }
-
-    for (AtomicReference<Throwable> t : errors) {
-      Throwable throwable = t.get();
-      assertNull(throwable == null ? null : throwable.getMessage(), throwable);
-    }
-
-    long oldest = -1;
-    long newest = -1;
-    for (AtomicLong time : timeRegisteredLst) {
-      long l = time.get();
-      if (oldest == -1 || l < oldest) {
-        oldest = l;
-      }
-      if (newest == -1 || l > newest) {
-        newest = l;
-      }
-    }
-    assertTrue("newest [" + newest + "] oldest [" + oldest + "]", (newest - oldest) < TimeUnit.SECONDS.toMillis(5));
-    ZooKeeperLockManager zooKeeperLockManager = new ZooKeeperLockManager(zk, "/testing/safemode");
-    Thread.sleep(5000);
-    assertEquals(0, zooKeeperLockManager.getNumberOfLockNodesPresent(SafeMode.STARTUP));
-  }
-
-  @Test
-  public void testBasicStartupWithMinimum() throws IOException, InterruptedException, KeeperException {
-    List<AtomicReference<Throwable>> errors = new ArrayList<AtomicReference<Throwable>>();
-    List<AtomicLong> timeRegisteredLst = new ArrayList<AtomicLong>();
-    List<Thread> threads = new ArrayList<Thread>();
-    int nodesThatStart = 10;
-    int minimumNodeBeforeStarting = 16;
-    for (int i = 0; i < 32; i++) {
-      AtomicReference<Throwable> ref = new AtomicReference<Throwable>();
-      AtomicLong timeRegistered = new AtomicLong();
-      errors.add(ref);
-      timeRegisteredLst.add(timeRegistered);
-      if (i < nodesThatStart) {
-        threads.add(startThread(zk, "node" + i, ref, timeRegistered, 0L, minimumNodeBeforeStarting));
-      } else {
-        threads.add(startThread(zk, "node" + i, ref, timeRegistered, 10000L, minimumNodeBeforeStarting));
-      }
-      Thread.sleep(100);
-    }
-
-    boolean alive = true;
-    while (alive) {
-      alive = false;
-      for (Thread t : threads) {
-        t.join(1000);
-        if (t.isAlive()) {
-          System.out.println("Thread [" + t + "] has not finished.");
-          alive = true;
-        }
-      }
-    }
-
-    for (AtomicReference<Throwable> t : errors) {
-      Throwable throwable = t.get();
-      assertNull(throwable == null ? null : throwable.getMessage(), throwable);
-    }
-
-    long oldest = -1;
-    long newest = -1;
-    for (AtomicLong time : timeRegisteredLst) {
-      long l = time.get();
-      if (oldest == -1 || l < oldest) {
-        oldest = l;
-      }
-      if (newest == -1 || l > newest) {
-        newest = l;
-      }
-    }
-    assertTrue("newest [" + newest + "] oldest [" + oldest + "]", (newest - oldest) < TimeUnit.SECONDS.toMillis(5));
-    ZooKeeperLockManager zooKeeperLockManager = new ZooKeeperLockManager(zk, "/testing/safemode");
-    Thread.sleep(5000);
-    assertEquals(0, zooKeeperLockManager.getNumberOfLockNodesPresent(SafeMode.STARTUP));
-  }
-
-  @Test
-  public void testExtraNodeStartup() throws IOException, InterruptedException, KeeperException {
-    ZooKeeper zk = new ZooKeeper(miniCluster.getZkConnectionString(), 20000, new Watcher() {
-      @Override
-      public void process(WatchedEvent event) {
-
-      }
-    });
-
-    SafeMode setupSafeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
-        TimeUnit.SECONDS, 60, 0);
-    setupSafeMode.registerNode("node1", null);
-
-    SafeMode safeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
-        TimeUnit.SECONDS, 60, 0);
-    long s = System.nanoTime();
-    safeMode.registerNode("node101", null);
-    long e = System.nanoTime();
-
-    assertTrue((e - s) < TimeUnit.SECONDS.toNanos(1));
-    zk.close();
-  }
-
-  @Test
-  public void testSecondNodeStartup() throws IOException, InterruptedException, KeeperException {
-    ZooKeeper zk = new ZooKeeper(miniCluster.getZkConnectionString(), 20000, new Watcher() {
-      @Override
-      public void process(WatchedEvent event) {
-
-      }
-    });
-
-    SafeMode setupSafeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
-        TimeUnit.SECONDS, 60, 0);
-    setupSafeMode.registerNode("node10", null);
-
-    SafeMode safeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
-        TimeUnit.SECONDS, 15, 0);
-    try {
-      safeMode.registerNode("node10", null);
-      fail("should throw exception.");
-    } catch (Exception e) {
-    }
-    zk.close();
-  }
-
-  private Thread startThread(final ZooKeeper zk, final String node, final AtomicReference<Throwable> errorRef,
-      final AtomicLong timeRegistered) {
-    return startThread(zk, node, errorRef, timeRegistered, 0L, 0);
-  }
-
-  private Thread startThread(final ZooKeeper zk, final String node, final AtomicReference<Throwable> errorRef,
-      final AtomicLong timeRegistered, final long waitTime, final int minimumNumberOfNodes) {
-    Runnable runnable = new Runnable() {
-      @Override
-      public void run() {
-        try {
-          Thread.sleep(waitTime);
-          SafeMode safeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
-              TimeUnit.SECONDS, 60, minimumNumberOfNodes);
-          safeMode.registerNode(node, null);
-          timeRegistered.set(System.currentTimeMillis());
-        } catch (Throwable t) {
-          errorRef.set(t);
-        }
-      }
-    };
-    Thread thread = new Thread(runnable);
-    thread.start();
-    return thread;
-  }
-
-  private static void rm(ZooKeeper zk, String path) throws KeeperException, InterruptedException {
-    List<String> children = zk.getChildren(path, false);
-    for (String c : children) {
-      rm(zk, path + "/" + c);
-    }
-    zk.delete(path, -1);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d8756092/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTestIT.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTestIT.java b/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTestIT.java
new file mode 100644
index 0000000..c0f6591
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTestIT.java
@@ -0,0 +1,257 @@
+package org.apache.blur.manager.indexserver;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.SuiteCluster;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.zookeeper.ZooKeeperClient;
+import org.apache.blur.zookeeper.ZooKeeperLockManager;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class SafeModeTestIT {
+
+  @BeforeClass
+  public static void startup() throws IOException, BlurException, TException {
+    SuiteCluster.setupMiniCluster(SafeModeTestIT.class);
+  }
+
+  @AfterClass
+  public static void shutdown() throws IOException {
+    SuiteCluster.shutdownMiniCluster(SafeModeTestIT.class);
+  }
+
+  private ZooKeeper zk;
+
+  @Before
+  public void setup() throws IOException {
+    zk = new ZooKeeperClient(SuiteCluster.getZooKeeperConnStr(SafeModeTestIT.class), 20000, new Watcher() {
+      @Override
+      public void process(WatchedEvent event) {
+
+      }
+    });
+  }
+
+  @After
+  public void teardown() throws KeeperException, InterruptedException {
+    rm(zk, "/testing");
+    zk.close();
+  }
+
+  @Test
+  public void testBasicStartup() throws IOException, InterruptedException, KeeperException {
+    List<AtomicReference<Throwable>> errors = new ArrayList<AtomicReference<Throwable>>();
+    List<AtomicLong> timeRegisteredLst = new ArrayList<AtomicLong>();
+    List<Thread> threads = new ArrayList<Thread>();
+    for (int i = 0; i < 32; i++) {
+      AtomicReference<Throwable> ref = new AtomicReference<Throwable>();
+      AtomicLong timeRegistered = new AtomicLong();
+      errors.add(ref);
+      timeRegisteredLst.add(timeRegistered);
+      threads.add(startThread(zk, "node" + i, ref, timeRegistered));
+      Thread.sleep(100);
+    }
+
+    boolean alive = true;
+    while (alive) {
+      alive = false;
+      for (Thread t : threads) {
+        t.join(1000);
+        if (t.isAlive()) {
+          System.out.println("Thread [" + t + "] has not finished.");
+          alive = true;
+        }
+      }
+    }
+
+    for (AtomicReference<Throwable> t : errors) {
+      Throwable throwable = t.get();
+      assertNull(throwable == null ? null : throwable.getMessage(), throwable);
+    }
+
+    long oldest = -1;
+    long newest = -1;
+    for (AtomicLong time : timeRegisteredLst) {
+      long l = time.get();
+      if (oldest == -1 || l < oldest) {
+        oldest = l;
+      }
+      if (newest == -1 || l > newest) {
+        newest = l;
+      }
+    }
+    assertTrue("newest [" + newest + "] oldest [" + oldest + "]", (newest - oldest) < TimeUnit.SECONDS.toMillis(5));
+    ZooKeeperLockManager zooKeeperLockManager = new ZooKeeperLockManager(zk, "/testing/safemode");
+    Thread.sleep(5000);
+    assertEquals(0, zooKeeperLockManager.getNumberOfLockNodesPresent(SafeMode.STARTUP));
+  }
+
+  @Test
+  public void testBasicStartupWithMinimum() throws IOException, InterruptedException, KeeperException {
+    List<AtomicReference<Throwable>> errors = new ArrayList<AtomicReference<Throwable>>();
+    List<AtomicLong> timeRegisteredLst = new ArrayList<AtomicLong>();
+    List<Thread> threads = new ArrayList<Thread>();
+    int nodesThatStart = 10;
+    int minimumNodeBeforeStarting = 16;
+    for (int i = 0; i < 32; i++) {
+      AtomicReference<Throwable> ref = new AtomicReference<Throwable>();
+      AtomicLong timeRegistered = new AtomicLong();
+      errors.add(ref);
+      timeRegisteredLst.add(timeRegistered);
+      if (i < nodesThatStart) {
+        threads.add(startThread(zk, "node" + i, ref, timeRegistered, 0L, minimumNodeBeforeStarting));
+      } else {
+        threads.add(startThread(zk, "node" + i, ref, timeRegistered, 10000L, minimumNodeBeforeStarting));
+      }
+      Thread.sleep(100);
+    }
+
+    boolean alive = true;
+    while (alive) {
+      alive = false;
+      for (Thread t : threads) {
+        t.join(1000);
+        if (t.isAlive()) {
+          System.out.println("Thread [" + t + "] has not finished.");
+          alive = true;
+        }
+      }
+    }
+
+    for (AtomicReference<Throwable> t : errors) {
+      Throwable throwable = t.get();
+      assertNull(throwable == null ? null : throwable.getMessage(), throwable);
+    }
+
+    long oldest = -1;
+    long newest = -1;
+    for (AtomicLong time : timeRegisteredLst) {
+      long l = time.get();
+      if (oldest == -1 || l < oldest) {
+        oldest = l;
+      }
+      if (newest == -1 || l > newest) {
+        newest = l;
+      }
+    }
+    assertTrue("newest [" + newest + "] oldest [" + oldest + "]", (newest - oldest) < TimeUnit.SECONDS.toMillis(5));
+    ZooKeeperLockManager zooKeeperLockManager = new ZooKeeperLockManager(zk, "/testing/safemode");
+    Thread.sleep(5000);
+    assertEquals(0, zooKeeperLockManager.getNumberOfLockNodesPresent(SafeMode.STARTUP));
+  }
+
+  @Test
+  public void testExtraNodeStartup() throws IOException, InterruptedException, KeeperException {
+    ZooKeeper zk = new ZooKeeperClient(SuiteCluster.getZooKeeperConnStr(SafeModeTestIT.class), 20000, new Watcher() {
+      @Override
+      public void process(WatchedEvent event) {
+
+      }
+    });
+
+    SafeMode setupSafeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
+        TimeUnit.SECONDS, 60, 0);
+    setupSafeMode.registerNode("node1", null);
+
+    SafeMode safeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
+        TimeUnit.SECONDS, 60, 0);
+    long s = System.nanoTime();
+    safeMode.registerNode("node101", null);
+    long e = System.nanoTime();
+
+    assertTrue((e - s) < TimeUnit.SECONDS.toNanos(1));
+    zk.close();
+  }
+
+  @Test
+  public void testSecondNodeStartup() throws IOException, InterruptedException, KeeperException {
+    ZooKeeper zk = new ZooKeeperClient(SuiteCluster.getZooKeeperConnStr(SafeModeTestIT.class), 20000, new Watcher() {
+      @Override
+      public void process(WatchedEvent event) {
+
+      }
+    });
+
+    SafeMode setupSafeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
+        TimeUnit.SECONDS, 60, 0);
+    setupSafeMode.registerNode("node10", null);
+
+    SafeMode safeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
+        TimeUnit.SECONDS, 15, 0);
+    try {
+      safeMode.registerNode("node10", null);
+      fail("should throw exception.");
+    } catch (Exception e) {
+    }
+    zk.close();
+  }
+
+  private Thread startThread(final ZooKeeper zk, final String node, final AtomicReference<Throwable> errorRef,
+      final AtomicLong timeRegistered) {
+    return startThread(zk, node, errorRef, timeRegistered, 0L, 0);
+  }
+
+  private Thread startThread(final ZooKeeper zk, final String node, final AtomicReference<Throwable> errorRef,
+      final AtomicLong timeRegistered, final long waitTime, final int minimumNumberOfNodes) {
+    Runnable runnable = new Runnable() {
+      @Override
+      public void run() {
+        try {
+          Thread.sleep(waitTime);
+          SafeMode safeMode = new SafeMode(zk, "/testing/safemode", "/testing/nodepath", TimeUnit.SECONDS, 5,
+              TimeUnit.SECONDS, 60, minimumNumberOfNodes);
+          safeMode.registerNode(node, null);
+          timeRegistered.set(System.currentTimeMillis());
+        } catch (Throwable t) {
+          errorRef.set(t);
+        }
+      }
+    };
+    Thread thread = new Thread(runnable);
+    thread.start();
+    return thread;
+  }
+
+  private static void rm(ZooKeeper zk, String path) throws KeeperException, InterruptedException {
+    List<String> children = zk.getChildren(path, false);
+    for (String c : children) {
+      rm(zk, path + "/" + c);
+    }
+    zk.delete(path, -1);
+  }
+}


Mime
View raw message