incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From twilli...@apache.org
Subject [1/2] git commit: BLUR-428 - Create rooted ZK path if it doesn't exist
Date Wed, 06 May 2015 01:56:49 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master f2bc7b395 -> d55db9476


BLUR-428 - Create rooted ZK path if it doesn't exist


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/415ce5e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/415ce5e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/415ce5e0

Branch: refs/heads/master
Commit: 415ce5e0513cf8001ac78ba559a42a8d398bf1da
Parents: 4d934d7
Author: twilliams <twilliams@apache.org>
Authored: Tue May 5 21:56:12 2015 -0400
Committer: twilliams <twilliams@apache.org>
Committed: Tue May 5 21:56:12 2015 -0400

----------------------------------------------------------------------
 .../blur/thrift/ThriftBlurControllerServer.java |  9 +--
 .../blur/thrift/ThriftBlurShardServer.java      |  9 +--
 .../org/apache/blur/thrift/ThriftServer.java    | 30 ++++++++
 .../test/java/org/apache/blur/MiniCluster.java  | 14 ++--
 .../org/apache/blur/thrift/BaseClusterTest.java |  4 +
 .../org/apache/blur/thrift/SuiteCluster.java    |  4 +
 .../apache/blur/thrift/ThriftServerTest.java    | 78 ++++++++++++++++++++
 7 files changed, 127 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/415ce5e0/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index a122808..e46a791 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -131,17 +131,12 @@ public class ThriftBlurControllerServer extends ThriftServer {
     String nodeName = getNodeName(configuration, BLUR_CONTROLLER_HOSTNAME);
     nodeName = nodeName + ":" + instanceBindPort;
     configuration.set(BLUR_NODENAME, nodeName);
-    String zkConnectionStr = isEmpty(configuration.get(BLUR_ZOOKEEPER_CONNECTION), BLUR_ZOOKEEPER_CONNECTION);
+    
 
     BlurQueryChecker queryChecker = new BlurQueryChecker(configuration);
 
-    int sessionTimeout = configuration.getInt(BLUR_ZOOKEEPER_TIMEOUT, BLUR_ZOOKEEPER_TIMEOUT_DEFAULT);
-
-    final ZooKeeper zooKeeper = ZkUtils.newZooKeeper(zkConnectionStr, sessionTimeout);
-
+    final ZooKeeper zooKeeper = setupZookeeper(configuration, null);
     
-    BlurUtil.setupZookeeper(zooKeeper, null);
-
     final ZookeeperClusterStatus clusterStatus = new ZookeeperClusterStatus(zooKeeper, configuration,
config);
 
     int timeout = configuration.getInt(BLUR_CONTROLLER_SHARD_CONNECTION_TIMEOUT, 60000);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/415ce5e0/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index c4c6c29..7809ffa 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -193,17 +193,12 @@ public class ThriftBlurShardServer extends ThriftServer {
     String nodeNameHostName = getNodeName(configuration, BLUR_SHARD_HOSTNAME);
     String nodeName = nodeNameHostName + ":" + instanceBindPort;
     configuration.set(BLUR_NODENAME, nodeName);
-    String zkConnectionStr = isEmpty(configuration.get(BLUR_ZOOKEEPER_CONNECTION), BLUR_ZOOKEEPER_CONNECTION);
 
     BlurQueryChecker queryChecker = new BlurQueryChecker(configuration);
 
-    int sessionTimeout = configuration.getInt(BLUR_ZOOKEEPER_TIMEOUT, BLUR_ZOOKEEPER_TIMEOUT_DEFAULT);
-
-    final ZooKeeper zooKeeper = ZkUtils.newZooKeeper(zkConnectionStr, sessionTimeout);
-
     String cluster = configuration.get(BLUR_CLUSTER_NAME, BLUR_CLUSTER);
-    BlurUtil.setupZookeeper(zooKeeper, cluster);
-
+    
+    final ZooKeeper zooKeeper = setupZookeeper(configuration, cluster);
     final ZookeeperClusterStatus clusterStatus = new ZookeeperClusterStatus(zooKeeper, configuration,
config);
 
     BlurFilterCache filterCache = getFilterCache(configuration);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/415ce5e0/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
index 7070424..51451fe 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
@@ -26,6 +26,9 @@ import static org.apache.blur.utils.BlurConstants.BLUR_HDFS_TRACE_PATH;
 import static org.apache.blur.utils.BlurConstants.BLUR_HOME;
 import static org.apache.blur.utils.BlurConstants.BLUR_SERVER_SECURITY_FILTER_CLASS;
 import static org.apache.blur.utils.BlurConstants.BLUR_TMP_PATH;
+import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
+import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
+import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -76,7 +79,11 @@ import org.apache.blur.thrift.server.TThreadedSelectorServer.Args.AcceptPolicy;
 import org.apache.blur.trace.LogTraceStorage;
 import org.apache.blur.trace.TraceStorage;
 import org.apache.blur.trace.hdfs.HdfsTraceStorage;
+import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
 
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Gauge;
@@ -485,4 +492,27 @@ public class ThriftServer {
     }
     return result;
   }
+
+  public static ZooKeeper setupZookeeper(BlurConfiguration conf, String cluster) throws IOException,
+      InterruptedException, KeeperException {
+    String zkConnectionStr = conf.getExpected(BLUR_ZOOKEEPER_CONNECTION);
+    int sessionTimeout = conf.getInt(BLUR_ZOOKEEPER_TIMEOUT, BLUR_ZOOKEEPER_TIMEOUT_DEFAULT);
+    int slash = zkConnectionStr.indexOf('/');
+
+    if ((slash != -1) && (slash != zkConnectionStr.length()-1)) {
+      ZooKeeper rootZk = ZkUtils.newZooKeeper(zkConnectionStr.substring(0, slash), sessionTimeout);
+      String rootPath = zkConnectionStr.substring(slash, zkConnectionStr.length());
+
+      if (!ZkUtils.exists(rootZk, rootPath)) {
+        LOG.info("Rooted ZooKeeper path [{0}] did not exist, creating now.", rootPath);
+        ZkUtils.mkNodesStr(rootZk, rootPath);
+      }
+      rootZk.close();
+    }
+    ZooKeeper zooKeeper = ZkUtils.newZooKeeper(zkConnectionStr, sessionTimeout);
+
+    BlurUtil.setupZookeeper(zooKeeper, cluster);
+
+    return zooKeeper;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/415ce5e0/blur-core/src/test/java/org/apache/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
index 8db632f..867a7ec 100644
--- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
@@ -116,7 +116,7 @@ public class MiniCluster {
     miniCluster.startControllers(1, false, false);
     miniCluster.startShards(1, false, false);
 
-    try {
+//    try {
       Iface client = BlurClient.getClient(miniCluster.getControllerConnectionStr());
       miniCluster.createTable("test", client);
       long start = System.nanoTime();
@@ -136,12 +136,12 @@ public class MiniCluster {
         miniCluster.searchRow("test", i, client);
       }
 
-    } finally {
-      miniCluster.stopShards();
-      miniCluster.stopControllers();
-      miniCluster.shutdownZooKeeper();
-      miniCluster.shutdownDfs();
-    }
+//    } finally {
+//      miniCluster.stopShards();
+//      miniCluster.stopControllers();
+//      miniCluster.shutdownZooKeeper();
+//      miniCluster.shutdownDfs();
+//    }
   }
 
   public void startBlurCluster(String path, int controllerCount, int shardCount) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/415ce5e0/blur-core/src/test/java/org/apache/blur/thrift/BaseClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BaseClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BaseClusterTest.java
index 21287a3..2934f31 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BaseClusterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BaseClusterTest.java
@@ -61,4 +61,8 @@ public class BaseClusterTest {
   public Iface getClient() throws IOException {
     return SuiteCluster.getClient();
   }
+  
+  protected String getZkConnString() {
+    return SuiteCluster.getZooKeeperConnStr();
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/415ce5e0/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java b/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java
index 46b9173..26f7494 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java
@@ -75,4 +75,8 @@ public class SuiteCluster {
     }
   }
 
+  public static String getZooKeeperConnStr() {
+    return cluster.getZkConnectionString();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/415ce5e0/blur-core/src/test/java/org/apache/blur/thrift/ThriftServerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/ThriftServerTest.java b/blur-core/src/test/java/org/apache/blur/thrift/ThriftServerTest.java
new file mode 100644
index 0000000..e451c20
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/thrift/ThriftServerTest.java
@@ -0,0 +1,78 @@
+package org.apache.blur.thrift;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.blur.BlurConfiguration;
+import org.apache.blur.utils.BlurConstants;
+import org.apache.blur.zookeeper.ZkUtils;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Test;
+
+public class ThriftServerTest extends BaseClusterTest {
+
+  @Test
+  public void testRootedZkPath() throws IOException, KeeperException, InterruptedException
{
+    ZooKeeper zk = null;
+    ZooKeeper nonRooted = null;
+    try {
+      BlurConfiguration conf = new BlurConfiguration();
+      conf.set(BlurConstants.BLUR_ZOOKEEPER_CONNECTION, getZkConnString() + "/rooted");
+      zk = ThriftServer.setupZookeeper(conf, "default");
+
+      assertNotNull("Should be addressable via rooted path",
+          zk.exists(ZookeeperPathConstants.getClusterPath("default"), false));
+
+      nonRooted = ZkUtils.newZooKeeper(getZkConnString(), 10000);
+      assertTrue("Should really be rooted.",
+          ZkUtils.exists(nonRooted, "/rooted" + ZookeeperPathConstants.getClusterPath("default")));
+    } finally {
+      if (zk != null) {
+        zk.close();
+      }
+      if(nonRooted != null) {
+        nonRooted.close();
+      }
+    }
+  }
+  @Test
+  public void testNonRootedZkPathWithSlash() throws IOException, KeeperException, InterruptedException
{
+    ZooKeeper zk = null;
+    ZooKeeper nonRooted = null;
+    try {
+      BlurConfiguration conf = new BlurConfiguration();
+      conf.set(BlurConstants.BLUR_ZOOKEEPER_CONNECTION, getZkConnString() + "/");
+      zk = ThriftServer.setupZookeeper(conf, "default");
+
+      nonRooted = ZkUtils.newZooKeeper(getZkConnString(), 10000);
+      assertTrue("Should really be rooted.",
+          ZkUtils.exists(nonRooted,  ZookeeperPathConstants.getClusterPath("default")));
+    } finally {
+      if (zk != null) {
+        zk.close();
+      }
+      if(nonRooted != null) {
+        nonRooted.close();
+      }
+    }
+  }
+}


Mime
View raw message