incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From twilli...@apache.org
Subject [1/4] git commit: initial cut at randomizing things - still a wip
Date Wed, 23 Jul 2014 20:07:19 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/blur-342a [created] 9acdad258


initial cut at randomizing things - still a wip


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d19cdf2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d19cdf2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d19cdf2a

Branch: refs/heads/blur-342a
Commit: d19cdf2a842ceee2de5416fe6f47731462c59b6d
Parents: 355d14f
Author: Tim <twilliams@apache.org>
Authored: Wed Jul 16 16:41:20 2014 -0400
Committer: Tim <twilliams@apache.org>
Committed: Wed Jul 23 16:04:06 2014 -0400

----------------------------------------------------------------------
 .../clusterstatus/ZookeeperClusterStatus.java   |  1 +
 .../clusterstatus/ZookeeperPathConstants.java   | 74 --------------------
 .../indexserver/DistributedIndexServer.java     |  2 +-
 .../MasterBasedDistributedLayoutFactory.java    |  2 +-
 .../blur/thrift/BlurControllerServer.java       |  2 +-
 .../java/org/apache/blur/thrift/Shutdown.java   |  2 +-
 .../blur/thrift/ThriftBlurControllerServer.java | 24 +++----
 .../blur/thrift/ThriftBlurShardServer.java      | 24 +++----
 .../java/org/apache/blur/utils/BlurUtil.java    |  2 +-
 .../test/java/org/apache/blur/MiniCluster.java  | 14 +++-
 .../ZookeeperClusterStatusTest.java             |  1 +
 .../main/java/org/apache/blur/shell/Main.java   | 69 ++++--------------
 .../java/org/apache/blur/thrift/BlurClient.java | 45 ++++++++++++
 .../java/org/apache/blur/BlurConfiguration.java |  8 +++
 .../blur/zookeeper/ZookeeperPathConstants.java  | 74 ++++++++++++++++++++
 .../src/main/resources/blur-default.properties  |  4 +-
 16 files changed, 186 insertions(+), 162 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
index df45b73..bf10b8c 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
@@ -48,6 +48,7 @@ import org.apache.blur.zookeeper.WatchChildren.OnChange;
 import org.apache.blur.zookeeper.WatchNodeData;
 import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.blur.zookeeper.ZooKeeperLockManager;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
deleted file mode 100644
index a8099c1..0000000
--- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
+++ /dev/null
@@ -1,74 +0,0 @@
-package org.apache.blur.manager.clusterstatus;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-public class ZookeeperPathConstants {
-
-  public static String getBasePath() {
-    return "/blur";
-  }
-
-  public static String getClusterPath(String cluster) {
-    return getClustersPath() + "/" + cluster;
-  }
-
-  public static String getClustersPath() {
-    return getBasePath() + "/clusters";
-  }
-
-  public static String getOnlineControllersPath() {
-    return getBasePath() + "/online-controller-nodes";
-  }
-
-  public static String getControllersPath() {
-    return getBasePath() + "/controller-nodes";
-  }
-
-  public static String getOnlineShardsPath(String cluster) {
-    return getClusterPath(cluster) + "/online-nodes";
-  }
-
-  public static String getTablesPath(String cluster) {
-    return getClusterPath(cluster) + "/tables";
-  }
-
-  public static String getTablePath(String cluster, String table) {
-    return getTablesPath(cluster) + "/" + table;
-  }
-
-  public static String getSafemodePath(String cluster) {
-    return getClusterPath(cluster) + "/safemode";
-  }
-  
-  public static String getShutdownPath(String cluster) {
-    return getClusterPath(cluster) + "/shutdown";
-  }
-
-  public static String getRegisteredShardsPath(String cluster) {
-    return getClusterPath(cluster) + "/registered-nodes";
-  }
-
-  public static String getShardLayoutPath(String cluster) {
-    return getClusterPath(cluster) + "/layout";
-  }
-
-  public static String getShardLayoutPathLocks(String cluster) {
-    return getShardLayoutPath(cluster) + "/locks";
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index 59d3f68..351d8fb 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -42,7 +42,6 @@ import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.BlurFilterCache;
 import org.apache.blur.manager.clusterstatus.ClusterStatus;
 import org.apache.blur.manager.clusterstatus.ClusterStatus.Action;
-import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.manager.writer.BlurIndexCloser;
 import org.apache.blur.manager.writer.BlurIndexReadOnly;
@@ -59,6 +58,7 @@ import org.apache.blur.thrift.generated.ShardState;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.zookeeper.WatchChildren;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
 import org.apache.blur.zookeeper.WatchChildren.OnChange;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
index 319ccf2..e7ad6fa 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
@@ -38,9 +38,9 @@ import java.util.concurrent.ConcurrentMap;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
-import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
 import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.blur.zookeeper.ZooKeeperLockManager;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index a1cec15..a0453bc 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -53,7 +53,6 @@ import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.BlurPartitioner;
 import org.apache.blur.manager.BlurQueryChecker;
 import org.apache.blur.manager.IndexManager;
-import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
 import org.apache.blur.manager.indexserver.DistributedLayout;
 import org.apache.blur.manager.indexserver.DistributedLayoutFactory;
 import org.apache.blur.manager.indexserver.DistributedLayoutFactoryImpl;
@@ -100,6 +99,7 @@ import org.apache.blur.utils.ForkJoin;
 import org.apache.blur.utils.ForkJoin.Merger;
 import org.apache.blur.utils.ForkJoin.ParallelCall;
 import org.apache.blur.zookeeper.WatchChildren;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
 import org.apache.blur.zookeeper.WatchChildren.OnChange;
 import org.apache.blur.zookeeper.WatchNodeExistance;
 import org.apache.zookeeper.CreateMode;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java b/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java
index 5f082e0..737794b 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/Shutdown.java
@@ -25,8 +25,8 @@ import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT
 import java.io.IOException;
 
 import org.apache.blur.BlurConfiguration;
-import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
 import org.apache.blur.zookeeper.ZkUtils;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index 76897f6..1297447 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -84,7 +84,7 @@ public class ThriftBlurControllerServer extends ThriftServer {
       ReporterSetup.setupReporters(configuration);
       MemoryReporter.enable();
       setupJvmMetrics();
-      ThriftServer server = createServer(serverIndex, configuration, false);
+      ThriftServer server = createServer(serverIndex, configuration);
       server.start();
     } catch (Throwable t) {
       t.printStackTrace();
@@ -92,25 +92,25 @@ public class ThriftBlurControllerServer extends ThriftServer {
     }
   }
 
-  public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration,
boolean randomPort)
+  public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration)
       throws Exception {
     Thread.setDefaultUncaughtExceptionHandler(new SimpleUncaughtExceptionHandler());
     String bindAddress = configuration.get(BLUR_CONTROLLER_BIND_ADDRESS);
-    int bindPort = configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1);
-    bindPort += serverIndex;
-    if (randomPort) {
-      bindPort = 0;
+    int configBindPort = configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1);
+    int instanceBindPort = configBindPort + serverIndex;
+    if (configBindPort == 0) {
+      instanceBindPort = 0;
     }
-    TNonblockingServerSocket tNonblockingServerSocket = ThriftServer.getTNonblockingServerSocket(bindAddress,
bindPort);
-    if (randomPort) {
-      bindPort = tNonblockingServerSocket.getServerSocket().getLocalPort();
+    TNonblockingServerSocket tNonblockingServerSocket = ThriftServer.getTNonblockingServerSocket(bindAddress,
instanceBindPort);
+    if (configBindPort == 0) {
+      instanceBindPort = tNonblockingServerSocket.getServerSocket().getLocalPort();
     }
 
-    LOG.info("Controller Server using index [{0}] bind address [{1}] random port assignment
[{2}]", serverIndex,
-        bindAddress + ":" + bindPort, randomPort);
+    LOG.info("Controller Server using index [{0}] bind address [{1}]", serverIndex,
+        bindAddress + ":" + instanceBindPort);
 
     String nodeName = getNodeName(configuration, BLUR_CONTROLLER_HOSTNAME);
-    nodeName = nodeName + ":" + bindPort;
+    nodeName = nodeName + ":" + instanceBindPort;
     String zkConnectionStr = isEmpty(configuration.get(BLUR_ZOOKEEPER_CONNECTION), BLUR_ZOOKEEPER_CONNECTION);
 
     BlurQueryChecker queryChecker = new BlurQueryChecker(configuration);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index cd39485..52bd2ff 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -121,7 +121,7 @@ public class ThriftBlurShardServer extends ThriftServer {
       setupJvmMetrics();
       // make this configurable
       GCWatcher.init(0.75);
-      ThriftServer server = createServer(serverIndex, configuration, false);
+      ThriftServer server = createServer(serverIndex, configuration);
       server.start();
     } catch (Throwable t) {
       t.printStackTrace();
@@ -129,21 +129,21 @@ public class ThriftBlurShardServer extends ThriftServer {
     }
   }
 
-  public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration,
boolean randomPort)
+  public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration)
       throws Exception {
     Configuration config = new Configuration();
     TableContext.setSystemBlurConfiguration(configuration);
     TableContext.setSystemConfiguration(config);
 
     String bindAddress = configuration.get(BLUR_SHARD_BIND_ADDRESS);
-    int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1);
-    bindPort += serverIndex;
-    if (randomPort) {
-      bindPort = 0;
+    int configBindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1);
+    int instanceBindPort = configBindPort + serverIndex;
+    if (configBindPort == 0) {
+      instanceBindPort = 0;
     }
-    TNonblockingServerSocket tNonblockingServerSocket = ThriftServer.getTNonblockingServerSocket(bindAddress,
bindPort);
-    if (randomPort) {
-      bindPort = tNonblockingServerSocket.getServerSocket().getLocalPort();
+    TNonblockingServerSocket tNonblockingServerSocket = ThriftServer.getTNonblockingServerSocket(bindAddress,
instanceBindPort);
+    if (configBindPort == 0) {
+      instanceBindPort = tNonblockingServerSocket.getServerSocket().getLocalPort();
     }
 
     int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT));
@@ -180,11 +180,11 @@ public class ThriftBlurShardServer extends ThriftServer {
     } else {
       throw new RuntimeException("Unknown block cache version [" + blockCacheVersion + "]
can be [v1,v2]");
     }
-    LOG.info("Shard Server using index [{0}] bind address [{1}] random port assignment [{2}]",
serverIndex, bindAddress
-        + ":" + bindPort, randomPort);
+    LOG.info("Shard Server using index [{0}] bind address [{1}]", serverIndex, bindAddress
+        + ":" + instanceBindPort);
 
     String nodeNameHostName = getNodeName(configuration, BLUR_SHARD_HOSTNAME);
-    String nodeName = nodeNameHostName + ":" + bindPort;
+    String nodeName = nodeNameHostName + ":" + instanceBindPort;
     String zkConnectionStr = isEmpty(configuration.get(BLUR_ZOOKEEPER_CONNECTION), BLUR_ZOOKEEPER_CONNECTION);
 
     BlurQueryChecker queryChecker = new BlurQueryChecker(configuration);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index 52fd09f..b57573d 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -68,7 +68,6 @@ import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.search.PrimeDocCache;
 import org.apache.blur.manager.BlurHighlighter;
-import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
 import org.apache.blur.manager.results.BlurResultComparator;
 import org.apache.blur.manager.results.BlurResultIterable;
 import org.apache.blur.manager.results.BlurResultPeekableIteratorComparator;
@@ -104,6 +103,7 @@ import org.apache.blur.trace.Trace;
 import org.apache.blur.trace.Tracer;
 import org.apache.blur.user.User;
 import org.apache.blur.user.UserContext;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
index 3a2f826..f188d90 100644
--- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
@@ -24,6 +24,8 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COU
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
+import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_BIND_PORT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT;
 
 import java.io.File;
 import java.io.IOException;
@@ -42,7 +44,6 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
-import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
 import org.apache.blur.store.buffer.BufferStore;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransportException;
@@ -63,6 +64,7 @@ import org.apache.blur.thrift.util.BlurThriftHelper;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.zookeeper.ZkMiniCluster;
 import org.apache.blur.zookeeper.ZooKeeperClient;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -262,9 +264,12 @@ public class MiniCluster {
 
   public void startControllers(BlurConfiguration configuration, int num, boolean randomPort)
{
     BlurConfiguration localConf = getBlurConfiguration(configuration);
+    if(randomPort) {
+    	localConf.setInt(BLUR_CONTROLLER_BIND_PORT, 0);
+    }
     for (int i = 0; i < num; i++) {
       try {
-        ThriftServer server = ThriftBlurControllerServer.createServer(i, localConf, randomPort);
+        ThriftServer server = ThriftBlurControllerServer.createServer(i, localConf);
         controllers.add(server);
         startServer(server);
       } catch (Exception e) {
@@ -281,6 +286,9 @@ public class MiniCluster {
 
   public void startShards(final BlurConfiguration configuration, int num, final boolean randomPort)
{
     final BlurConfiguration localConf = getBlurConfiguration(configuration);
+    if(randomPort) {
+    	localConf.setInt(BLUR_SHARD_BIND_PORT, 0);
+    }
     ExecutorService executorService = Executors.newFixedThreadPool(num);
     List<Future<ThriftServer>> futures = new ArrayList<Future<ThriftServer>>();
     for (int i = 0; i < num; i++) {
@@ -288,7 +296,7 @@ public class MiniCluster {
       futures.add(executorService.submit(new Callable<ThriftServer>() {
         @Override
         public ThriftServer call() throws Exception {
-          return ThriftBlurShardServer.createServer(index, localConf, randomPort);
+          return ThriftBlurShardServer.createServer(index, localConf);
         }
       }));
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java
b/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java
index d5236f8..deb930d 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java
@@ -35,6 +35,7 @@ import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.zookeeper.ZooKeeperClient;
 import org.apache.blur.zookeeper.ZooKeeperLockManager;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-shell/src/main/java/org/apache/blur/shell/Main.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/Main.java b/blur-shell/src/main/java/org/apache/blur/shell/Main.java
index 8e9bfd3..b7654c9 100644
--- a/blur-shell/src/main/java/org/apache/blur/shell/Main.java
+++ b/blur-shell/src/main/java/org/apache/blur/shell/Main.java
@@ -99,7 +99,7 @@ public class Main {
 
   private static class CliShellOptions {
 
-    private String _controllerConnectionString;
+    private Iface _client;
     private boolean _shell;
     private String[] _args;
     private String _command;
@@ -108,8 +108,8 @@ public class Main {
       return _shell;
     }
 
-    public String getControllerConnectionString() {
-      return _controllerConnectionString;
+    public Iface getClient() {
+      return _client;
     }
 
     public String getCommand() {
@@ -120,8 +120,8 @@ public class Main {
       return _args;
     }
 
-    public void setControllerConnectionString(String controllerConnectionString) {
-      this._controllerConnectionString = controllerConnectionString;
+    public void setClient(Iface client) {
+      this._client = client;
     }
 
     public void setShell(boolean shell) {
@@ -550,12 +550,12 @@ public class Main {
     }
 
     try {
-      Blur.Iface client = BlurClient.getClient(cliShellOptions.getControllerConnectionString());
+      Blur.Iface client = cliShellOptions.getClient();
       if (cliShellOptions.isShell()) {
         ConsoleReader reader = new ConsoleReader();
         PrintWriter out = new PrintWriter(reader.getOutput());
         setConsoleReader(commands, reader);
-        setPrompt(client, reader, cliShellOptions.getControllerConnectionString(), out);
+        setPrompt(client, reader, out);
 
         List<Completer> completors = new LinkedList<Completer>();
 
@@ -621,7 +621,7 @@ public class Main {
                   out.println("Last command took " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime()
- start) + "ms");
                 }
               }
-              setPrompt(client, reader, cliShellOptions.getControllerConnectionString(),
out);
+              setPrompt(client, reader, out);
             }
           }
         } finally {
@@ -704,18 +704,13 @@ public class Main {
     builder.put(command.name(), command);
   }
 
-  private static void setPrompt(Iface client, ConsoleReader reader, String connectionStr,
PrintWriter out)
+  private static void setPrompt(Iface client, ConsoleReader reader, PrintWriter out)
       throws BlurException, TException, CommandException, IOException {
     List<String> shardClusterList;
     try {
-      shardClusterList = BlurClientManager.execute(connectionStr, new BlurCommand<List<String>>()
{
-        @Override
-        public List<String> call(Client client) throws BlurException, TException {
-          return client.shardClusterList();
-        }
-      }, 0, 0, 0);
-    } catch (BadConnectionException e) {
-      out.println(e.getMessage() + " Connection (" + connectionStr + ")");
+      shardClusterList = client.shardClusterList();
+    } catch (BlurException e) {
+      out.println("Unable to retrieve cluster information - " + e.getMessage());
       out.flush();
       if (debug) {
         e.printStackTrace(out);
@@ -751,13 +746,7 @@ public class Main {
   private static CliShellOptions getCliShellOptions(String[] args) throws IOException {
     CliShellOptions cliShellOptions = new CliShellOptions();
     if (args.length == 0) {
-      String controllerConnectionString = loadControllerConnectionString();
-      if (controllerConnectionString == null) {
-        System.err
-            .println("Could not locate controller connection string in the blu-site.properties
file and it was not passed in via command line args.");
-        return null;
-      }
-      cliShellOptions.setControllerConnectionString(controllerConnectionString);
+      cliShellOptions.setClient(BlurClient.getClient());
       cliShellOptions.setShell(true);
       return cliShellOptions;
     } else {
@@ -773,7 +762,7 @@ public class Main {
               + message + "]");
           return null;
         }
-        cliShellOptions.setControllerConnectionString(arg0);
+        cliShellOptions.setClient(BlurClient.getClient(arg0));
         if (args.length > 1) {
           // there's might be a command after the connection string
           cliShellOptions.setShell(false);
@@ -794,13 +783,7 @@ public class Main {
           return cliShellOptions;
         }
       } else {
-        String controllerConnectionString = loadControllerConnectionString();
-        if (controllerConnectionString == null) {
-          System.err
-              .println("Could not locate controller connection string in the blur-site.properties
file and it was not passed in via command line args.");
-          return null;
-        }
-        cliShellOptions.setControllerConnectionString(controllerConnectionString);
+        cliShellOptions.setClient(BlurClient.getClient());
         // command was found at arg0
         cliShellOptions.setShell(false);
         cliShellOptions.setArgs(args);
@@ -816,26 +799,4 @@ public class Main {
     }
   }
 
-  private static String loadControllerConnectionString() throws IOException {
-    StringBuilder builder = new StringBuilder();
-    InputStream inputStream = Main.class.getResourceAsStream("/controllers");
-    if (inputStream == null) {
-      return null;
-    }
-    BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
-    String line;
-    while ((line = bufferedReader.readLine()) != null) {
-      if (builder.length() != 0) {
-        builder.append(',');
-      }
-      String trim = line.trim();
-      if (trim.startsWith("#")) {
-        continue;
-      }
-      builder.append(trim).append(":40010");
-    }
-    bufferedReader.close();
-    return builder.toString();
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java b/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java
index 7d5e1d2..ad4813a 100644
--- a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClient.java
@@ -2,6 +2,7 @@ package org.apache.blur.thrift;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
+
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
@@ -16,6 +17,11 @@ package org.apache.blur.thrift;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
+import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
+import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT;
+
+import java.io.IOException;
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -23,11 +29,19 @@ import java.lang.reflect.Proxy;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.blur.BlurConfiguration;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.commands.BlurCommand;
 import org.apache.blur.thrift.generated.Blur.Client;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.zookeeper.ZkUtils;
+import org.apache.blur.zookeeper.ZookeeperPathConstants;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+
+
 
 public class BlurClient {
 
@@ -77,6 +91,20 @@ public class BlurClient {
     }
 
   }
+  
+  public static Iface getClient() {
+	try {
+	  return getClient(new BlurConfiguration());
+	} catch (IOException e) {
+		throw new RuntimeException("Unable to load configurations.", e);
+	}
+  }
+  
+  public static Iface getClient(BlurConfiguration conf) {
+	List<String> onlineControllers = getOnlineControllers(conf);
+	  
+	return getClient(StringUtils.join(onlineControllers, ","));
+  }
 
   /**
    * Returns a client interface to Blur based on the connectionStr.
@@ -123,5 +151,22 @@ public class BlurClient {
     return (Iface) Proxy.newProxyInstance(Iface.class.getClassLoader(), new Class[] { Iface.class
},
         new BlurClientInvocationHandler(connections, maxRetries, backOffTime, maxBackOffTime));
   }
+  
+  private static List<String> getOnlineControllers(BlurConfiguration conf) {
+	  String zkConn = conf.getExpected(BLUR_ZOOKEEPER_CONNECTION);
+	  int zkSessionTimeout = conf.getInt(BLUR_ZOOKEEPER_TIMEOUT, BLUR_ZOOKEEPER_TIMEOUT_DEFAULT);
+	  
+	  ZooKeeper zkClient = null;
+	  try {
+		  zkClient = ZkUtils.newZooKeeper(zkConn, zkSessionTimeout);
+		  return zkClient.getChildren(ZookeeperPathConstants.getOnlineControllersPath(), false);
+	  } catch (KeeperException e) {
+		  throw new RuntimeException("Error communicating with Zookeeper", e);
+	  } catch (InterruptedException e) {
+		  throw new RuntimeException("Error communicating with Zookeeper", e);
+	  } catch (IOException e) {
+		  throw new RuntimeException("Unable to initialize Zookeeper", e);
+	  }
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java b/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java
index faeb11e..c6b256f 100644
--- a/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java
+++ b/blur-util/src/main/java/org/apache/blur/BlurConfiguration.java
@@ -88,6 +88,14 @@ public class BlurConfiguration implements Cloneable {
   public String get(String name) {
     return get(name, null);
   }
+  
+  public String getExpected(String name) {
+	  String val = get(name);
+	  if (val == null || val.trim().isEmpty()) {
+	      throw new IllegalArgumentException("Property [" + name + "] is missing or blank.");
+	    }
+	    return val;
+  }
 
   public String get(String name, String defaultValue) {
     String property = _properties.getProperty(name, defaultValue);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-util/src/main/java/org/apache/blur/zookeeper/ZookeeperPathConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/zookeeper/ZookeeperPathConstants.java
b/blur-util/src/main/java/org/apache/blur/zookeeper/ZookeeperPathConstants.java
new file mode 100644
index 0000000..325ef6e
--- /dev/null
+++ b/blur-util/src/main/java/org/apache/blur/zookeeper/ZookeeperPathConstants.java
@@ -0,0 +1,74 @@
+package org.apache.blur.zookeeper;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+public class ZookeeperPathConstants {
+
+  public static String getBasePath() {
+    return "/blur";
+  }
+
+  public static String getClusterPath(String cluster) {
+    return getClustersPath() + "/" + cluster;
+  }
+
+  public static String getClustersPath() {
+    return getBasePath() + "/clusters";
+  }
+
+  public static String getOnlineControllersPath() {
+    return getBasePath() + "/online-controller-nodes";
+  }
+
+  public static String getControllersPath() {
+    return getBasePath() + "/controller-nodes";
+  }
+
+  public static String getOnlineShardsPath(String cluster) {
+    return getClusterPath(cluster) + "/online-nodes";
+  }
+
+  public static String getTablesPath(String cluster) {
+    return getClusterPath(cluster) + "/tables";
+  }
+
+  public static String getTablePath(String cluster, String table) {
+    return getTablesPath(cluster) + "/" + table;
+  }
+
+  public static String getSafemodePath(String cluster) {
+    return getClusterPath(cluster) + "/safemode";
+  }
+  
+  public static String getShutdownPath(String cluster) {
+    return getClusterPath(cluster) + "/shutdown";
+  }
+
+  public static String getRegisteredShardsPath(String cluster) {
+    return getClusterPath(cluster) + "/registered-nodes";
+  }
+
+  public static String getShardLayoutPath(String cluster) {
+    return getClusterPath(cluster) + "/layout";
+  }
+
+  public static String getShardLayoutPathLocks(String cluster) {
+    return getShardLayoutPath(cluster) + "/locks";
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d19cdf2a/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 739f49e..0674f74 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -42,7 +42,7 @@ blur.shard.hostname=
 # The binding address of the shard
 blur.shard.bind.address=0.0.0.0
 
-# The default binding port of the shard server
+# The default binding port of the shard server, 0 for random
 blur.shard.bind.port=40020
 
 # The number of fetcher threads
@@ -214,7 +214,7 @@ blur.controller.hostname=
 # The binding address of the controller
 blur.controller.bind.address=0.0.0.0
 
-# The default binding port of the controller server
+# The default binding port of the controller server, 0 for random
 blur.controller.bind.port=40010
 
 # The connection timeout, NOTE: this will be the maximum amount of time you can wait for
a query.


Mime
View raw message