incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [22/29] git commit: Adding a way to configure extra hadoop props from the blur config. Anything starting with 'hadoop_conf.' will be added to the base hadoop configuration. This is helpful when blur is running indexing (mr jobs) on a different hdfs clu
Date Fri, 31 Jul 2015 18:05:28 GMT
Adding a way to configure extra hadoop props from the blur config.  Anything starting with
'hadoop_conf.' will be added to the base hadoop configuration.  This is helpful when blur
is running indexing (mr jobs) on a different hdfs cluster and that cluster is a HA cluster..


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/c1c05734
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/c1c05734
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/c1c05734

Branch: refs/heads/blur-0.2.4-parcel
Commit: c1c05734fa7cdb480a9704443edebc66a842ac4e
Parents: 7bdacf1
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Jul 31 13:23:58 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Jul 31 13:23:58 2015 -0400

----------------------------------------------------------------------
 .../org/apache/blur/server/TableContext.java    |  2 +-
 .../blur/thrift/ThriftBlurControllerServer.java |  6 +----
 .../blur/thrift/ThriftBlurShardServer.java      |  6 +----
 .../java/org/apache/blur/utils/BlurUtil.java    | 25 ++++++++++++++++----
 4 files changed, 23 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1c05734/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index d8cf3e2..46a649b 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -343,7 +343,7 @@ public class TableContext implements Cloneable {
 
   public static synchronized Configuration getSystemConfiguration() {
     if (_systemConfiguration == null) {
-      _systemConfiguration = BlurUtil.newHadoopConfiguration();
+      _systemConfiguration = BlurUtil.newHadoopConfiguration(null);
     }
     return new Configuration(_systemConfiguration);
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1c05734/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index e46a791..a1c0f88 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -46,9 +46,6 @@ import static org.apache.blur.utils.BlurConstants.BLUR_NODENAME;
 import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_DEFAULT_MAX_FRAME_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_TMP_PATH;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT;
 import static org.apache.blur.utils.BlurUtil.quietClose;
 
 import java.io.File;
@@ -81,7 +78,6 @@ import org.apache.blur.trace.TraceStorage;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.utils.GCWatcher;
 import org.apache.blur.utils.MemoryReporter;
-import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.zookeeper.ZooKeeper;
 import org.eclipse.jetty.servlet.ServletHolder;
@@ -111,7 +107,7 @@ public class ThriftBlurControllerServer extends ThriftServer {
   }
 
   public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration)
throws Exception {
-    Configuration config = BlurUtil.newHadoopConfiguration();
+    Configuration config = BlurUtil.newHadoopConfiguration(configuration);
     TableContext.setSystemBlurConfiguration(configuration);
     TableContext.setSystemConfiguration(config);
     

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1c05734/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 7809ffa..15d861c 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -53,9 +53,6 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_MAX_READ_BUF
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_SELECTOR_THREADS;
 import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_DEFAULT_MAX_FRAME_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT;
 import static org.apache.blur.utils.BlurUtil.quietClose;
 
 import java.io.Closeable;
@@ -112,7 +109,6 @@ import org.apache.blur.trace.TraceStorage;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.utils.GCWatcher;
 import org.apache.blur.utils.MemoryReporter;
-import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.zookeeper.ZooKeeper;
@@ -148,7 +144,7 @@ public class ThriftBlurShardServer extends ThriftServer {
   }
 
   public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration)
throws Exception {
-    Configuration config = BlurUtil.newHadoopConfiguration();
+    Configuration config = BlurUtil.newHadoopConfiguration(configuration);
     TableContext.setSystemBlurConfiguration(configuration);
     TableContext.setSystemConfiguration(config);
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1c05734/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index e4db03b..fb31057 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -48,6 +48,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
@@ -157,6 +158,8 @@ public class BlurUtil {
   public static final Comparator<? super BlurResult> HITS_COMPARATOR = new BlurResultComparator();
   public static final Term PRIME_DOC_TERM = new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE);
 
+  private static final String HADOOP_CONF = "hadoop_conf.";
+
   static class LoggerArgsState {
 
     private static final int MAX_BUFFER_SIZE = 16384;
@@ -703,7 +706,7 @@ public class BlurUtil {
     FileSystem fileSystem = tablePath.getFileSystem(configuration);
     if (createPath(fileSystem, tablePath)) {
       LOG.info("Table uri existed.");
-//      validateShardCount(shardCount, fileSystem, tablePath);
+      // validateShardCount(shardCount, fileSystem, tablePath);
     }
     ThreadWatcher.resetStatus();
   }
@@ -1286,14 +1289,26 @@ public class BlurUtil {
       throw new RuntimeException(e);
     }
   }
-  
-  public static Configuration newHadoopConfiguration() {
-    return addHdfsConfig(new Configuration());
+
+  public static Configuration newHadoopConfiguration(BlurConfiguration blurConfiguration)
{
+    return addHdfsConfig(new Configuration(), blurConfiguration);
   }
 
-  public static Configuration addHdfsConfig(Configuration configuration) {
+  public static Configuration addHdfsConfig(Configuration configuration, BlurConfiguration
blurConfiguration) {
     configuration.addResource("hdfs-default.xml");
     configuration.addResource("hdfs-site.xml");
+    if (blurConfiguration != null) {
+      Map<String, String> properties = blurConfiguration.getProperties();
+      for (Entry<String, String> e : properties.entrySet()) {
+        String key = e.getKey();
+        if (key.startsWith(HADOOP_CONF)) {
+          String hadoopKey = key.substring(HADOOP_CONF.length());
+          String hadoopValue = e.getValue();
+          LOG.info("Adding hadoop configuration item [{0}] [{1}]", hadoopKey, hadoopValue);
+          configuration.set(hadoopKey, hadoopValue);
+        }
+      }
+    }
     return configuration;
   }
 


Mime
View raw message