kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-1910 Support Separate HBase Cluster with NN HA and Kerberos Authentication
Date Thu, 18 Aug 2016 23:27:41 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 6c88139d4 -> af2a82714


KYLIN-1910 Support Separate HBase Cluster with NN HA and Kerberos Authentication

Signed-off-by: Yang Li <liyang@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af2a8271
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af2a8271
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af2a8271

Branch: refs/heads/master
Commit: af2a82714d54a640c8e3941b4d79e9fb2bbe77a2
Parents: 6c88139
Author: sunyerui <sunyerui@gmail.com>
Authored: Thu Jul 21 17:41:32 2016 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Fri Aug 19 07:14:41 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/KylinConfigBase.java    |  4 +++
 .../kylin/storage/hbase/HBaseConnection.java    | 34 ++++++++++++++++++++
 .../kylin/storage/hbase/steps/CubeHFileJob.java |  2 ++
 3 files changed, 40 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/af2a8271/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7a7d796..9a8e6fd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -208,6 +208,10 @@ abstract public class KylinConfigBase implements Serializable {
         return getOptional("kylin.hbase.cluster.fs", "");
     }
 
+    public String getHBaseClusterHDFSConfigFile() {
+        return getOptional("kylin.hbase.cluster.hdfs.config.file", "");
+    }
+
     public String getKylinJobLogDir() {
         return getOptional("kylin.job.log.dir", "/tmp/kylin/logs");
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/af2a8271/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index d4dd3ae..e93c2bd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -20,6 +20,7 @@ package org.apache.kylin.storage.hbase;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.StorageException;
 import org.apache.kylin.engine.mr.HadoopUtil;
@@ -140,6 +142,7 @@ public class HBaseConnection {
 
     private static Configuration newHBaseConfiguration(String url) {
         Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
+        addHBaseClusterNNHAConfiguration(conf);
 
         // using a hbase:xxx URL is deprecated, instead hbase config is always loaded from
hbase-site.xml in classpath
         if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
@@ -168,6 +171,37 @@ public class HBaseConnection {
         return conf;
     }
 
+    // See YARN-3021. Copy here in case of missing in dependency MR client jars
+    public static final String JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE = "mapreduce.job.hdfs-servers.token-renewal.exclude";
+
+    public static void addHBaseClusterNNHAConfiguration(Configuration conf) {
+        String hdfsConfigFile = KylinConfig.getInstanceFromEnv().getHBaseClusterHDFSConfigFile();
+        if (hdfsConfigFile == null || hdfsConfigFile.isEmpty()) {
+            return;
+        }
+        Configuration hdfsConf = new Configuration(false);
+        hdfsConf.addResource(hdfsConfigFile);
+        Collection<String> nameServices = hdfsConf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
+        Collection<String> mainNameServices = conf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
+        for (String serviceId : nameServices) {
+            mainNameServices.add(serviceId);
+
+            String serviceConfKey = DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + serviceId;
+            String proxyConfKey = DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX
+ "." + serviceId;
+            conf.set(serviceConfKey, hdfsConf.get(serviceConfKey, ""));
+            conf.set(proxyConfKey, hdfsConf.get(proxyConfKey, ""));
+
+            Collection<String> nameNodes = hdfsConf.getTrimmedStringCollection(serviceConfKey);
+            for (String nameNode : nameNodes) {
+                String rpcConfKey = DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + serviceId
+ "." + nameNode;
+                conf.set(rpcConfKey, hdfsConf.get(rpcConfKey, ""));
+            }
+        }
+        conf.setStrings(DFSConfigKeys.DFS_NAMESERVICES, mainNameServices.toArray(new String[0]));
+        // See YARN-3021, instruct RM skip renew token of hbase cluster name services
+        conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, nameServices.toArray(new String[0]));
+    }
+
     public static String makeQualifiedPathInHBaseCluster(String path) {
         try {
             FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());

http://git-wip-us.apache.org/repos/asf/kylin/blob/af2a8271/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index e4a9c1e..9145cef 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -42,6 +42,7 @@ import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -75,6 +76,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
             job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
 
             setJobClasspath(job, cube.getConfig());
+            HBaseConnection.addHBaseClusterNNHAConfiguration(job.getConfiguration());
 
             addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
             FileOutputFormat.setOutputPath(job, output);


Mime
View raw message