kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [09/32] kylin git commit: KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder
Date Wed, 02 Nov 2016 08:38:31 GMT
KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e419681e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e419681e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e419681e

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: e419681e8c9c6135af40dcdc2b9b3b367b3e2456
Parents: 46a894d
Author: shaofengshi <shaofengshi@apache.org>
Authored: Fri Oct 28 18:15:33 2016 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Fri Oct 28 22:11:33 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/util/StringUtil.java    | 13 +++++++++
 .../engine/mr/common/AbstractHadoopJob.java     | 29 ++++++--------------
 2 files changed, 22 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 15872aa..b336e4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -124,4 +124,17 @@ public class StringUtil {
         return result;
     }
 
+    public static void appendWithSeparator(StringBuilder src, String append) {
+        if (src == null) {
+            throw new IllegalArgumentException();
+        }
+        if (src.length() > 0 && src.toString().endsWith(",") == false) {
+            src.append(",");
+        }
+
+        if (StringUtils.isBlank(append) == false) {
+            src.append(append);
+        }
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a5b2d2e..417d63d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.HadoopUtil;
@@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
             String filteredHive = filterKylinHiveDependency(kylinHiveDependency);
             logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
-            if (kylinDependency.length() > 0)
-                kylinDependency.append(",");
-            kylinDependency.append(filteredHive);
+            StringUtil.appendWithSeparator(kylinDependency, filteredHive);
         } else {
 
             logger.info("No hive dependency jars set in the environment, will find them from
jvm:");
 
             try {
                 String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
-                kylinDependency.append(hiveExecJarPath).append(",");
+
+                StringUtil.appendWithSeparator(kylinDependency, hiveExecJarPath);
                 logger.info("hive-exec jar file: " + hiveExecJarPath);
 
                 String hiveHCatJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
-                kylinDependency.append(hiveHCatJarPath).append(",");
+                StringUtil.appendWithSeparator(kylinDependency, hiveHCatJarPath);
                 logger.info("hive-catalog jar file: " + hiveHCatJarPath);
 
                 String hiveMetaStoreJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
-                kylinDependency.append(hiveMetaStoreJarPath).append(",");
+                StringUtil.appendWithSeparator(kylinDependency, hiveMetaStoreJarPath);
                 logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath);
             } catch (ClassNotFoundException e) {
                 logger.error("Cannot found hive dependency jars: " + e);
@@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
         // for kafka dependencies
         if (kylinKafkaDependency != null) {
             kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
-
             logger.info("Kafka Dependencies Before Filtered: " + kylinKafkaDependency);
-
-            if (kylinDependency.length() > 0)
-                kylinDependency.append(",");
-            kylinDependency.append(kylinKafkaDependency);
+            StringUtil.appendWithSeparator(kylinDependency, kylinKafkaDependency);
         } else {
-
             logger.info("No Kafka dependency jars set in the environment, will find them
from jvm:");
-
             try {
                 String kafkaClientJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer"));
-                kylinDependency.append(kafkaClientJarPath).append(",");
+                StringUtil.appendWithSeparator(kylinDependency, kafkaClientJarPath);
                 logger.info("kafka jar file: " + kafkaClientJarPath);
 
             } catch (ClassNotFoundException e) {
@@ -251,12 +245,7 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
 
         // for KylinJobMRLibDir
         String mrLibDir = kylinConf.getKylinJobMRLibDir();
-        if (!StringUtils.isBlank(mrLibDir)) {
-            if (kylinDependency.length() > 0) {
-                kylinDependency.append(",");
-            }
-            kylinDependency.append(mrLibDir);
-        }
+        StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
 
         setJobTmpJarsAndFiles(job, kylinDependency.toString());
 


Mime
View raw message