kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [32/50] [abbrv] kylin git commit: KYLIN-1469 Find hive dependencies from jvm if not exists in environment variables
Date Sun, 27 Mar 2016 01:32:35 GMT
KYLIN-1469 Find hive dependencies from jvm if not exists in environment variables


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f47d500
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f47d500
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f47d500

Branch: refs/heads/1.3.x
Commit: 3f47d500767a37f15accac07eef0c5b3d1214b76
Parents: 2cf6dd1
Author: lidongsjtu <lidong@apache.org>
Authored: Fri Mar 4 10:37:56 2016 +0800
Committer: lidongsjtu <lidong@apache.org>
Committed: Fri Mar 4 13:42:33 2016 +0800

----------------------------------------------------------------------
 examples/test_case_data/sandbox/mapred-site.xml     |  2 +-
 .../apache/kylin/job/hadoop/AbstractHadoopJob.java  | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/3f47d500/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index a8ff87f..ff1c7eb 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -58,7 +58,7 @@
 
     <property>
         <name>mapreduce.application.classpath</name>
-        <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
+        <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
     </property>
 
     <property>

http://git-wip-us.apache.org/repos/asf/kylin/blob/3f47d500/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index 8217b4d..4a89c01 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.util.ClassUtil;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.kylin.common.KylinConfig;
@@ -200,6 +201,21 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
             if (kylinDependency.length() > 0)
                 kylinDependency.append(",");
             kylinDependency.append(filteredHive);
+        } else {
+            logger.info("No hive dependency jars set in the environment, will find them from
jvm:");
+
+            String hiveExecJarPath = ClassUtil.findContainingJar(org.apache.hadoop.hive.ql.Driver.class);
+            kylinDependency.append(hiveExecJarPath).append(",");
+            logger.info("hive-exec jar file: " + hiveExecJarPath);
+
+            String hiveHCatJarPath = ClassUtil.findContainingJar(org.apache.hive.hcatalog.mapreduce.HCatInputFormat.class);
+            kylinDependency.append(hiveHCatJarPath).append(",");
+            logger.info("hive-catalog jar file: " + hiveHCatJarPath);
+
+            String hiveMetaStoreJarPath = ClassUtil.findContainingJar(org.apache.hadoop.hive.metastore.api.Table.class);
+            kylinDependency.append(hiveMetaStoreJarPath).append(",");
+            logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath);
+
         }
 
         // for KylinJobMRLibDir


Mime
View raw message