Return-Path: X-Original-To: apmail-kylin-commits-archive@minotaur.apache.org Delivered-To: apmail-kylin-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7C05919F8A for ; Sun, 27 Mar 2016 01:32:06 +0000 (UTC) Received: (qmail 33276 invoked by uid 500); 27 Mar 2016 01:32:06 -0000 Delivered-To: apmail-kylin-commits-archive@kylin.apache.org Received: (qmail 33210 invoked by uid 500); 27 Mar 2016 01:32:06 -0000 Mailing-List: contact commits-help@kylin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.apache.org Delivered-To: mailing list commits@kylin.apache.org Received: (qmail 31994 invoked by uid 99); 27 Mar 2016 01:32:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 27 Mar 2016 01:32:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6C8A6E981C; Sun, 27 Mar 2016 01:32:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shaofengshi@apache.org To: commits@kylin.apache.org Date: Sun, 27 Mar 2016 01:32:35 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [32/50] [abbrv] kylin git commit: KYLIN-1469 Find hive dependencies from jvm if not exists in environment variables KYLIN-1469 Find hive dependencies from jvm if not exists in environment variables Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f47d500 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f47d500 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f47d500 Branch: refs/heads/1.3.x Commit: 3f47d500767a37f15accac07eef0c5b3d1214b76 Parents: 2cf6dd1 Author: lidongsjtu Authored: Fri Mar 4 10:37:56 2016 +0800 Committer: lidongsjtu Committed: Fri Mar 4 13:42:33 2016 +0800 ---------------------------------------------------------------------- examples/test_case_data/sandbox/mapred-site.xml | 2 +- .../apache/kylin/job/hadoop/AbstractHadoopJob.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/3f47d500/examples/test_case_data/sandbox/mapred-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml index a8ff87f..ff1c7eb 100644 --- a/examples/test_case_data/sandbox/mapred-site.xml +++ b/examples/test_case_data/sandbox/mapred-site.xml @@ -58,7 +58,7 @@ mapreduce.application.classpath - /tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure + /tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure http://git-wip-us.apache.org/repos/asf/kylin/blob/3f47d500/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java index 8217b4d..4a89c01 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java @@ -49,6 +49,7 @@ import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.util.ClassUtil; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; import org.apache.kylin.common.KylinConfig; @@ -200,6 +201,21 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { if (kylinDependency.length() > 0) kylinDependency.append(","); kylinDependency.append(filteredHive); + } else { + logger.info("No hive dependency jars set in the environment, will find them from jvm:"); + + String hiveExecJarPath = ClassUtil.findContainingJar(org.apache.hadoop.hive.ql.Driver.class); + kylinDependency.append(hiveExecJarPath).append(","); + logger.info("hive-exec jar file: " + hiveExecJarPath); + + String hiveHCatJarPath = ClassUtil.findContainingJar(org.apache.hive.hcatalog.mapreduce.HCatInputFormat.class); + kylinDependency.append(hiveHCatJarPath).append(","); + logger.info("hive-catalog jar file: " + hiveHCatJarPath); + + String hiveMetaStoreJarPath = ClassUtil.findContainingJar(org.apache.hadoop.hive.metastore.api.Table.class); + kylinDependency.append(hiveMetaStoreJarPath).append(","); + logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath); + } // for KylinJobMRLibDir