kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [19/32] kylin git commit: AbstractHadoopJob: set hive dependency filter as a config entry
Date Wed, 02 Nov 2016 08:38:41 GMT
AbstractHadoopJob: set hive dependency filter as a config entry


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8922e87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8922e87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8922e87

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: a8922e87b1f483dc010185f4a08aae970898c45d
Parents: 21ecfce
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Mon Oct 31 17:03:42 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Mon Oct 31 17:11:01 2016 +0800

----------------------------------------------------------------------
 .../main/java/org/apache/kylin/common/KylinConfigBase.java  | 8 ++++++--
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java    | 9 +++------
 2 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/a8922e87/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 26c280b..011155f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -22,9 +22,9 @@ import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.SortedSet;
-import java.util.Map.Entry;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -504,7 +504,7 @@ abstract public class KylinConfigBase implements Serializable {
     public boolean getBadQueryPersistentEnabled() {
         return Boolean.parseBoolean(getOptional("kylin.query.badquery.persistent.enable",
"true"));
     }
-    
+
     public String[] getQueryTransformers() {
         return getOptionalStringArray("kylin.query.transformers", new String[0]);
     }
@@ -633,6 +633,10 @@ abstract public class KylinConfigBase implements Serializable {
         return this.getOptional("kylin.job.hive.database.for.intermediatetable", "default");
     }
 
+    public String getHiveDependencyFilterList() {
+        return this.getOptional("kylin.job.dependency.filterlist", "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar"
+ "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar");
+    }
+
     public String getKylinOwner() {
         return this.getOptional("kylin.owner", "");
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a8922e87/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 72f4437..77791ce 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -96,8 +96,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool
{
 
     private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
 
-    private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar"
+ "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar";
-
     protected static void runJob(Tool job, String[] args) {
         try {
             int exitCode = ToolRunner.run(job, args);
@@ -200,8 +198,7 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
             kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
             logger.info("Hive Dependencies Before Filtered: " + kylinHiveDependency);
-            //String filteredHive = filterKylinHiveDependency(kylinHiveDependency);
-            String filteredHive = kylinHiveDependency;
+            String filteredHive = filterKylinHiveDependency(kylinHiveDependency, kylinConf);
             logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
             StringUtil.appendWithSeparator(kylinDependency, filteredHive);
@@ -259,13 +256,13 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
         }
     }
 
-    private String filterKylinHiveDependency(String kylinHiveDependency) {
+    private String filterKylinHiveDependency(String kylinHiveDependency, KylinConfig config)
{
         if (StringUtils.isBlank(kylinHiveDependency))
             return "";
 
         StringBuilder jarList = new StringBuilder();
 
-        Pattern hivePattern = Pattern.compile(KYLIN_HIVE_DEPENDENCY_JARS);
+        Pattern hivePattern = Pattern.compile(config.getHiveDependencyFilterList());
         Matcher matcher = hivePattern.matcher(kylinHiveDependency);
 
         while (matcher.find()) {


Mime
View raw message