hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mit...@apache.org
Subject hive git commit: HIVE-17621: Hive-site settings are ignored during HCatInputFormat split-calculation (Chris Drome, reviewed by Mithun Radhakrishnan)
Date Thu, 28 Sep 2017 23:13:56 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2 61bbad6ea -> 5c2e3b4d8


HIVE-17621: Hive-site settings are ignored during HCatInputFormat split-calculation (Chris
Drome, reviewed by Mithun Radhakrishnan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5c2e3b4d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5c2e3b4d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5c2e3b4d

Branch: refs/heads/branch-2
Commit: 5c2e3b4d80b8fe13b41aa43ea0c2dbbefa283037
Parents: 61bbad6
Author: Mithun RK <mithun@apache.org>
Authored: Wed Sep 27 11:10:36 2017 -0700
Committer: Mithun RK <mithunr@oath.com>
Committed: Thu Sep 28 16:10:21 2017 -0700

----------------------------------------------------------------------
 .../apache/hive/hcatalog/common/HCatUtil.java   | 24 +++++++++++++++++---
 .../hcatalog/mapreduce/HCatBaseInputFormat.java |  8 +++++--
 2 files changed, 27 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5c2e3b4d/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
index 8b927af..d18e206 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
@@ -33,6 +33,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
+import javax.security.auth.login.LoginException;
+
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -76,8 +78,6 @@ import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.security.auth.login.LoginException;
-
 public class HCatUtil {
 
   private static final Logger LOG = LoggerFactory.getLogger(HCatUtil.class);
@@ -642,6 +642,25 @@ public class HCatUtil {
     return jobConf;
   }
 
+  public static Map<String,String> getHCatKeyHiveConf(JobConf conf) {
+    Map<String,String> hiveProperties = new HashMap<String,String>();
+    if (conf.get(HCatConstants.HCAT_KEY_HIVE_CONF) != null) {
+      try {
+        Properties properties = (Properties) HCatUtil.deserialize(
+          conf.get(HCatConstants.HCAT_KEY_HIVE_CONF));
+        for (Map.Entry<Object, Object> prop : properties.entrySet()) {
+          if (conf.get((String)prop.getKey()) == null) {
+            hiveProperties.put(prop.getKey().toString(), prop.getValue().toString());
+          }
+        }
+      } catch (IOException e) {
+        throw new IllegalStateException(
+            "Failed to deserialize hive conf", e);
+      }
+    }
+    return hiveProperties;
+  }
+
   public static void copyJobPropertiesToJobConf(
     Map<String, String> jobProperties, JobConf jobConf) {
     for (Map.Entry<String, String> entry : jobProperties.entrySet()) {
@@ -649,7 +668,6 @@ public class HCatUtil {
     }
   }
 
-
   public static boolean isHadoop23() {
     String version = org.apache.hadoop.util.VersionInfo.getVersion();
     if (version.matches("\\b0\\.23\\..+\\b")||version.matches("\\b2\\..*"))

http://git-wip-us.apache.org/repos/asf/hive/blob/5c2e3b4d/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
index 9caff7a..ec1e1ca 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
@@ -126,16 +126,20 @@ public abstract class HCatBaseInputFormat
     }
 
     HiveStorageHandler storageHandler;
-    JobConf jobConf;
+    Map<String,String> hiveProps = null;
     //For each matching partition, call getSplits on the underlying InputFormat
     for (PartInfo partitionInfo : partitionInfoList) {
-      jobConf = HCatUtil.getJobConfFromContext(jobContext);
+      JobConf jobConf = HCatUtil.getJobConfFromContext(jobContext);
+      if (hiveProps == null) {
+        hiveProps = HCatUtil.getHCatKeyHiveConf(jobConf);
+      }
       List<String> setInputPath = setInputPath(jobConf, partitionInfo.getLocation());
       if (setInputPath.isEmpty()) {
         continue;
       }
       Map<String, String> jobProperties = partitionInfo.getJobProperties();
 
+      HCatUtil.copyJobPropertiesToJobConf(hiveProps, jobConf);
       HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf);
 
       storageHandler = HCatUtil.getStorageHandler(


Mime
View raw message