hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sunc...@apache.org
Subject hive git commit: HIVE-16483: HoS should populate split related configurations to HiveConf (Chao Sun, reviewed by Xuefu Zhang)
Date Mon, 24 Apr 2017 16:57:02 GMT
Repository: hive
Updated Branches:
  refs/heads/master 19a1a4b42 -> 26dd70e20


HIVE-16483: HoS should populate split related configurations to HiveConf (Chao Sun, reviewed
by Xuefu Zhang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/26dd70e2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/26dd70e2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/26dd70e2

Branch: refs/heads/master
Commit: 26dd70e2074d1878f0d7962fc3b9c782803eca7e
Parents: 19a1a4b
Author: Chao Sun <sunchao@apache.org>
Authored: Mon Apr 24 09:56:02 2017 -0700
Committer: Chao Sun <sunchao@apache.org>
Committed: Mon Apr 24 09:56:02 2017 -0700

----------------------------------------------------------------------
 .../hive/ql/exec/spark/SparkPlanGenerator.java  | 23 +++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/26dd70e2/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
index 12a76a7..5f85f9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
@@ -298,12 +298,13 @@ public class SparkPlanGenerator {
       throw new IllegalArgumentException(msg, e);
     }
     if (work instanceof MapWork) {
+      MapWork mapWork = (MapWork) work;
       cloned.setBoolean("mapred.task.is.map", true);
-      List<Path> inputPaths = Utilities.getInputPaths(cloned, (MapWork) work,
+      List<Path> inputPaths = Utilities.getInputPaths(cloned, mapWork,
           scratchDir, context, false);
       Utilities.setInputPaths(cloned, inputPaths);
-      Utilities.setMapWork(cloned, (MapWork) work, scratchDir, false);
-      Utilities.createTmpDirs(cloned, (MapWork) work);
+      Utilities.setMapWork(cloned, mapWork, scratchDir, false);
+      Utilities.createTmpDirs(cloned, mapWork);
       if (work instanceof MergeFileWork) {
         MergeFileWork mergeFileWork = (MergeFileWork) work;
         cloned.set(Utilities.MAPRED_MAPPER_CLASS, MergeFileMapper.class.getName());
@@ -313,9 +314,21 @@ public class SparkPlanGenerator {
       } else {
         cloned.set(Utilities.MAPRED_MAPPER_CLASS, ExecMapper.class.getName());
       }
-      if (((MapWork) work).getMinSplitSize() != null) {
+      if (mapWork.getMaxSplitSize() != null) {
+        HiveConf.setLongVar(cloned, HiveConf.ConfVars.MAPREDMAXSPLITSIZE,
+            mapWork.getMaxSplitSize());
+      }
+      if (mapWork.getMinSplitSize() != null) {
         HiveConf.setLongVar(cloned, HiveConf.ConfVars.MAPREDMINSPLITSIZE,
-            ((MapWork) work).getMinSplitSize());
+            mapWork.getMinSplitSize());
+      }
+      if (mapWork.getMinSplitSizePerNode() != null) {
+        HiveConf.setLongVar(cloned, HiveConf.ConfVars.MAPREDMINSPLITSIZEPERNODE,
+            mapWork.getMinSplitSizePerNode());
+      }
+      if (mapWork.getMinSplitSizePerRack() != null) {
+        HiveConf.setLongVar(cloned, HiveConf.ConfVars.MAPREDMINSPLITSIZEPERRACK,
+            mapWork.getMinSplitSizePerRack());
       }
       // remember the JobConf cloned for each MapWork, so we won't clone for it again
       workToJobConf.put(work, cloned);


Mime
View raw message