hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1552482 - /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
Date Fri, 20 Dec 2013 02:56:24 GMT
Author: gunther
Date: Fri Dec 20 02:56:24 2013
New Revision: 1552482

URL: http://svn.apache.org/r1552482
Log:
HIVE-6078: Choosing conditional task for merging files is not deterministic in tez (Vikram
Dixit K via Gunther Hagleitner)

Modified:
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1552482&r1=1552481&r2=1552482&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
(original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
Fri Dec 20 02:56:24 2013
@@ -1628,15 +1628,21 @@ public final class GenMapRedUtils {
           // There are separate configuration parameters to control whether to
           // merge for a map-only job
           // or for a map-reduce job
-          ReduceWork reduceWork = currTask.getWork() instanceof MapredWork
-              ? ((MapredWork) currTask.getWork()).getReduceWork() : null;
-          boolean mergeMapOnly =
-              hconf.getBoolVar(ConfVars.HIVEMERGEMAPFILES) && reduceWork == null;
-          boolean mergeMapRed =
-              hconf.getBoolVar(ConfVars.HIVEMERGEMAPREDFILES) &&
-              reduceWork != null;
-          if (mergeMapOnly || mergeMapRed) {
-            return true;
+          if (currTask.getWork() instanceof TezWork) {
+            return hconf.getBoolVar(ConfVars.HIVEMERGEMAPFILES) || 
+                hconf.getBoolVar(ConfVars.HIVEMERGEMAPREDFILES);
+          } else if (currTask.getWork() instanceof MapredWork) {
+            ReduceWork reduceWork = ((MapredWork) currTask.getWork()).getReduceWork();
+            boolean mergeMapOnly =
+                hconf.getBoolVar(ConfVars.HIVEMERGEMAPFILES) && reduceWork == null;
+            boolean mergeMapRed =
+                hconf.getBoolVar(ConfVars.HIVEMERGEMAPREDFILES) &&
+                reduceWork != null;
+            if (mergeMapOnly || mergeMapRed) {
+              return true;
+            }
+          } else {
+            return false;
           }
         }
       }



Mime
View raw message