hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1623837 - /hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
Date Tue, 09 Sep 2014 14:56:19 GMT
Author: brock
Date: Tue Sep  9 14:56:19 2014
New Revision: 1623837

URL: http://svn.apache.org/r1623837
Log:
HIVE-8029 - Remove reducers number configure in SparkTask [Spark Branch] (Rui Li via Brock)

Modified:
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java?rev=1623837&r1=1623836&r2=1623837&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java Tue
Sep  9 14:56:19 2014
@@ -36,7 +36,6 @@ import org.apache.hadoop.hive.ql.exec.sp
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.BaseWork;
 import org.apache.hadoop.hive.ql.plan.MapWork;
-import org.apache.hadoop.hive.ql.plan.ReduceWork;
 import org.apache.hadoop.hive.ql.plan.SparkWork;
 import org.apache.hadoop.hive.ql.plan.UnionWork;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
@@ -62,7 +61,7 @@ public class SparkTask extends Task<Spar
     SparkSession sparkSession = null;
     SparkSessionManager sparkSessionManager = null;
     try {
-      configureNumberOfReducers();
+      printConfigInfo();
       sparkSessionManager = SparkSessionManagerImpl.getInstance();
       sparkSession = SessionState.get().getSparkSession();
       
@@ -158,12 +157,7 @@ public class SparkTask extends Task<Spar
   /**
    * Set the number of reducers for the spark work.
    */
-  private void configureNumberOfReducers() throws IOException {
-    for (BaseWork baseWork : work.getAllWork()) {
-      if (baseWork instanceof ReduceWork) {
-        configureNumberOfReducers((ReduceWork) baseWork);
-      }
-    }
+  private void printConfigInfo() throws IOException {
 
     console.printInfo("In order to change the average load for a reducer (in bytes):");
     console.printInfo("  set " + HiveConf.ConfVars.BYTESPERREDUCER.varname + "=<number>");
@@ -172,32 +166,4 @@ public class SparkTask extends Task<Spar
     console.printInfo("In order to set a constant number of reducers:");
     console.printInfo("  set " + HiveConf.ConfVars.HADOOPNUMREDUCERS + "=<number>");
   }
-
-  private void configureNumberOfReducers(ReduceWork rWork) throws IOException {
-    // this is a temporary hack to fix things that are not fixed in the compiler
-    Integer numReducersFromWork = rWork == null ? 0 : rWork.getNumReduceTasks();
-
-    if (rWork == null) {
-      console.printInfo("Number of reduce tasks is set to 0 since there's no reduce operator");
-    } else {
-      if (numReducersFromWork >= 0) {
-        console.printInfo("Number of reduce tasks determined at compile time: "
-          + rWork.getNumReduceTasks());
-      } else if (job.getNumReduceTasks() > 0) {
-        int reducers = job.getNumReduceTasks();
-        rWork.setNumReduceTasks(reducers);
-        console.printInfo("Number of reduce tasks not specified. Defaulting to jobconf value
of: "
-            + reducers);
-      } else {
-        if (inputSummary == null) {
-          inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(),
null);
-        }
-        int reducers = Utilities.estimateNumberOfReducers(conf, inputSummary, work.getMapWork(),
-          false);
-        rWork.setNumReduceTasks(reducers);
-        console.printInfo("Number of reduce tasks not specified. Estimated from input data
size: "
-            + reducers);
-      }
-    }
-  }
 }



Mime
View raw message