tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject git commit: TEZ-105. Invalid introduction of combiner broke pi job.
Date Tue, 07 May 2013 22:10:59 GMT
Updated Branches:
  refs/heads/TEZ-1 18f0ebd8e -> b4698e8ed


TEZ-105. Invalid introduction of combiner broke pi job.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/b4698e8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/b4698e8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/b4698e8e

Branch: refs/heads/TEZ-1
Commit: b4698e8ed1390618ed1fd10b69f277e4dc5e34cc
Parents: 18f0ebd
Author: Hitesh Shah <hitesh@apache.org>
Authored: Tue May 7 15:10:25 2013 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Tue May 7 15:10:25 2013 -0700

----------------------------------------------------------------------
 .../apache/tez/mapreduce/combine/MRCombiner.java   |    2 +-
 .../org/apache/tez/mapreduce/processor/MRTask.java |   19 +++++++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b4698e8e/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
index 42eddee..050f608 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
@@ -241,7 +241,7 @@ public class MRCombiner implements Processor {
     // make a reducer
     org.apache.hadoop.mapreduce.Reducer reducer =
         (org.apache.hadoop.mapreduce.Reducer)
-        ReflectionUtils.newInstance(taskContext.getReducerClass(), job);
+        ReflectionUtils.newInstance(taskContext.getCombinerClass(), job);
 
     org.apache.hadoop.mapreduce.RecordWriter trackedRW =
         new org.apache.hadoop.mapreduce.RecordWriter() {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b4698e8e/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
index d17e477..bd39b67 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
@@ -217,8 +217,23 @@ extends RunningTaskContext {
     
     partitioner = new MRPartitioner(this);
     ((MRPartitioner)partitioner).initialize(job, getTaskReporter());
-    combineProcessor = new MRCombiner(this);
-    combineProcessor.initialize(job, getTaskReporter());
+
+    boolean useCombiner = false;
+    combineProcessor = null;
+    if (useNewApi) {
+      try {
+        useCombiner = (taskAttemptContext.getCombinerClass() != null);
+      } catch (ClassNotFoundException e) {
+        throw new IOException("Could not find combiner class", e);
+      }
+    } else {
+      useCombiner = (job.getCombinerClass() != null);
+    }
+    if (useCombiner) {
+      combineProcessor = new MRCombiner(this);
+      combineProcessor.initialize(job, getTaskReporter());
+    } else {
+    }
 
     localizeConfiguration(jobConf);
   }


Mime
View raw message