Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5F506103D7 for ; Tue, 7 May 2013 22:11:24 +0000 (UTC) Received: (qmail 6067 invoked by uid 500); 7 May 2013 22:11:24 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 6039 invoked by uid 500); 7 May 2013 22:11:24 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 6022 invoked by uid 99); 7 May 2013 22:11:24 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 May 2013 22:11:24 +0000 X-ASF-Spam-Status: No, hits=-2001.2 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Tue, 07 May 2013 22:11:21 +0000 Received: (qmail 4906 invoked by uid 99); 7 May 2013 22:10:59 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 May 2013 22:10:59 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 90F7C888F6C; Tue, 7 May 2013 22:10:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hitesh@apache.org To: commits@tez.incubator.apache.org Message-Id: <4716d3345e564527933971effaca5c03@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: TEZ-105. Invalid introduction of combiner broke pi job. Date: Tue, 7 May 2013 22:10:59 +0000 (UTC) X-Virus-Checked: Checked by ClamAV on apache.org Updated Branches: refs/heads/TEZ-1 18f0ebd8e -> b4698e8ed TEZ-105. Invalid introduction of combiner broke pi job. Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/b4698e8e Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/b4698e8e Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/b4698e8e Branch: refs/heads/TEZ-1 Commit: b4698e8ed1390618ed1fd10b69f277e4dc5e34cc Parents: 18f0ebd Author: Hitesh Shah Authored: Tue May 7 15:10:25 2013 -0700 Committer: Hitesh Shah Committed: Tue May 7 15:10:25 2013 -0700 ---------------------------------------------------------------------- .../apache/tez/mapreduce/combine/MRCombiner.java | 2 +- .../org/apache/tez/mapreduce/processor/MRTask.java | 19 +++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b4698e8e/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java index 42eddee..050f608 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java @@ -241,7 +241,7 @@ public class MRCombiner implements Processor { // make a reducer org.apache.hadoop.mapreduce.Reducer reducer = (org.apache.hadoop.mapreduce.Reducer) - ReflectionUtils.newInstance(taskContext.getReducerClass(), job); + ReflectionUtils.newInstance(taskContext.getCombinerClass(), job); org.apache.hadoop.mapreduce.RecordWriter trackedRW = new org.apache.hadoop.mapreduce.RecordWriter() { http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b4698e8e/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java index d17e477..bd39b67 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java @@ -217,8 +217,23 @@ extends RunningTaskContext { partitioner = new MRPartitioner(this); ((MRPartitioner)partitioner).initialize(job, getTaskReporter()); - combineProcessor = new MRCombiner(this); - combineProcessor.initialize(job, getTaskReporter()); + + boolean useCombiner = false; + combineProcessor = null; + if (useNewApi) { + try { + useCombiner = (taskAttemptContext.getCombinerClass() != null); + } catch (ClassNotFoundException e) { + throw new IOException("Could not find combiner class", e); + } + } else { + useCombiner = (job.getCombinerClass() != null); + } + if (useCombiner) { + combineProcessor = new MRCombiner(this); + combineProcessor.initialize(job, getTaskReporter()); + } else { + } localizeConfiguration(jobConf); }