From hcatalog-commits-return-874-apmail-incubator-hcatalog-commits-archive=incubator.apache.org@incubator.apache.org Mon May 7 18:52:21 2012 Return-Path: X-Original-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 37DE9CF7A for ; Mon, 7 May 2012 18:52:21 +0000 (UTC) Received: (qmail 21414 invoked by uid 500); 7 May 2012 18:52:21 -0000 Delivered-To: apmail-incubator-hcatalog-commits-archive@incubator.apache.org Received: (qmail 21387 invoked by uid 500); 7 May 2012 18:52:21 -0000 Mailing-List: contact hcatalog-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hcatalog-dev@incubator.apache.org Delivered-To: mailing list hcatalog-commits@incubator.apache.org Received: (qmail 21380 invoked by uid 99); 7 May 2012 18:52:21 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 May 2012 18:52:21 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 May 2012 18:52:19 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id A2A052388B71; Mon, 7 May 2012 18:51:59 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1335189 - in /incubator/hcatalog/trunk: CHANGES.txt src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java Date: Mon, 07 May 2012 18:51:59 -0000 To: hcatalog-commits@incubator.apache.org From: avandana@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120507185159.A2A052388B71@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: avandana Date: Mon May 7 18:51:59 2012 New Revision: 1335189 URL: http://svn.apache.org/viewvc?rev=1335189&view=rev Log: hcat-403 MultiOutputFormat will call taskCommit on all formats if only one ever needs commit Modified: incubator/hcatalog/trunk/CHANGES.txt incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java Modified: incubator/hcatalog/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1335189&r1=1335188&r2=1335189&view=diff ============================================================================== --- incubator/hcatalog/trunk/CHANGES.txt (original) +++ incubator/hcatalog/trunk/CHANGES.txt Mon May 7 18:51:59 2012 @@ -37,6 +37,8 @@ Trunk (unreleased changes) OPTIMIZATIONS BUG FIXES + HCAT-403 MultiOutputFormat will call taskCommit on all formats if only one ever needs commit (rohini via avandana) + HCAT-400 HCatalog build fails from git checkout due to missing lib directory (traviscrawford via gates) Release 0.4.0 - Unreleased Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1335189&r1=1335188&r2=1335189&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (original) +++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java Mon May 7 18:51:59 2012 @@ -467,7 +467,7 @@ public class MultiOutputFormat extends O } } - private class MultiOutputCommitter extends OutputCommitter { + public class MultiOutputCommitter extends OutputCommitter { private final Map outputCommitters; @@ -516,7 +516,11 @@ public class MultiOutputFormat extends O public void commitTask(TaskAttemptContext taskContext) throws IOException { for (String alias : outputCommitters.keySet()) { BaseOutputCommitterContainer outputContainer = outputCommitters.get(alias); - outputContainer.getBaseCommitter().commitTask(outputContainer.getContext()); + OutputCommitter baseCommitter = outputContainer.getBaseCommitter(); + TaskAttemptContext committerContext = outputContainer.getContext(); + if (baseCommitter.needsTaskCommit(committerContext)) { + baseCommitter.commitTask(committerContext); + } } } Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1335189&r1=1335188&r2=1335189&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java (original) +++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java Mon May 7 18:51:59 2012 @@ -37,8 +37,11 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; @@ -157,6 +160,8 @@ public class TestMultiOutputFormat { configurer.addOutputFormat("out1", TextOutputFormat.class, IntWritable.class, Text.class); configurer.addOutputFormat("out2", SequenceFileOutputFormat.class, Text.class, IntWritable.class); + configurer.addOutputFormat("out3", NullOutputFormat.class, Text.class, + IntWritable.class); Path outDir = new Path(workDir.getPath(), job.getJobName()); FileOutputFormat.setOutputPath(configurer.getJob("out1"), new Path(outDir, "out1")); FileOutputFormat.setOutputPath(configurer.getJob("out2"), new Path(outDir, "out2")); @@ -249,18 +254,40 @@ public class TestMultiOutputFormat { private static class MultiOutWordCountReducer extends Reducer { - private IntWritable result = new IntWritable(); + private IntWritable count = new IntWritable(); @Override - protected void reduce(Text key, Iterable values, Context context) + protected void reduce(Text word, Iterable values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } - result.set(sum); - MultiOutputFormat.write("out1", result, key, context); - MultiOutputFormat.write("out2", key, result, context); + count.set(sum); + MultiOutputFormat.write("out1", count, word, context); + MultiOutputFormat.write("out2", word, count, context); + MultiOutputFormat.write("out3", word, count, context); + } + } + + private static class NullOutputFormat extends + org.apache.hadoop.mapreduce.lib.output.NullOutputFormat { + + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext context) { + return new OutputCommitter() { + public void abortTask(TaskAttemptContext taskContext) { } + public void cleanupJob(JobContext jobContext) { } + public void commitJob(JobContext jobContext) { } + public void commitTask(TaskAttemptContext taskContext) { + Assert.fail("needsTaskCommit is false but commitTask was called"); + } + public boolean needsTaskCommit(TaskAttemptContext taskContext) { + return false; + } + public void setupJob(JobContext jobContext) { } + public void setupTask(TaskAttemptContext taskContext) { } + }; } }