incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From avand...@apache.org
Subject svn commit: r1335190 - in /incubator/hcatalog/branches/branch-0.4: CHANGES.txt src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
Date Mon, 07 May 2012 18:53:47 GMT
Author: avandana
Date: Mon May  7 18:53:46 2012
New Revision: 1335190

URL: http://svn.apache.org/viewvc?rev=1335190&view=rev
Log:
branch 0.4 commit: hcat-403

Modified:
    incubator/hcatalog/branches/branch-0.4/CHANGES.txt
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
    incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java

Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1335190&r1=1335189&r2=1335190&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Mon May  7 18:53:46 2012
@@ -28,6 +28,8 @@ Release 0.4.1 - Unreleased
   HCAT-368 Documentation improvements: doc set & API docs (lefty via gates)
 
   BUG FIXES
+  HCAT-403 MultiOutputFormat will call taskCommit on all formats if only one ever needs commit
(rohini via avandana)
+
   HCAT-400 HCatalog build fails from git checkout due to missing lib directory (traviscrawford
via gates)
 
 Release 0.4.0 - Unreleased

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1335190&r1=1335189&r2=1335190&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
Mon May  7 18:53:46 2012
@@ -467,7 +467,7 @@ public class MultiOutputFormat extends O
         }
     }
 
-    private class MultiOutputCommitter extends OutputCommitter {
+    public class MultiOutputCommitter extends OutputCommitter {
 
         private final Map<String, BaseOutputCommitterContainer> outputCommitters;
 
@@ -516,7 +516,11 @@ public class MultiOutputFormat extends O
         public void commitTask(TaskAttemptContext taskContext) throws IOException {
             for (String alias : outputCommitters.keySet()) {
                 BaseOutputCommitterContainer outputContainer = outputCommitters.get(alias);
-                outputContainer.getBaseCommitter().commitTask(outputContainer.getContext());
+                OutputCommitter baseCommitter = outputContainer.getBaseCommitter();
+                TaskAttemptContext committerContext = outputContainer.getContext();
+                if (baseCommitter.needsTaskCommit(committerContext)) {
+                    baseCommitter.commitTask(committerContext);
+                }
             }
         }
 

Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1335190&r1=1335189&r2=1335190&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
Mon May  7 18:53:46 2012
@@ -37,8 +37,11 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
@@ -157,6 +160,8 @@ public class TestMultiOutputFormat {
         configurer.addOutputFormat("out1", TextOutputFormat.class, IntWritable.class, Text.class);
         configurer.addOutputFormat("out2", SequenceFileOutputFormat.class, Text.class,
                 IntWritable.class);
+        configurer.addOutputFormat("out3", NullOutputFormat.class, Text.class,
+                IntWritable.class);
         Path outDir = new Path(workDir.getPath(), job.getJobName());
         FileOutputFormat.setOutputPath(configurer.getJob("out1"), new Path(outDir, "out1"));
         FileOutputFormat.setOutputPath(configurer.getJob("out2"), new Path(outDir, "out2"));
@@ -249,18 +254,40 @@ public class TestMultiOutputFormat {
     private static class MultiOutWordCountReducer extends
             Reducer<Text, IntWritable, Writable, Writable> {
 
-        private IntWritable result = new IntWritable();
+        private IntWritable count = new IntWritable();
 
         @Override
-        protected void reduce(Text key, Iterable<IntWritable> values, Context context)
+        protected void reduce(Text word, Iterable<IntWritable> values, Context context)
                 throws IOException, InterruptedException {
             int sum = 0;
             for (IntWritable val : values) {
                 sum += val.get();
             }
-            result.set(sum);
-            MultiOutputFormat.write("out1", result, key, context);
-            MultiOutputFormat.write("out2", key, result, context);
+            count.set(sum);
+            MultiOutputFormat.write("out1", count, word, context);
+            MultiOutputFormat.write("out2", word, count, context);
+            MultiOutputFormat.write("out3", word, count, context);
+        }
+    }
+
+    private static class NullOutputFormat<K, V> extends
+            org.apache.hadoop.mapreduce.lib.output.NullOutputFormat<K, V> {
+
+        @Override
+        public OutputCommitter getOutputCommitter(TaskAttemptContext context) {
+            return new OutputCommitter() {
+                public void abortTask(TaskAttemptContext taskContext) { }
+                public void cleanupJob(JobContext jobContext) { }
+                public void commitJob(JobContext jobContext) { }
+                public void commitTask(TaskAttemptContext taskContext) {
+                    Assert.fail("needsTaskCommit is false but commitTask was called");
+                }
+                public boolean needsTaskCommit(TaskAttemptContext taskContext) {
+                  return false;
+                }
+                public void setupJob(JobContext jobContext) { }
+                public void setupTask(TaskAttemptContext taskContext) { }
+              };
         }
     }
 



Mime
View raw message