hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1299047 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/ja...
Date Fri, 09 Mar 2012 21:13:42 GMT
Author: vinodkv
Date: Fri Mar  9 21:13:42 2012
New Revision: 1299047

URL: http://svn.apache.org/viewvc?rev=1299047&view=rev
Log:
MAPREDUCE-3982. Fixed FileOutputCommitter to not err out for an 'empty-job' whose tasks don't
write any outputs. Contributed by Robert Joseph Evans.

Modified:
    hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1299047&r1=1299046&r2=1299047&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Mar  9 21:13:42 2012
@@ -304,6 +304,9 @@ Release 0.23.2 - UNRELEASED
     MAPREDUCE-3975. Default value not set for Configuration parameter
     mapreduce.job.local.dir (Eric Payne via bobby)
 
+    MAPREDUCE-3982. Fixed FileOutputCommitter to not err out for an 'empty-job'
+    whose tasks don't write any outputs. (Robert Joseph Evans via vinodkv)
+
 Release 0.23.1 - 2012-02-17
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java?rev=1299047&r1=1299046&r2=1299047&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
Fri Mar  9 21:13:42 2012
@@ -278,11 +278,11 @@ public class FileOutputCommitter extends
    */
   public void setupJob(JobContext context) throws IOException {
     if (hasOutputPath()) {
-      Path pendingJobAttemptsPath = getPendingJobAttemptsPath();
-      FileSystem fs = pendingJobAttemptsPath.getFileSystem(
+      Path jobAttemptPath = getJobAttemptPath(context);
+      FileSystem fs = jobAttemptPath.getFileSystem(
           context.getConfiguration());
-      if (!fs.mkdirs(pendingJobAttemptsPath)) {
-        LOG.error("Mkdirs failed to create " + pendingJobAttemptsPath);
+      if (!fs.mkdirs(jobAttemptPath)) {
+        LOG.error("Mkdirs failed to create " + jobAttemptPath);
       }
     } else {
       LOG.warn("Output Path is null in setupJob()");

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java?rev=1299047&r1=1299046&r2=1299047&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
Fri Mar  9 21:13:42 2012
@@ -122,6 +122,28 @@ public class TestFileOutputCommitter ext
     assertEquals(output, expectedOutput.toString());
     FileUtil.fullyDelete(new File(outDir.toString()));
   }
+  
+  public void testEmptyOutput() throws Exception {
+    Job job = Job.getInstance();
+    FileOutputFormat.setOutputPath(job, outDir);
+    Configuration conf = job.getConfiguration();
+    conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
+    JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
+    TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
+    FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);
+
+    // setup
+    committer.setupJob(jContext);
+    committer.setupTask(tContext);
+
+    // Do not write any output
+
+    // do commit
+    committer.commitTask(tContext);
+    committer.commitJob(jContext);
+    
+    FileUtil.fullyDelete(new File(outDir.toString()));
+  }
 
   @SuppressWarnings("unchecked")
   public void testAbort() throws IOException, InterruptedException {



Mime
View raw message