hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1329779 - in /hadoop/common/branches/branch-1.0: CHANGES.txt src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
Date Tue, 24 Apr 2012 15:21:12 GMT
Author: omalley
Date: Tue Apr 24 15:21:12 2012
New Revision: 1329779

URL: http://svn.apache.org/viewvc?rev=1329779&view=rev
Log:
MAPREDUCE-4154. streaming MR job succeeds even if the streaming command 
fails. (Devaraj Das via tgraves)

Modified:
    hadoop/common/branches/branch-1.0/CHANGES.txt
    hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
    hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java

Modified: hadoop/common/branches/branch-1.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/CHANGES.txt?rev=1329779&r1=1329778&r2=1329779&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.0/CHANGES.txt Tue Apr 24 15:21:12 2012
@@ -36,6 +36,9 @@ Release 1.0.3 - unreleased
     HADOOP-8293. Fix the Makefile.am for the native library to include the
     JNI path. (omalley)
 
+    MAPREDUCE-4154. streaming MR job succeeds even if the streaming command 
+    fails. (Devaraj Das via tgraves)
+
 Release 1.0.2 - 2012.03.24
 
   NEW FEATURES

Modified: hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?rev=1329779&r1=1329778&r2=1329779&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
(original)
+++ hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
Tue Apr 24 15:21:12 2012
@@ -569,6 +569,10 @@ public abstract class PipeMapRed {
           clientOut_.flush();
           clientOut_.close();
         }
+      } catch (IOException io) {
+        LOG.warn(StringUtils.stringifyException(io));
+      }
+      try {
         waitOutputThreads();
       } catch (IOException io) {
         LOG.warn(StringUtils.stringifyException(io));

Modified: hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java?rev=1329779&r1=1329778&r2=1329779&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
(original)
+++ hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
Tue Apr 24 15:21:12 2012
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * This class tests if hadoopStreaming returns Exception 
@@ -82,6 +83,25 @@ public class TestStreamingFailure extend
       }
     }
   }
+  
+  public void testStreamingFailureForFailedProcess() throws Exception {
+    int ret = 0;
+    try {
+      createInput();
+      String[] args = {
+          "-input", INPUT_FILE.getAbsolutePath(),
+          "-output", OUTPUT_DIR.getAbsolutePath(),
+          "-mapper", "/bin/ls dsdsdsds-does-not-exist",
+          "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data",
+              "/tmp"),
+      };
+      ret = ToolRunner.run(new StreamJob(), args);
+    } finally {
+      INPUT_FILE.delete();
+      FileUtil.fullyDelete(OUTPUT_DIR.getAbsoluteFile());
+    }
+    assertEquals("Streaming job failure code expected", 1, ret);
+  }
 
   public static void main(String[]args) throws Exception
   {



Mime
View raw message