Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: common-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1466870 - in
 /hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src:
 main/java/org/apache/hadoop/streaming/ test/java/org/apache/hadoop/streaming/
Date: Thu, 11 Apr 2013 13:04:30 -0000
To: common-commits@hadoop.apache.org
From: tucu@apache.org
Message-Id: <20130411130430.77FA523888E7@eris.apache.org>

Author: tucu
Date: Thu Apr 11 13:04:29 2013
New Revision: 1466870

URL: http://svn.apache.org/r1466870
Log:
MAPREDUCE-5113. Streaming input/output types are ignored with java mapper/reducer. (sandyr via tucu)

Modified:
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1466870&r1=1466869&r2=1466870&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Thu Apr 11 13:04:29 2013
@@ -875,7 +875,7 @@ public class StreamJob implements Tool {
         IdentifierResolver.TEXT_ID));
     jobConf_.setClass("stream.map.output.reader.class",
       idResolver.getOutputReaderClass(), OutputReader.class);
-    if (isMapperACommand) {
+    if (isMapperACommand || jobConf_.get("stream.map.output") != null) {
       // if mapper is a command, then map output key/value classes come from the
       // idResolver
       jobConf_.setMapOutputKeyClass(idResolver.getOutputKeyClass());
@@ -891,7 +891,7 @@ public class StreamJob implements Tool {
         IdentifierResolver.TEXT_ID));
     jobConf_.setClass("stream.reduce.output.reader.class",
       idResolver.getOutputReaderClass(), OutputReader.class);
-    if (isReducerACommand) {
+    if (isReducerACommand || jobConf_.get("stream.reduce.output") != null) {
       // if reducer is a command, then output key/value classes come from the
       // idResolver
       jobConf_.setOutputKeyClass(idResolver.getOutputKeyClass());

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java?rev=1466870&r1=1466869&r2=1466870&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java Thu Apr 11 13:04:29 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.mapred.MapReduc
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.junit.Before;
 import org.junit.Test;
@@ -177,6 +178,17 @@ public class TestStreamingOutputKeyValue
     args.add("0");
     super.testCommandLine();
   }
+  
+  @Test
+  public void testDefaultToIdentityReducer() throws Exception {
+    args.add("-mapper");args.add(map);
+    args.add("-jobconf");
+    args.add("mapreduce.task.files.preserve.failedtasks=true");
+    args.add("-jobconf");
+    args.add("stream.tmpdir="+System.getProperty("test.build.data","/tmp"));
+    args.add("-inputformat");args.add(TextInputFormat.class.getName());
+    super.testCommandLine();
+  }
 
   @Override
   @Test

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java?rev=1466870&r1=1466869&r2=1466870&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java Thu Apr 11 13:04:29 2013
@@ -43,7 +43,6 @@ public class TrApp
     // Note the dots translated to underscore: 
     // property names have been escaped in PipeMapRed.safeEnvVarName()
     expectDefined("mapreduce_cluster_local_dir");
-    expect("mapred_output_format_class", "org.apache.hadoop.mapred.TextOutputFormat");
     expect("mapreduce_map_output_key_class", "org.apache.hadoop.io.Text");
     expect("mapreduce_map_output_value_class", "org.apache.hadoop.io.Text");