hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r477430 - in /lucene/hadoop/trunk: ./ src/contrib/streaming/src/java/org/apache/hadoop/streaming/
Date Tue, 21 Nov 2006 00:17:11 GMT
Author: cutting
Date: Mon Nov 20 16:17:10 2006
New Revision: 477430

URL: http://svn.apache.org/viewvc?view=rev&rev=477430
Log:
HADOOP-709.  Fix contrib/streaming to work with commands that contain control characters.
 Contributed by Dhruba.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeCombiner.java
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapper.java
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeReducer.java
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=477430&r1=477429&r2=477430
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Nov 20 16:17:10 2006
@@ -103,6 +103,9 @@
 31. HADOOP-733.  Make exit codes in DFShell consistent and add a unit
     test.  (Dhruba Borthakur via cutting)
 
+32. HADOOP-709.  Fix contrib/streaming to work with commands that
+    contain control characters.  (Dhruba Borthakur via cutting)
+
 
 Release 0.8.0 - 2006-11-03
 

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeCombiner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeCombiner.java?view=diff&rev=477430&r1=477429&r2=477430
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeCombiner.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeCombiner.java
Mon Nov 20 16:17:10 2006
@@ -19,7 +19,9 @@
 package org.apache.hadoop.streaming;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.util.Iterator;
+import java.net.URLDecoder;
 
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reporter;
@@ -45,7 +47,17 @@
 public class PipeCombiner extends PipeReducer {
 
   String getPipeCommand(JobConf job) {
-    return job.get("stream.combine.streamprocessor");
+    String str = job.get("stream.combine.streamprocessor");
+    if (str == null) {
+      System.err.println("X1003");
+      return str;
+    }
+    try {
+      return URLDecoder.decode(str, "UTF-8");
+    } catch (UnsupportedEncodingException e) {
+        System.err.println("stream.combine.streamprocessor in jobconf not found");
+        return null;
+    }
   }
 
 }

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapper.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapper.java?view=diff&rev=477430&r1=477429&r2=477430
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapper.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapper.java
Mon Nov 20 16:17:10 2006
@@ -19,6 +19,7 @@
 package org.apache.hadoop.streaming;
 
 import java.io.*;
+import java.net.URLDecoder;
 
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Mapper;
@@ -35,7 +36,17 @@
 public class PipeMapper extends PipeMapRed implements Mapper {
 
   String getPipeCommand(JobConf job) {
-    return job.get("stream.map.streamprocessor");
+    String str = job.get("stream.map.streamprocessor");
+    if (str == null) {
+      return str;
+    }
+    try {
+      return URLDecoder.decode(str, "UTF-8");
+    }
+    catch (UnsupportedEncodingException e) {
+      System.err.println("stream.map.streamprocessor in jobconf not found");
+      return null;
+    }
   }
 
   String getKeyColPropName() {

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeReducer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeReducer.java?view=diff&rev=477430&r1=477429&r2=477430
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeReducer.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeReducer.java
Mon Nov 20 16:17:10 2006
@@ -19,7 +19,9 @@
 package org.apache.hadoop.streaming;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.util.Iterator;
+import java.net.URLDecoder;
 
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reducer;
@@ -36,7 +38,16 @@
 public class PipeReducer extends PipeMapRed implements Reducer {
 
   String getPipeCommand(JobConf job) {
-    return job.get("stream.reduce.streamprocessor");
+    String str = job.get("stream.reduce.streamprocessor");
+    if (str == null) {
+      return str;
+    }
+    try {
+      return URLDecoder.decode(str, "UTF-8");
+    } catch (UnsupportedEncodingException e) {
+        System.err.println("stream.reduce.streamprocessor in jobconf not found");
+        return null;
+    }
   }
 
   boolean getDoPipe() {

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java?view=diff&rev=477430&r1=477429&r2=477430
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
Mon Nov 20 16:17:10 2006
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -549,7 +550,7 @@
       jobConf_.setMapperClass(c);
     } else {
       jobConf_.setMapperClass(PipeMapper.class);
-      jobConf_.set("stream.map.streamprocessor", mapCmd_);
+      jobConf_.set("stream.map.streamprocessor", URLEncoder.encode(mapCmd_, "UTF-8"));
     }
 
     if (comCmd_ != null) {
@@ -558,7 +559,7 @@
         jobConf_.setCombinerClass(c);
       } else {
         jobConf_.setCombinerClass(PipeCombiner.class);
-        jobConf_.set("stream.combine.streamprocessor", comCmd_);
+        jobConf_.set("stream.combine.streamprocessor", URLEncoder.encode(comCmd_, "UTF-8"));
       }
     }
 
@@ -570,7 +571,7 @@
         jobConf_.setReducerClass(c);
       } else {
         jobConf_.setReducerClass(PipeReducer.class);
-        jobConf_.set("stream.reduce.streamprocessor", redCmd_);
+        jobConf_.set("stream.reduce.streamprocessor", URLEncoder.encode(redCmd_, "UTF-8"));
       }
     }
 

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java?view=diff&rev=477430&r1=477429&r2=477430
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java
Mon Nov 20 16:17:10 2006
@@ -455,6 +455,15 @@
 
   static boolean getUseMapSideEffect(JobConf job) {
     String reduce = job.get("stream.reduce.streamprocessor");
+    if (reduce == null) {
+      return false;
+    }
+    try {
+      reduce = URLDecoder.decode(reduce, "UTF-8");
+    } catch (UnsupportedEncodingException e) {
+      System.err.println("stream.reduce.streamprocessor in jobconf not found");
+      return false;
+    }
     return StreamJob.REDUCE_NONE.equals(reduce);
   }
 



Mime
View raw message