Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: mapreduce-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1173452 - in
 /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: CHANGES.txt
 src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
Date: Wed, 21 Sep 2011 01:11:42 -0000
To: mapreduce-commits@hadoop.apache.org
From: acmurthy@apache.org
Message-Id: <20110921011142.91037238889B@eris.apache.org>

Author: acmurthy
Date: Wed Sep 21 01:11:42 2011
New Revision: 1173452

URL: http://svn.apache.org/viewvc?rev=1173452&view=rev
Log:
Merge -r 1173450:1173451 from trunk to branch-0.23 to fix MAPREDUCE-3018.

Modified:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1173452&r1=1173451&r2=1173452&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed Sep 21 01:11:42 2011
@@ -1338,6 +1338,8 @@ Release 0.23.0 - Unreleased
     YarnClientProtocolProvider and ensured MiniMRYarnCluster sets JobHistory
     configuration for tests. (acmurthy) 
 
+    MAPREDUCE-3018. Fixed -file option for streaming. (mahadev via acmurthy) 
+
 Release 0.22.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java?rev=1173452&r1=1173451&r2=1173452&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java Wed Sep 21 01:11:42 2011
@@ -22,8 +22,10 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -43,6 +45,7 @@ import org.apache.hadoop.mapreduce.MRCon
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.mapred.FileInputFormat;
@@ -277,19 +280,25 @@ public class StreamJob implements Tool {
       if (values != null && values.length > 0) {
         LOG.warn("-file option is deprecated, please use generic option" +
         		" -files instead.");
-        StringBuilder unpackRegex = new StringBuilder(
-          config_.getPattern(MRJobConfig.JAR_UNPACK_PATTERN,
-                             JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern());
+
+        String fileList = null;
         for (String file : values) {
           packageFiles_.add(file);
-          String fname = new File(file).getName();
-          unpackRegex.append("|(?:").append(Pattern.quote(fname)).append(")");
+          try {
+            URI pathURI = new URI(file);
+            Path path = new Path(pathURI);
+            FileSystem localFs = FileSystem.getLocal(config_);
+            String finalPath = path.makeQualified(localFs).toString();
+            fileList = fileList == null ? finalPath : fileList + "," + finalPath;
+          } catch (Exception e) {
+            throw new IllegalArgumentException(e);
+          }
         }
-        config_.setPattern(MRJobConfig.JAR_UNPACK_PATTERN,
-                           Pattern.compile(unpackRegex.toString()));
+        config_.set("tmpfiles", config_.get("tmpfiles", "") +
+                                  (fileList == null ? "" : fileList));
         validate(packageFiles_);
       }
-         
+
       String fsName = cmdLine.getOptionValue("dfs");
       if (null != fsName){
         LOG.warn("-dfs option is deprecated, please use -fs instead.");