hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1128184 - in /hadoop/mapreduce/branches/MR-279: ./ src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/ src/contrib/mumak/src/test/org/apache/hadoop/mapred/ src/cont...
Date Fri, 27 May 2011 07:53:16 GMT
Author: vinodkv
Date: Fri May 27 07:53:15 2011
New Revision: 1128184

URL: http://svn.apache.org/viewvc?rev=1128184&view=rev
Log:
Making streaming -file option work. Also minor fixes for successful compilation of contrib
tests. (vinodkv)

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
    hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
    hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java
    hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java
    hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
    hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri May 27 07:53:15 2011
@@ -4,6 +4,9 @@ Trunk (unreleased changes)
 
   MAPREDUCE-279
    
+    Making streaming -file option work. Also minor fixes for successful
+    compilation of contrib tests. (vinodkv)
+
     Fix for public dist cache to work with non default hdfs (mahadev &ddas)
 
     Fixed an NPE during handling of unnecessary reservations in CS. (acmurthy)

Modified: hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
(original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
Fri May 27 07:53:15 2011
@@ -668,8 +668,7 @@ public class CapacityTestUtils {
         numTrackers, 0,
         ttExpiryInterval, maps, reduces,
         numTrackers * maxMapTasksPerTracker,
-        numTrackers * maxReduceTasksPerTracker,
-        JobTracker.State.RUNNING);
+        numTrackers * maxReduceTasksPerTracker);
     }
 
     public int getNumberOfUniqueHosts() {

Modified: hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
Fri May 27 07:53:15 2011
@@ -362,8 +362,7 @@ public class TestFairScheduler extends T
       return new ClusterStatus(numTrackers, 0,
           ttExpiryInterval, maps, reduces,
           numTrackers * maxMapTasksPerTracker,
-          numTrackers * maxReduceTasksPerTracker,
-          JobTracker.State.RUNNING);
+          numTrackers * maxReduceTasksPerTracker);
     }
 
     @Override

Modified: hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java
(original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java
Fri May 27 07:53:15 2011
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtocolSignature;
 import org.apache.hadoop.mapred.TaskStatus.State;
 import org.apache.hadoop.mapred.TaskStatus.Phase;
 import org.apache.hadoop.mapreduce.ClusterMetrics;
@@ -379,12 +380,6 @@ public class MockSimulatorJobTracker imp
   }
 
   @Override
-  public org.apache.hadoop.mapreduce.server.jobtracker.State getJobTrackerState()
-      throws IOException, InterruptedException {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
   public QueueInfo getQueue(String queueName) throws IOException,
       InterruptedException {
     throw new UnsupportedOperationException();
@@ -467,4 +462,11 @@ public class MockSimulatorJobTracker imp
                                       ) throws IOException,InterruptedException{
     return 0;
   }
+
+  @Override
+  public ProtocolSignature getProtocolSignature(String protocol,
+      long clientVersion, int clientMethodsHash) throws IOException {
+    return ProtocolSignature.getProtocolSigature(
+        this, protocol, clientVersion, clientMethodsHash);
+  }
 }

Modified: hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java
(original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java
Fri May 27 07:53:15 2011
@@ -145,7 +145,8 @@ public class TestSimulatorJobTracker {
       // HashSet<> numLaunchTaskActions
       for (TaskTrackerAction action : actions) {
         if (action instanceof SimulatorLaunchTaskAction) {
-          Task task = ((SimulatorLaunchTaskAction) action).getTask();
+          TTTask ttTask = ((SimulatorLaunchTaskAction) action).getTask();
+          Task task = ttTask.getTask();
 
           numLaunchTaskActions++;
           TaskAttemptID taskId = task.getTaskID();

Modified: hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
Fri May 27 07:53:15 2011
@@ -22,8 +22,10 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -43,6 +45,7 @@ import org.apache.hadoop.mapreduce.MRCon
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.mapred.FileInputFormat;
@@ -276,19 +279,25 @@ public class StreamJob implements Tool {
       if (values != null && values.length > 0) {
         LOG.warn("-file option is deprecated, please use generic option" +
         		" -files instead.");
-        StringBuilder unpackRegex = new StringBuilder(
-          config_.getPattern(MRJobConfig.JAR_UNPACK_PATTERN,
-                             JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern());
+
+        String fileList = null;
         for (String file : values) {
           packageFiles_.add(file);
-          String fname = new File(file).getName();
-          unpackRegex.append("|(?:").append(Pattern.quote(fname)).append(")");
+          try {
+            URI pathURI = new URI(file);
+            Path path = new Path(pathURI);
+            FileSystem localFs = FileSystem.getLocal(config_);
+            String finalPath = path.makeQualified(localFs).toString();
+            fileList = fileList == null ? finalPath : fileList + "," + finalPath;
+          } catch (Exception e) {
+            throw new IllegalArgumentException(e);
+          }
         }
-        config_.setPattern(MRJobConfig.JAR_UNPACK_PATTERN,
-                           Pattern.compile(unpackRegex.toString()));
+        config_.set("tmpfiles", config_.get("tmpfiles", "") +
+                                  (fileList == null ? "" : fileList));
         validate(packageFiles_);
       }
-         
+
       String fsName = cmdLine.getOptionValue("dfs");
       if (null != fsName){
         LOG.warn("-dfs option is deprecated, please use -fs instead.");

Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java
(original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java
Fri May 27 07:53:15 2011
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtocolSignature;
 import org.apache.hadoop.mapred.SortedRanges.Range;
 import org.apache.hadoop.mapreduce.TaskType;
 
@@ -168,6 +169,13 @@ public class TestTaskCommit extends Hado
                                        long[] sizes) throws IOException {
       // NOTHING
     }
+
+    @Override
+    public ProtocolSignature getProtocolSignature(String protocol,
+        long clientVersion, int clientMethodsHash) throws IOException {
+      return ProtocolSignature.getProtocolSigature(
+                  this, protocol, clientVersion, clientMethodsHash);
+    }
   }
   
   /**



Mime
View raw message