hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject svn commit: r1531743 - in /hadoop/common/branches/branch-2.2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapreduce-client/hadoop-mapreduce-client-app...
Date Sun, 13 Oct 2013 20:20:11 GMT
Author: sseth
Date: Sun Oct 13 20:20:11 2013
New Revision: 1531743

URL: http://svn.apache.org/r1531743
Log:
MAPREDUCE-5329. Allow MR applications to use additional AuxServices, which are compatible
with the default MapReduce shuffle. Contributed by Avner BenHanoch.

Added:
    hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestShuffleProvider.java
      - copied unchanged from r1531742, hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestShuffleProvider.java
Modified:
    hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
    hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

Modified: hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt?rev=1531743&r1=1531742&r2=1531743&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt Sun Oct 13 20:20:11
2013
@@ -7,6 +7,10 @@ Release 2.2.1 - UNRELEASED
   NEW FEATURES
 
   IMPROVEMENTS
+  
+    MAPREDUCE-5329. Allow MR applications to use additional AuxServices,
+    which are compatible with the default MapReduce shuffle.
+    (Avner BenHanoch via sseth)
 
   OPTIMIZATIONS
 

Modified: hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1531743&r1=1531742&r2=1531743&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++ hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Sun Oct 13 20:20:11 2013
@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.Collection;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
@@ -722,6 +723,32 @@ public abstract class TaskAttemptImpl im
       serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
           ShuffleHandler.serializeServiceData(shuffleToken));
 
+      // add external shuffle-providers - if any
+      Collection<String> shuffleProviders = conf.getStringCollection(
+          MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES);
+      if (! shuffleProviders.isEmpty()) {
+        Collection<String> auxNames = conf.getStringCollection(
+            YarnConfiguration.NM_AUX_SERVICES);
+
+        for (final String shuffleProvider : shuffleProviders) {
+          if (shuffleProvider.equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) {
+            continue; // skip built-in shuffle-provider that was already inserted with shuffle
secret key
+          }
+          if (auxNames.contains(shuffleProvider)) {
+                LOG.info("Adding ShuffleProvider Service: " + shuffleProvider + " to serviceData");
+                // This only serves for INIT_APP notifications
+                // The shuffle service needs to be able to work with the host:port information
provided by the AM
+                // (i.e. shuffle services which require custom location / other configuration
are not supported)
+                serviceData.put(shuffleProvider, ByteBuffer.allocate(0));
+          }
+          else {
+            throw new YarnRuntimeException("ShuffleProvider Service: " + shuffleProvider
+
+            " was NOT found in the list of aux-services that are available in this NM." +
+            " You may need to specify this ShuffleProvider as an aux-service in your yarn-site.xml");
+          }
+        }
+      }
+
       Apps.addToEnvironment(
           environment,  
           Environment.CLASSPATH.name(), 

Modified: hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1531743&r1=1531742&r2=1531743&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
(original)
+++ hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
Sun Oct 13 20:20:11 2013
@@ -133,6 +133,13 @@ public interface MRJobConfig {
 
   public static final String MAPREDUCE_JOB_CLASSLOADER = "mapreduce.job.classloader";
 
+  /**
+   * A comma-separated list of services that function as ShuffleProvider aux-services
+   * (in addition to the built-in ShuffleHandler).
+   * These services can serve shuffle requests from reducetasks.
+   */
+  public static final String MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES = "mapreduce.job.shuffle.provider.services";
+
   public static final String MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES = "mapreduce.job.classloader.system.classes";
 
   public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";



Mime
View raw message