Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 374501080C for ; Sun, 13 Oct 2013 20:20:38 +0000 (UTC) Received: (qmail 32910 invoked by uid 500); 13 Oct 2013 20:20:36 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 32843 invoked by uid 500); 13 Oct 2013 20:20:36 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 32835 invoked by uid 99); 13 Oct 2013 20:20:36 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 13 Oct 2013 20:20:36 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 13 Oct 2013 20:20:33 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id C36BE2388868; Sun, 13 Oct 2013 20:20:11 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1531743 - in /hadoop/common/branches/branch-2.2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapreduce-client/hadoop-mapreduce-client-app... Date: Sun, 13 Oct 2013 20:20:11 -0000 To: mapreduce-commits@hadoop.apache.org From: sseth@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131013202011.C36BE2388868@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: sseth Date: Sun Oct 13 20:20:11 2013 New Revision: 1531743 URL: http://svn.apache.org/r1531743 Log: MAPREDUCE-5329. Allow MR applications to use additional AuxServices, which are compatible with the default MapReduce shuffle. Contributed by Avner BenHanoch. Added: hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestShuffleProvider.java - copied unchanged from r1531742, hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestShuffleProvider.java Modified: hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Modified: hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt?rev=1531743&r1=1531742&r2=1531743&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/CHANGES.txt Sun Oct 13 20:20:11 2013 @@ -7,6 +7,10 @@ Release 2.2.1 - UNRELEASED NEW FEATURES IMPROVEMENTS + + MAPREDUCE-5329. Allow MR applications to use additional AuxServices, + which are compatible with the default MapReduce shuffle. + (Avner BenHanoch via sseth) OPTIMIZATIONS Modified: hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1531743&r1=1531742&r2=1531743&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Sun Oct 13 20:20:11 2013 @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.Collection; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; @@ -722,6 +723,32 @@ public abstract class TaskAttemptImpl im serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, ShuffleHandler.serializeServiceData(shuffleToken)); + // add external shuffle-providers - if any + Collection shuffleProviders = conf.getStringCollection( + MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES); + if (! shuffleProviders.isEmpty()) { + Collection auxNames = conf.getStringCollection( + YarnConfiguration.NM_AUX_SERVICES); + + for (final String shuffleProvider : shuffleProviders) { + if (shuffleProvider.equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) { + continue; // skip built-in shuffle-provider that was already inserted with shuffle secret key + } + if (auxNames.contains(shuffleProvider)) { + LOG.info("Adding ShuffleProvider Service: " + shuffleProvider + " to serviceData"); + // This only serves for INIT_APP notifications + // The shuffle service needs to be able to work with the host:port information provided by the AM + // (i.e. shuffle services which require custom location / other configuration are not supported) + serviceData.put(shuffleProvider, ByteBuffer.allocate(0)); + } + else { + throw new YarnRuntimeException("ShuffleProvider Service: " + shuffleProvider + + " was NOT found in the list of aux-services that are available in this NM." + + " You may need to specify this ShuffleProvider as an aux-service in your yarn-site.xml"); + } + } + } + Apps.addToEnvironment( environment, Environment.CLASSPATH.name(), Modified: hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1531743&r1=1531742&r2=1531743&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Sun Oct 13 20:20:11 2013 @@ -133,6 +133,13 @@ public interface MRJobConfig { public static final String MAPREDUCE_JOB_CLASSLOADER = "mapreduce.job.classloader"; + /** + * A comma-separated list of services that function as ShuffleProvider aux-services + * (in addition to the built-in ShuffleHandler). + * These services can serve shuffle requests from reducetasks. + */ + public static final String MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES = "mapreduce.job.shuffle.provider.services"; + public static final String MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES = "mapreduce.job.classloader.system.classes"; public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";