hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject git commit: MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling. Contributed by Gera Shegalov (cherry picked from commit f19771a24c2f90982cf6dec35889836a6146c968)
Date Wed, 15 Oct 2014 17:57:47 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 439560dcf -> ec38eec16


MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling. Contributed by Gera Shegalov
(cherry picked from commit f19771a24c2f90982cf6dec35889836a6146c968)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec38eec1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec38eec1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec38eec1

Branch: refs/heads/branch-2.6
Commit: ec38eec1650ff0f2ae5592d6e76a8e4bef64de73
Parents: 439560d
Author: Jason Lowe <jlowe@apache.org>
Authored: Wed Oct 15 17:50:31 2014 +0000
Committer: Jason Lowe <jlowe@apache.org>
Committed: Wed Oct 15 17:52:12 2014 +0000

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  3 ++
 .../java/org/apache/hadoop/mapred/JobConf.java  |  3 +-
 .../apache/hadoop/mapreduce/MRJobConfig.java    | 11 ++++++-
 .../org/apache/hadoop/mapred/YARNRunner.java    | 15 +++++++--
 .../apache/hadoop/mapred/TestYARNRunner.java    | 32 +++++++++++++++++++-
 5 files changed, 58 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec38eec1/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a9aaab8..35c0e6a 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -43,6 +43,9 @@ Release 2.6.0 - UNRELEASED
 
     MAPREDUCE-6072. Remove INSTALL document (Akira AJISAKA via aw)
 
+    MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling (Gera
+    Shegalov via jlowe)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec38eec1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index de78e20..03f1160 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -1643,8 +1643,7 @@ public class JobConf extends Configuration {
    */
   public String getProfileParams() {
     return get(JobContext.TASK_PROFILE_PARAMS,
-               "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y," +
-                 "verbose=n,file=%s");
+        MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec38eec1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 33c0461..12bf6df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -179,6 +179,10 @@ public interface MRJobConfig {
 
   public static final String TASK_PROFILE_PARAMS = "mapreduce.task.profile.params";
 
+  public static final String DEFAULT_TASK_PROFILE_PARAMS =
+      "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,"
+          + "verbose=n,file=%s";
+
   public static final String NUM_MAP_PROFILES = "mapreduce.task.profile.maps";
 
   public static final String NUM_REDUCE_PROFILES = "mapreduce.task.profile.reduces";
@@ -603,7 +607,12 @@ public interface MRJobConfig {
   
   public static final String MR_AM_ADMIN_USER_ENV =
       MR_AM_PREFIX + "admin.user.env";
-  
+
+  public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile";
+  public static final boolean DEFAULT_MR_AM_PROFILE = false;
+  public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX
+      + "profile.params";
+
   public static final String MAPRED_MAP_ADMIN_JAVA_OPTS =
       "mapreduce.admin.map.child.java.opts";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec38eec1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 9419d03..d3b80f3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -408,7 +408,7 @@ public class YARNRunner implements ClientProtocol {
     warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce", 
         MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV);
     warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce",

-        MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);  

+        MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
 
     // Add AM admin command opts before user command opts
     // so that it can be overridden by user
@@ -424,7 +424,18 @@ public class YARNRunner implements ClientProtocol {
     warnForJavaLibPath(mrAppMasterUserOptions, "app master", 
         MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
     vargs.add(mrAppMasterUserOptions);
-    
+
+    if (jobConf.getBoolean(MRJobConfig.MR_AM_PROFILE,
+        MRJobConfig.DEFAULT_MR_AM_PROFILE)) {
+      final String profileParams = jobConf.get(MRJobConfig.MR_AM_PROFILE_PARAMS,
+          MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS);
+      if (profileParams != null) {
+        vargs.add(String.format(profileParams,
+            ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR
+                + TaskLog.LogName.PROFILE));
+      }
+    }
+
     vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
     vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
         Path.SEPARATOR + ApplicationConstants.STDOUT);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec38eec1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 38a79fb..2567785 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -29,7 +29,6 @@ import static org.mockito.Mockito.when;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -115,6 +114,11 @@ public class TestYARNRunner extends TestCase {
   private static final Log LOG = LogFactory.getLog(TestYARNRunner.class);
   private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
 
+  // prefix before <LOG_DIR>/profile.out
+  private static final String PROFILE_PARAMS =
+      MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.substring(0,
+          MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.lastIndexOf("%"));
+
   private YARNRunner yarnRunner;
   private ResourceMgrDelegate resourceMgrDelegate;
   private YarnConfiguration conf;
@@ -423,6 +427,8 @@ public class TestYARNRunner extends TestCase {
     
     for(String command : commands) {
       if(command != null) {
+        assertFalse("Profiler should be disabled by default",
+            command.contains(PROFILE_PARAMS));
         adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true");
         if(adminPos >= 0)
           adminIndex = index;
@@ -479,6 +485,30 @@ public class TestYARNRunner extends TestCase {
         "using yarn.app.mapreduce.am.env config settings."));
   }
 
+  @Test(timeout=20000)
+  public void testAMProfiler() throws Exception {
+    JobConf jobConf = new JobConf();
+
+    jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);
+
+    YARNRunner yarnRunner = new YARNRunner(jobConf);
+
+    ApplicationSubmissionContext submissionContext =
+        buildSubmitContext(yarnRunner, jobConf);
+
+    ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
+    List<String> commands = containerSpec.getCommands();
+
+    for(String command : commands) {
+      if (command != null) {
+        if (command.contains(PROFILE_PARAMS)) {
+          return;
+        }
+      }
+    }
+    throw new IllegalStateException("Profiler opts not found!");
+  }
+
   @Test
   public void testAMStandardEnv() throws Exception {
     final String ADMIN_LIB_PATH = "foo";


Mime
View raw message