hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject hadoop git commit: MAPREDUCE-6484. Yarn Client uses local address instead of RM address as token renewer in a secure cluster when RM HA is enabled. Contributed by Zhihai Xu
Date Thu, 24 Sep 2015 15:17:29 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 a0ff9af82 -> e5ba1a052


MAPREDUCE-6484. Yarn Client uses local address instead of RM address as token renewer in a
secure cluster when RM HA is enabled. Contributed by Zhihai Xu

(cherry picked from commit 97a08807ec328a6f455d941fe3d9a469d885caa6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5ba1a05
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5ba1a05
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5ba1a05

Branch: refs/heads/branch-2
Commit: e5ba1a05210ee883c9acca93bb84676df93d405d
Parents: a0ff9af
Author: Zhihai Xu <zxu@apache.org>
Authored: Thu Sep 24 08:12:07 2015 -0700
Committer: Zhihai Xu <zxu@apache.org>
Committed: Thu Sep 24 08:14:11 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  3 +++
 .../java/org/apache/hadoop/mapred/Master.java   | 27 +++++++++++++++++---
 .../org/apache/hadoop/mapred/TestMaster.java    | 13 ++++++++++
 3 files changed, 40 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5ba1a05/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 96620b7..ca21c6c 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -308,6 +308,9 @@ Release 2.8.0 - UNRELEASED
     MAPREDUCE-6460. TestRMContainerAllocator.
     testAttemptNotFoundCausesRMCommunicatorException fails. (Zhihai Xu)
 
+    MAPREDUCE-6484. Yarn Client uses local address instead of RM address as
+    token renewer in a secure cluster when RM HA is enabled. (Zhihai Xu)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5ba1a05/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
index 3bacc5e..d84e395 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
@@ -21,18 +21,23 @@ package org.apache.hadoop.mapred;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
 @Private
 @Unstable
 public class Master {
-  
+
+  private static final Log LOG = LogFactory.getLog(Master.class);
+
   public enum State {
     INITIALIZING, RUNNING;
   }
@@ -53,8 +58,24 @@ public class Master {
     if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
       masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
       return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
-    } 
-    else {
+    } else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) &&
+        HAUtil.isHAEnabled(conf)) {
+      YarnConfiguration yarnConf = new YarnConfiguration(conf);
+      if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
+        String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
+        if (rmIds != null && rmIds.length > 0) {
+          // If RM_HA_ID is not configured, use the first one.
+          // Because any valid RM HA ID should work.
+          yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
+        } else {
+          LOG.warn("RM_HA_IDS is not configured when RM HA is enabled");
+        }
+      }
+      return yarnConf.getSocketAddr(
+          YarnConfiguration.RM_ADDRESS,
+          YarnConfiguration.DEFAULT_RM_ADDRESS,
+          YarnConfiguration.DEFAULT_RM_PORT);
+    } else {
       return conf.getSocketAddr(
           YarnConfiguration.RM_ADDRESS,
           YarnConfiguration.DEFAULT_RM_ADDRESS,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5ba1a05/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
index 58bbb3e..498abbc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
@@ -64,6 +64,19 @@ public class TestMaster {
     masterHostname = Master.getMasterAddress(conf).getHostName();
     assertEquals(masterHostname, "foo1.com");
 
+    // change framework to yarn and enable HA
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+    conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
+    conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192");
+    conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192");
+    masterHostname = Master.getMasterAddress(conf).getHostName();
+    // If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used.
+    assertEquals(masterHostname, "rm1.com");
+    conf.set(YarnConfiguration.RM_HA_ID, "rm2");
+    masterHostname = Master.getMasterAddress(conf).getHostName();
+    // If RM_HA_ID is configured, use the given RM_HA_ID.
+    assertEquals(masterHostname, "rm2.com");
   }
 
   @Test 


Mime
View raw message