hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject [hadoop] branch trunk updated: YARN-9956. Improved connection error message for YARN ApiServerClient. Contributed by Prabhu Joseph
Date Mon, 06 Jan 2020 18:26:22 GMT
This is an automated email from the ASF dual-hosted git repository.

eyang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new d81d45f  YARN-9956. Improved connection error message for YARN ApiServerClient. 
          Contributed by Prabhu Joseph
d81d45f is described below

commit d81d45ff2fc9a1c424222e021f9306bf64c916b2
Author: Eric Yang <eyang@apache.org>
AuthorDate: Mon Jan 6 13:24:16 2020 -0500

    YARN-9956. Improved connection error message for YARN ApiServerClient.
               Contributed by Prabhu Joseph
---
 .../yarn/service/client/ApiServiceClient.java      | 78 ++++++++++++----------
 .../yarn/service/client/TestApiServiceClient.java  | 22 ++++++
 .../service/client/TestSecureApiServiceClient.java |  2 +
 .../hadoop/yarn/client/util/YarnClientUtils.java   |  2 +-
 4 files changed, 68 insertions(+), 36 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 834bb03..3c2c3c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.client.api.AppAdminClient;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.util.YarnClientUtils;
+import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.service.api.records.Component;
@@ -94,7 +95,7 @@ public class ApiServiceClient extends AppAdminClient {
   /**
    * Calculate Resource Manager address base on working REST API.
    */
-  String getRMWebAddress() {
+  String getRMWebAddress() throws IOException {
     Configuration conf = getConfig();
     String scheme = "http://";
     String path = "/app/v1/services/version";
@@ -105,43 +106,50 @@ public class ApiServiceClient extends AppAdminClient {
       rmAddress = conf
           .get("yarn.resourcemanager.webapp.https.address");
     }
-    boolean useKerberos = UserGroupInformation.isSecurityEnabled();
-    List<String> rmServers = getRMHAWebAddresses(conf);
-    for (String host : rmServers) {
-      try {
-        Client client = Client.create();
-        client.setFollowRedirects(false);
-        StringBuilder sb = new StringBuilder();
-        sb.append(scheme)
-            .append(host)
-            .append(path);
-        if (!useKerberos) {
-          try {
-            String username = UserGroupInformation.getCurrentUser().getShortUserName();
-            sb.append("?user.name=")
-                .append(username);
-          } catch (IOException e) {
-            LOG.debug("Fail to resolve username: {}", e);
+
+    if (HAUtil.isHAEnabled(conf)) {
+      boolean useKerberos = UserGroupInformation.isSecurityEnabled();
+      List<String> rmServers = getRMHAWebAddresses(conf);
+      StringBuilder diagnosticsMsg = new StringBuilder();
+      for (String host : rmServers) {
+        try {
+          Client client = Client.create();
+          client.setFollowRedirects(false);
+          StringBuilder sb = new StringBuilder();
+          sb.append(scheme)
+              .append(host)
+              .append(path);
+          if (!useKerberos) {
+            try {
+              String username = UserGroupInformation.getCurrentUser()
+                  .getShortUserName();
+              sb.append("?user.name=")
+                  .append(username);
+            } catch (IOException e) {
+              LOG.debug("Fail to resolve username: {}", e);
+            }
           }
+          Builder builder = client
+              .resource(sb.toString()).type(MediaType.APPLICATION_JSON);
+          if (useKerberos) {
+            String[] server = host.split(":");
+            String challenge = YarnClientUtils.generateToken(server[0]);
+            builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
+                challenge);
+            LOG.debug("Authorization: Negotiate {}", challenge);
+          }
+          ClientResponse test = builder.get(ClientResponse.class);
+          if (test.getStatus() == 200) {
+            return scheme + host;
+          }
+        } catch (Exception e) {
+          LOG.info("Fail to connect to: " + host);
+          LOG.debug("Root cause: ", e);
+          diagnosticsMsg.append("Error connecting to " + host
+              + " due to " + e.getMessage() + "\n");
         }
-        Builder builder = client
-            .resource(sb.toString()).type(MediaType.APPLICATION_JSON);
-        if (useKerberos) {
-          String[] server = host.split(":");
-          String challenge = YarnClientUtils.generateToken(server[0]);
-          builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
-              challenge);
-          LOG.debug("Authorization: Negotiate {}", challenge);
-        }
-        ClientResponse test = builder.get(ClientResponse.class);
-        if (test.getStatus() == 200) {
-          rmAddress = host;
-          break;
-        }
-      } catch (Exception e) {
-        LOG.info("Fail to connect to: "+host);
-        LOG.debug("Root cause: {}", e);
       }
+      throw new IOException(diagnosticsMsg.toString());
     }
     return scheme+rmAddress;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
index afdf16e..65849a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
@@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletResponse;
 import com.google.common.collect.Lists;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
@@ -118,6 +119,27 @@ public class TestApiServiceClient {
   }
 
   @Test
+  public void testGetRMWebAddress() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+    conf.set(YarnConfiguration.RM_HA_IDS, "rm1");
+    conf.set(YarnConfiguration.RM_HA_ID, "rm1");
+    conf.set("yarn.resourcemanager.webapp.address.rm1", "localhost:0");
+    ApiServiceClient asc1 = new ApiServiceClient(conf);
+    boolean exceptionCaught = false;
+    String diagnosticsMsg = null;
+    try {
+      String rmWebAddress = asc1.getRMWebAddress();
+    } catch (IOException e){
+      exceptionCaught = true;
+      diagnosticsMsg = e.getMessage();
+    }
+    assertTrue("ApiServiceClient failed to throw exception", exceptionCaught);
+    assertTrue("Exception Message does not match",
+        diagnosticsMsg.contains("Error connecting to localhost:0"));
+  }
+
+  @Test
   public void testLaunch() {
     String fileName = "target/test-classes/example-app.json";
     String appName = "example-app";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
index 8c7da62..e2d613a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.client.util.YarnClientUtils;
 import org.apache.log4j.Logger;
 import org.eclipse.jetty.server.Server;
@@ -152,6 +153,7 @@ public class TestSecureApiServiceClient extends KerberosSecurityTestcase
{
     rmServers.add("localhost:8088");
     testConf.set("yarn.resourcemanager.webapp.address",
         "localhost:8088");
+    testConf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
     asc = new ApiServiceClient() {
       @Override
       List<String> getRMHAWebAddresses(Configuration conf) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
index abed6c6..94b13a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
@@ -247,7 +247,7 @@ public abstract class YarnClientUtils {
                   StandardCharsets.US_ASCII);
             } catch (GSSException | IllegalAccessException
                 | NoSuchFieldException | ClassNotFoundException e) {
-              LOG.error("Error: {}", e);
+              LOG.error("Error: ", e);
               throw new AuthenticationException(e);
             }
           }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message