hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject hadoop git commit: YARN-4092. Fixed UI redirection to print useful messages when both RMs are in standby mode. Contributed by Xuan Gong (cherry picked from commit a3fd2ccc869dfc1f04d1cf0a8678d4d90a43a80f) (cherry picked from commit 48f5161cd5d4c2f4e385b2
Date Tue, 01 Sep 2015 00:46:35 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 dce4ef20f -> f44ed4f4b


YARN-4092. Fixed UI redirection to print useful messages when both RMs are in standby mode.
Contributed by Xuan Gong
(cherry picked from commit a3fd2ccc869dfc1f04d1cf0a8678d4d90a43a80f)
(cherry picked from commit 48f5161cd5d4c2f4e385b253a5bea066b2e23b9e)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f44ed4f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f44ed4f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f44ed4f4

Branch: refs/heads/branch-2.7
Commit: f44ed4f4b09b6a3f8b3679f687b3c69bf95adcb7
Parents: dce4ef2
Author: Jian He <jianhe@apache.org>
Authored: Mon Aug 31 17:33:24 2015 -0700
Committer: Jian He <jianhe@apache.org>
Committed: Mon Aug 31 17:46:27 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../hadoop/yarn/client/TestRMFailover.java      | 27 ++++++
 .../hadoop/yarn/webapp/YarnWebParams.java       |  1 +
 .../resourcemanager/webapp/RMWebAppFilter.java  | 90 +++++++++++++++++++-
 4 files changed, 117 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f44ed4f4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0fc28b2..e07c8ae 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -17,6 +17,9 @@ Release 2.7.2 - UNRELEASED
     YARN-3978. Configurably turn off the saving of container info in Generic AHS
     (Eric Payne via jeagles)
 
+    YARN-4092. Fixed UI redirection to print useful messages when both RMs are
+    in standby mode. (Xuan Gong via jianhe)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f44ed4f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index cd22743..d42d14a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -27,6 +27,7 @@ import static org.junit.Assert.fail;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
+
 import javax.servlet.http.HttpServletResponse;
 
 import org.apache.commons.logging.Log;
@@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -285,6 +287,7 @@ public class TestRMFailover extends ClientBaseWithFixes {
     getAdminService(0).transitionToActive(req);
     String rm1Url = "http://0.0.0.0:18088";
     String rm2Url = "http://0.0.0.0:28088";
+
     String redirectURL = getRedirectURL(rm2Url);
     // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection
url
     assertEquals(redirectURL,rm1Url+"/");
@@ -324,6 +327,17 @@ public class TestRMFailover extends ClientBaseWithFixes {
 
     redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
     assertNull(redirectURL);
+
+    // transit the active RM to standby
+    // Both of RMs are in standby mode
+    getAdminService(0).transitionToStandby(req);
+    // RM2 is expected to send the httpRequest to itself.
+    // The Header Field: Refresh is expected to be set.
+    redirectURL = getRefreshURL(rm2Url);
+    assertTrue(redirectURL != null
+        && redirectURL.contains(YarnWebParams.NEXT_REFRESH_INTERVAL)
+        && redirectURL.contains(rm2Url));
+
   }
 
   // set up http connection with the given url and get the redirection url from the response
@@ -343,4 +357,17 @@ public class TestRMFailover extends ClientBaseWithFixes {
     return redirectUrl;
   }
 
+  static String getRefreshURL(String url) {
+    String redirectUrl = null;
+    try {
+      HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
+      // do not automatically follow the redirection
+      // otherwise we get too many redirections exception
+      conn.setInstanceFollowRedirects(false);
+      redirectUrl = conn.getHeaderField("Refresh");
+    } catch (Exception e) {
+      // throw new RuntimeException(e);
+    }
+    return redirectUrl;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f44ed4f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 1200690..405d319 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -37,4 +37,5 @@ public interface YarnWebParams {
   String NODE_STATE = "node.state";
   String NODE_LABEL = "node.label";
   String WEB_UI_TYPE = "web.ui.type";
+  String NEXT_REFRESH_INTERVAL = "next.fresh.interval";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f44ed4f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
index 500f17a..a8f793a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Random;
 import java.util.Set;
 
 import javax.inject.Inject;
@@ -29,8 +33,11 @@ import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HtmlQuoting;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
 
 import com.google.common.collect.Sets;
 import com.google.inject.Injector;
@@ -48,11 +55,26 @@ public class RMWebAppFilter extends GuiceContainer {
   // define a set of URIs which do not need to do redirection
   private static final Set<String> NON_REDIRECTED_URIS = Sets.newHashSet(
       "/conf", "/stacks", "/logLevel", "/logs");
+  private String path;
+  private static final int BASIC_SLEEP_TIME = 5;
+  private static final int MAX_SLEEP_TIME = 5 * 60;
 
   @Inject
-  public RMWebAppFilter(Injector injector) {
+  public RMWebAppFilter(Injector injector, Configuration conf) {
     super(injector);
     this.injector=injector;
+    InetSocketAddress sock = YarnConfiguration.useHttps(conf)
+        ? conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
+            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
+            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT)
+        : conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
+            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
+            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+
+    path = sock.getHostName() + ":" + Integer.toString(sock.getPort());
+    path = YarnConfiguration.useHttps(conf)
+        ? "https://" + path
+        : "http://" + path;
   }
 
   @Override
@@ -69,9 +91,11 @@ public class RMWebAppFilter extends GuiceContainer {
     rmWebApp.checkIfStandbyRM();
     if (rmWebApp.isStandby()
         && shouldRedirect(rmWebApp, uri)) {
-      String redirectPath = rmWebApp.getRedirectPath() + uri;
+
+      String redirectPath = rmWebApp.getRedirectPath();
 
       if (redirectPath != null && !redirectPath.isEmpty()) {
+        redirectPath += uri;
         String redirectMsg =
             "This is standby RM. The redirect url is: " + redirectPath;
         PrintWriter out = response.getWriter();
@@ -79,11 +103,40 @@ public class RMWebAppFilter extends GuiceContainer {
         response.setHeader("Location", redirectPath);
         response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
         return;
+      } else {
+        boolean doRetry = true;
+        String retryIntervalStr =
+            request.getParameter(YarnWebParams.NEXT_REFRESH_INTERVAL);
+        int retryInterval = 0;
+        if (retryIntervalStr != null) {
+          try {
+            retryInterval = Integer.parseInt(retryIntervalStr.trim());
+          } catch (NumberFormatException ex) {
+            doRetry = false;
+          }
+        }
+        int next = calculateExponentialTime(retryInterval);
+
+        String redirectUrl =
+            appendOrReplaceParamter(path + uri,
+              YarnWebParams.NEXT_REFRESH_INTERVAL + "=" + (retryInterval + 1));
+        if (redirectUrl == null || next > MAX_SLEEP_TIME) {
+          doRetry = false;
+        }
+        String redirectMsg =
+            doRetry ? "Can not find any active RM. Will retry in next " + next
+                + " seconds." : "There is no active RM right now.";
+        PrintWriter out = response.getWriter();
+        out.println(redirectMsg);
+        if (doRetry) {
+          response.setHeader("Refresh", next + ";url=" + redirectUrl);
+          response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+        }
       }
+      return;
     }
 
     super.doFilter(request, response, chain);
-
   }
 
   private boolean shouldRedirect(RMWebApp rmWebApp, String uri) {
@@ -92,4 +145,33 @@ public class RMWebAppFilter extends GuiceContainer {
         && !uri.startsWith(ProxyUriUtils.PROXY_BASE)
         && !NON_REDIRECTED_URIS.contains(uri);
   }
-}
+
+  private String appendOrReplaceParamter(String uri, String newQuery) {
+    if (uri.contains(YarnWebParams.NEXT_REFRESH_INTERVAL + "=")) {
+      return uri.replaceAll(YarnWebParams.NEXT_REFRESH_INTERVAL + "=[^&]+",
+        newQuery);
+    }
+    try {
+      URI oldUri = new URI(uri);
+      String appendQuery = oldUri.getQuery();
+      if (appendQuery == null) {
+        appendQuery = newQuery;
+      } else {
+        appendQuery += "&" + newQuery;
+      }
+
+      URI newUri =
+          new URI(oldUri.getScheme(), oldUri.getAuthority(), oldUri.getPath(),
+            appendQuery, oldUri.getFragment());
+
+      return newUri.toString();
+    } catch (URISyntaxException e) {
+      return null;
+    }
+  }
+
+  private static int calculateExponentialTime(int retries) {
+    long baseTime = BASIC_SLEEP_TIME * (1L << retries);
+    return (int) (baseTime * ((new Random()).nextDouble() + 0.5));
+  }
+}
\ No newline at end of file


Mime
View raw message