Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 19AC310C7C for ; Tue, 1 Sep 2015 00:46:36 +0000 (UTC) Received: (qmail 36913 invoked by uid 500); 1 Sep 2015 00:46:35 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 36847 invoked by uid 500); 1 Sep 2015 00:46:35 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 36838 invoked by uid 99); 1 Sep 2015 00:46:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Sep 2015 00:46:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BC132E0454; Tue, 1 Sep 2015 00:46:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Message-Id: <597de8681a084d70b77ea0fd35c65577@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-4092. Fixed UI redirection to print useful messages when both RMs are in standby mode. Contributed by Xuan Gong (cherry picked from commit a3fd2ccc869dfc1f04d1cf0a8678d4d90a43a80f) (cherry picked from commit 48f5161cd5d4c2f4e385b2 Date: Tue, 1 Sep 2015 00:46:35 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.7 dce4ef20f -> f44ed4f4b YARN-4092. Fixed UI redirection to print useful messages when both RMs are in standby mode. Contributed by Xuan Gong (cherry picked from commit a3fd2ccc869dfc1f04d1cf0a8678d4d90a43a80f) (cherry picked from commit 48f5161cd5d4c2f4e385b253a5bea066b2e23b9e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f44ed4f4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f44ed4f4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f44ed4f4 Branch: refs/heads/branch-2.7 Commit: f44ed4f4b09b6a3f8b3679f687b3c69bf95adcb7 Parents: dce4ef2 Author: Jian He Authored: Mon Aug 31 17:33:24 2015 -0700 Committer: Jian He Committed: Mon Aug 31 17:46:27 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/client/TestRMFailover.java | 27 ++++++ .../hadoop/yarn/webapp/YarnWebParams.java | 1 + .../resourcemanager/webapp/RMWebAppFilter.java | 90 +++++++++++++++++++- 4 files changed, 117 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f44ed4f4/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0fc28b2..e07c8ae 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -17,6 +17,9 @@ Release 2.7.2 - UNRELEASED YARN-3978. Configurably turn off the saving of container info in Generic AHS (Eric Payne via jeagles) + YARN-4092. Fixed UI redirection to print useful messages when both RMs are + in standby mode. (Xuan Gong via jianhe) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/f44ed4f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index cd22743..d42d14a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -27,6 +27,7 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; + import javax.servlet.http.HttpServletResponse; import org.apache.commons.logging.Log; @@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -285,6 +287,7 @@ public class TestRMFailover extends ClientBaseWithFixes { getAdminService(0).transitionToActive(req); String rm1Url = "http://0.0.0.0:18088"; String rm2Url = "http://0.0.0.0:28088"; + String redirectURL = getRedirectURL(rm2Url); // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url assertEquals(redirectURL,rm1Url+"/"); @@ -324,6 +327,17 @@ public class TestRMFailover extends ClientBaseWithFixes { redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId); assertNull(redirectURL); + + // transit the active RM to standby + // Both of RMs are in standby mode + getAdminService(0).transitionToStandby(req); + // RM2 is expected to send the httpRequest to itself. + // The Header Field: Refresh is expected to be set. + redirectURL = getRefreshURL(rm2Url); + assertTrue(redirectURL != null + && redirectURL.contains(YarnWebParams.NEXT_REFRESH_INTERVAL) + && redirectURL.contains(rm2Url)); + } // set up http connection with the given url and get the redirection url from the response @@ -343,4 +357,17 @@ public class TestRMFailover extends ClientBaseWithFixes { return redirectUrl; } + static String getRefreshURL(String url) { + String redirectUrl = null; + try { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + // do not automatically follow the redirection + // otherwise we get too many redirections exception + conn.setInstanceFollowRedirects(false); + redirectUrl = conn.getHeaderField("Refresh"); + } catch (Exception e) { + // throw new RuntimeException(e); + } + return redirectUrl; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f44ed4f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index 1200690..405d319 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -37,4 +37,5 @@ public interface YarnWebParams { String NODE_STATE = "node.state"; String NODE_LABEL = "node.label"; String WEB_UI_TYPE = "web.ui.type"; + String NEXT_REFRESH_INTERVAL = "next.fresh.interval"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f44ed4f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java index 500f17a..a8f793a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java @@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; import java.io.PrintWriter; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Random; import java.util.Set; import javax.inject.Inject; @@ -29,8 +33,11 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HtmlQuoting; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import com.google.common.collect.Sets; import com.google.inject.Injector; @@ -48,11 +55,26 @@ public class RMWebAppFilter extends GuiceContainer { // define a set of URIs which do not need to do redirection private static final Set NON_REDIRECTED_URIS = Sets.newHashSet( "/conf", "/stacks", "/logLevel", "/logs"); + private String path; + private static final int BASIC_SLEEP_TIME = 5; + private static final int MAX_SLEEP_TIME = 5 * 60; @Inject - public RMWebAppFilter(Injector injector) { + public RMWebAppFilter(Injector injector, Configuration conf) { super(injector); this.injector=injector; + InetSocketAddress sock = YarnConfiguration.useHttps(conf) + ? conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT) + : conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); + + path = sock.getHostName() + ":" + Integer.toString(sock.getPort()); + path = YarnConfiguration.useHttps(conf) + ? "https://" + path + : "http://" + path; } @Override @@ -69,9 +91,11 @@ public class RMWebAppFilter extends GuiceContainer { rmWebApp.checkIfStandbyRM(); if (rmWebApp.isStandby() && shouldRedirect(rmWebApp, uri)) { - String redirectPath = rmWebApp.getRedirectPath() + uri; + + String redirectPath = rmWebApp.getRedirectPath(); if (redirectPath != null && !redirectPath.isEmpty()) { + redirectPath += uri; String redirectMsg = "This is standby RM. The redirect url is: " + redirectPath; PrintWriter out = response.getWriter(); @@ -79,11 +103,40 @@ public class RMWebAppFilter extends GuiceContainer { response.setHeader("Location", redirectPath); response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); return; + } else { + boolean doRetry = true; + String retryIntervalStr = + request.getParameter(YarnWebParams.NEXT_REFRESH_INTERVAL); + int retryInterval = 0; + if (retryIntervalStr != null) { + try { + retryInterval = Integer.parseInt(retryIntervalStr.trim()); + } catch (NumberFormatException ex) { + doRetry = false; + } + } + int next = calculateExponentialTime(retryInterval); + + String redirectUrl = + appendOrReplaceParamter(path + uri, + YarnWebParams.NEXT_REFRESH_INTERVAL + "=" + (retryInterval + 1)); + if (redirectUrl == null || next > MAX_SLEEP_TIME) { + doRetry = false; + } + String redirectMsg = + doRetry ? "Can not find any active RM. Will retry in next " + next + + " seconds." : "There is no active RM right now."; + PrintWriter out = response.getWriter(); + out.println(redirectMsg); + if (doRetry) { + response.setHeader("Refresh", next + ";url=" + redirectUrl); + response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); + } } + return; } super.doFilter(request, response, chain); - } private boolean shouldRedirect(RMWebApp rmWebApp, String uri) { @@ -92,4 +145,33 @@ public class RMWebAppFilter extends GuiceContainer { && !uri.startsWith(ProxyUriUtils.PROXY_BASE) && !NON_REDIRECTED_URIS.contains(uri); } -} + + private String appendOrReplaceParamter(String uri, String newQuery) { + if (uri.contains(YarnWebParams.NEXT_REFRESH_INTERVAL + "=")) { + return uri.replaceAll(YarnWebParams.NEXT_REFRESH_INTERVAL + "=[^&]+", + newQuery); + } + try { + URI oldUri = new URI(uri); + String appendQuery = oldUri.getQuery(); + if (appendQuery == null) { + appendQuery = newQuery; + } else { + appendQuery += "&" + newQuery; + } + + URI newUri = + new URI(oldUri.getScheme(), oldUri.getAuthority(), oldUri.getPath(), + appendQuery, oldUri.getFragment()); + + return newUri.toString(); + } catch (URISyntaxException e) { + return null; + } + } + + private static int calculateExponentialTime(int retries) { + long baseTime = BASIC_SLEEP_TIME * (1L << retries); + return (int) (baseTime * ((new Random()).nextDouble() + 0.5)); + } +} \ No newline at end of file