YARN-7269. Tracking URL in the app state does not get redirected to ApplicationMaster for Running
applications. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4111e6c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4111e6c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4111e6c7
Branch: refs/heads/HDFS-9806
Commit: 4111e6c7819d66e17e0932302c589777a355dbae
Parents: 453d48b
Author: Jian He <jianhe@apache.org>
Authored: Tue Oct 3 09:54:38 2017 -0700
Committer: Jian He <jianhe@apache.org>
Committed: Tue Oct 3 09:54:38 2017 -0700
----------------------------------------------------------------------
.../webproxy/amfilter/AmFilterInitializer.java | 38 +++++++++--
.../server/webproxy/amfilter/AmIpFilter.java | 67 ++++++++------------
.../server/webproxy/amfilter/TestAmFilter.java | 8 +--
.../amfilter/TestAmFilterInitializer.java | 14 ++--
4 files changed, 70 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4111e6c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
index 6841616..fa54c79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
@@ -18,21 +18,26 @@
package org.apache.hadoop.yarn.server.webproxy.amfilter;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.FilterContainer;
import org.apache.hadoop.http.FilterInitializer;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
public class AmFilterInitializer extends FilterInitializer {
private static final String FILTER_NAME = "AM_PROXY_FILTER";
private static final String FILTER_CLASS = AmIpFilter.class.getCanonicalName();
+ public static final String RM_HA_URLS = "RM_HA_URLS";
@Override
public void initFilter(FilterContainer container, Configuration conf) {
@@ -55,6 +60,29 @@ public class AmFilterInitializer extends FilterInitializer {
sb.setLength(sb.length() - 1);
params.put(AmIpFilter.PROXY_URI_BASES, sb.toString());
container.addFilter(FILTER_NAME, FILTER_CLASS, params);
+
+ // Handle RM HA urls
+ List<String> urls = new ArrayList<>();
+
+ // Include yarn-site.xml in the classpath
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ for (String rmId : getRmIds(yarnConf)) {
+ String url = getUrlByRmId(yarnConf, rmId);
+ urls.add(url);
+ }
+ params.put(RM_HA_URLS, StringUtils.join(",", urls));
+ }
+
+ private Collection<String> getRmIds(Configuration conf) {
+ return conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
+ }
+
+ private String getUrlByRmId(Configuration conf, String rmId) {
+ String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ?
+ YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS :
+ YarnConfiguration.RM_WEBAPP_ADDRESS;
+ String host = conf.get(HAUtil.addSuffix(addressPropertyPrefix, rmId));
+ return host;
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4111e6c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
index cdab405..ac4afa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
@@ -18,17 +18,12 @@
package org.apache.hadoop.yarn.server.webproxy.amfilter;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.net.HttpURLConnection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.Collection;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -39,15 +34,16 @@ import javax.servlet.ServletResponse;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
-import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
@Public
public class AmIpFilter implements Filter {
@@ -70,6 +66,7 @@ public class AmIpFilter implements Filter {
private long lastUpdate;
@VisibleForTesting
Map<String, String> proxyUriBases;
+ String rmUrls[] = null;
@Override
public void init(FilterConfig conf) throws ServletException {
@@ -95,6 +92,10 @@ public class AmIpFilter implements Filter {
}
}
}
+
+ if (conf.getInitParameter(AmFilterInitializer.RM_HA_URLS) != null) {
+ rmUrls = conf.getInitParameter(AmFilterInitializer.RM_HA_URLS).split(",");
+ }
}
protected Set<String> getProxyAddresses() throws ServletException {
@@ -196,13 +197,11 @@ public class AmIpFilter implements Filter {
if (proxyUriBases.size() == 1) {
// external proxy or not RM HA
addr = proxyUriBases.values().iterator().next();
- } else {
- // RM HA
- YarnConfiguration conf = new YarnConfiguration();
- for (String rmId : getRmIds(conf)) {
- String url = getUrlByRmId(conf, rmId);
- if (isValidUrl(url)) {
- addr = url;
+ } else if (rmUrls != null) {
+ for (String url : rmUrls) {
+ String host = proxyUriBases.get(url);
+ if (isValidUrl(host)) {
+ addr = host;
break;
}
}
@@ -215,20 +214,6 @@ public class AmIpFilter implements Filter {
return addr;
}
- @VisibleForTesting
- Collection<String> getRmIds(YarnConfiguration conf) {
- return conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
- }
-
- @VisibleForTesting
- String getUrlByRmId(YarnConfiguration conf, String rmId) {
- String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ?
- YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS :
- YarnConfiguration.RM_WEBAPP_ADDRESS;
- String host = conf.get(HAUtil.addSuffix(addressPropertyPrefix, rmId));
- return proxyUriBases.get(host);
- }
-
private boolean isValidUrl(String url) {
boolean isValid = false;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4111e6c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
index 687faea..f775c6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
@@ -159,13 +159,7 @@ public class TestAmFilter {
spy.proxyUriBases = new HashMap<>();
spy.proxyUriBases.put(rm1, rm1Url);
spy.proxyUriBases.put(rm2, rm2Url);
-
- Collection<String> rmIds = new ArrayList<>(Arrays.asList(rm1, rm2));
- Mockito.doReturn(rmIds).when(spy).getRmIds(Mockito.any());
- Mockito.doReturn(rm1Url).when(spy)
- .getUrlByRmId(Mockito.any(), Mockito.eq(rm2));
- Mockito.doReturn(rm2Url).when(spy)
- .getUrlByRmId(Mockito.any(), Mockito.eq(rm1));
+ spy.rmUrls = new String[] { rm1, rm2 };
assertEquals(spy.findRedirectUrl(), rm1Url);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4111e6c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
index 07478ca..b621012 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
@@ -56,10 +56,11 @@ public class TestAmFilterInitializer {
AmFilterInitializer afi = new MockAmFilterInitializer();
assertNull(con.givenParameters);
afi.initFilter(con, conf);
- assertEquals(2, con.givenParameters.size());
+ assertEquals(3, con.givenParameters.size());
assertEquals("host1", con.givenParameters.get(AmIpFilter.PROXY_HOSTS));
assertEquals("http://host1:1000/foo",
con.givenParameters.get(AmIpFilter.PROXY_URI_BASES));
+ assertEquals("", con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
// Check a single RM_WEBAPP_ADDRESS
con = new MockFilterContainer();
@@ -68,10 +69,11 @@ public class TestAmFilterInitializer {
afi = new MockAmFilterInitializer();
assertNull(con.givenParameters);
afi.initFilter(con, conf);
- assertEquals(2, con.givenParameters.size());
+ assertEquals(3, con.givenParameters.size());
assertEquals("host2", con.givenParameters.get(AmIpFilter.PROXY_HOSTS));
assertEquals("http://host2:2000/foo",
con.givenParameters.get(AmIpFilter.PROXY_URI_BASES));
+ assertEquals("", con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
// Check multiple RM_WEBAPP_ADDRESSes (RM HA)
con = new MockFilterContainer();
@@ -84,7 +86,7 @@ public class TestAmFilterInitializer {
afi = new MockAmFilterInitializer();
assertNull(con.givenParameters);
afi.initFilter(con, conf);
- assertEquals(2, con.givenParameters.size());
+ assertEquals(3, con.givenParameters.size());
String[] proxyHosts = con.givenParameters.get(AmIpFilter.PROXY_HOSTS)
.split(AmIpFilter.PROXY_HOSTS_DELIMITER);
assertEquals(3, proxyHosts.length);
@@ -99,6 +101,8 @@ public class TestAmFilterInitializer {
assertEquals("http://host2:2000/foo", proxyBases[0]);
assertEquals("http://host3:3000/foo", proxyBases[1]);
assertEquals("http://host4:4000/foo", proxyBases[2]);
+ assertEquals("host2:2000,host3:3000,host4:4000",
+ con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
// Check multiple RM_WEBAPP_ADDRESSes (RM HA) with HTTPS
con = new MockFilterContainer();
@@ -112,7 +116,7 @@ public class TestAmFilterInitializer {
afi = new MockAmFilterInitializer();
assertNull(con.givenParameters);
afi.initFilter(con, conf);
- assertEquals(2, con.givenParameters.size());
+ assertEquals(3, con.givenParameters.size());
proxyHosts = con.givenParameters.get(AmIpFilter.PROXY_HOSTS)
.split(AmIpFilter.PROXY_HOSTS_DELIMITER);
assertEquals(2, proxyHosts.length);
@@ -125,6 +129,8 @@ public class TestAmFilterInitializer {
Arrays.sort(proxyBases);
assertEquals("https://host5:5000/foo", proxyBases[0]);
assertEquals("https://host6:6000/foo", proxyBases[1]);
+ assertEquals("host5:5000,host6:6000",
+ con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
|