hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sunc...@apache.org
Subject [4/8] hadoop git commit: YARN-9130. Add Bind_HOST configuration for Yarn Web Proxy. Contributed by Rong Tang.
Date Thu, 20 Dec 2018 06:21:36 GMT
YARN-9130. Add Bind_HOST configuration for Yarn Web Proxy. Contributed by Rong Tang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5df9fb16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5df9fb16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5df9fb16

Branch: refs/heads/HDFS-12943
Commit: 5df9fb16b9871618f19f3081b7006d3e77106737
Parents: 499c70e
Author: Inigo Goiri <inigoiri@apache.org>
Authored: Wed Dec 19 18:58:00 2018 -0800
Committer: Inigo Goiri <inigoiri@apache.org>
Committed: Wed Dec 19 18:58:00 2018 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/conf/YarnConfiguration.java     |  6 +++-
 .../src/main/resources/yarn-default.xml         |  9 ++++++
 .../yarn/server/webproxy/WebAppProxy.java       | 14 +++++++--
 .../yarn/server/webproxy/WebAppProxyServer.java | 10 ++++---
 .../server/webproxy/TestWebAppProxyServer.java  | 31 ++++++++++++++++----
 5 files changed, 57 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 72cc0d6..d84d11a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2302,13 +2302,17 @@ public class YarnConfiguration extends Configuration {
   
   /** Keytab for Proxy.*/
   public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab";
-  
+
   /** The address for the web proxy.*/
   public static final String PROXY_ADDRESS =
     PROXY_PREFIX + "address";
   public static final int DEFAULT_PROXY_PORT = 9099;
   public static final String DEFAULT_PROXY_ADDRESS =
     "0.0.0.0:" + DEFAULT_PROXY_PORT;
+
+  /** Binding address for the web proxy. */
+  public static final String PROXY_BIND_HOST =
+      PROXY_PREFIX + "bind-host";
   
   /**
    * YARN Service Level Authorization

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index ccceaab..12c5ff6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2157,6 +2157,15 @@
      <value/>
   </property>
 
+  <property>
+    <description>The actual address the web proxy will bind to. If this optional
+     address is set, it overrides only the hostname portion of yarn.web-proxy.address.
+     This is useful for making the web proxy server listen on all interfaces by setting
+     it to 0.0.0.0 </description>
+    <name>yarn.web-proxy.bind-host</name>
+    <value/>
+  </property>
+
   <!-- Applications' Configuration -->
 
   <property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
index f205cf7..1382fc6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
@@ -74,16 +74,26 @@ public class WebAppProxy extends AbstractService {
     fetcher = new AppReportFetcher(conf);
     bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
     if(bindAddress == null || bindAddress.isEmpty()) {
-      throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS + 
+      throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS +
           " is not set so the proxy will not run.");
     }
-    LOG.info("Instantiating Proxy at " + bindAddress);
+
     String[] parts = StringUtils.split(bindAddress, ':');
     port = 0;
     if (parts.length == 2) {
       bindAddress = parts[0];
       port = Integer.parseInt(parts[1]);
     }
+
+    String bindHost = conf.getTrimmed(YarnConfiguration.PROXY_BIND_HOST, null);
+    if (bindHost != null) {
+      LOG.debug("{} is set, will be used to run proxy.",
+          YarnConfiguration.PROXY_BIND_HOST);
+      bindAddress = bindHost;
+    }
+
+    LOG.info("Instantiating Proxy at {}:{}", bindAddress, port);
+
     acl = new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL, 
         YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
     super.serviceInit(conf);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
index a5dd8f6..6be6039 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
@@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  * ProxyServer will sit in between the end user and AppMaster
- * web interfaces. 
+ * web interfaces.
  */
 public class WebAppProxyServer extends CompositeService {
 
@@ -103,9 +103,11 @@ public class WebAppProxyServer extends CompositeService {
    * @return InetSocketAddress
    */
   public static InetSocketAddress getBindAddress(Configuration conf) {
-    return conf.getSocketAddr(YarnConfiguration.PROXY_ADDRESS,
-      YarnConfiguration.DEFAULT_PROXY_ADDRESS,
-      YarnConfiguration.DEFAULT_PROXY_PORT);
+    return conf.getSocketAddr(
+        YarnConfiguration.PROXY_BIND_HOST,
+        YarnConfiguration.PROXY_ADDRESS,
+        YarnConfiguration.DEFAULT_PROXY_ADDRESS,
+        YarnConfiguration.DEFAULT_PROXY_PORT);
   }
 
   public static void main(String[] args) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
index 93479a9..87907a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertEquals;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -33,14 +32,15 @@ import java.net.InetSocketAddress;
 
 public class TestWebAppProxyServer {
   private WebAppProxyServer webAppProxy = null;
-  private final String proxyAddress = "0.0.0.0:8888";
+  private final String port = "8888";
+  private final String proxyAddress = "localhost:" + port;
+  private YarnConfiguration conf = null;
 
   @Before
   public void setUp() throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
+    conf = new YarnConfiguration();
     conf.set(YarnConfiguration.PROXY_ADDRESS, proxyAddress);
     webAppProxy = new WebAppProxyServer();
-    webAppProxy.init(conf);
   }
 
   @After
@@ -50,19 +50,38 @@ public class TestWebAppProxyServer {
 
   @Test
   public void testStart() {
+    webAppProxy.init(conf);
     assertEquals(STATE.INITED, webAppProxy.getServiceState());
     webAppProxy.start();
     for (Service service : webAppProxy.getServices()) {
       if (service instanceof WebAppProxy) {
-        assertEquals(((WebAppProxy) service).getBindAddress(), proxyAddress);
+        assertEquals(proxyAddress, ((WebAppProxy) service).getBindAddress());
       }
     }
     assertEquals(STATE.STARTED, webAppProxy.getServiceState());
   }
 
   @Test
+  public void testStartWithBindHost() {
+    String bindHost = "0.0.0.0";
+    conf.set(YarnConfiguration.PROXY_BIND_HOST, bindHost);
+    webAppProxy.init(conf);
+
+    assertEquals(STATE.INITED, webAppProxy.getServiceState());
+    webAppProxy.start();
+    for (Service service : webAppProxy.getServices()) {
+      if (service instanceof WebAppProxy) {
+        assertEquals(bindHost + ":" + port,
+            ((WebAppProxy) service).getBindAddress());
+      }
+    }
+    assertEquals(STATE.STARTED, webAppProxy.getServiceState());
+  }
+
+
+  @Test
   public void testBindAddress() {
-    YarnConfiguration conf = new YarnConfiguration();
+    conf = new YarnConfiguration();
 
     InetSocketAddress defaultBindAddress = WebAppProxyServer.getBindAddress(conf);
     Assert.assertEquals("Web Proxy default bind address port is incorrect",


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message