hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1296880 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ hadoop-yarn/hadoop-yarn-server/had...
Date Sun, 04 Mar 2012 21:15:54 GMT
Author: acmurthy
Date: Sun Mar  4 21:15:54 2012
New Revision: 1296880

URL: http://svn.apache.org/viewvc?rev=1296880&view=rev
Log:
Merge -c 1296878 from trunk to branch-0.23 to fix MAPREDUCE-3960. Fix web-proxy to forward
request to AM with configured hostname or IP.

Modified:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1296880&r1=1296879&r2=1296880&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Sun Mar  4 21:15:54
2012
@@ -170,6 +170,9 @@ Release 0.23.2 - UNRELEASED
     MAPREDUCE-3929. Fixed output of 'bin/mapred queue -showacl' command to 
     clarify ACLs for users. (John George via acmurthy) 
 
+    MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured
+    hostname or IP. (tgraves via acmurthy) 
+
 Release 0.23.1 - 2012-02-17
 
   NEW FEATURES

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1296880&r1=1296879&r2=1296880&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Sun Mar  4 21:15:54 2012
@@ -432,6 +432,10 @@ public class ResourceManager extends Com
       builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME, 
           ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
       builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
+      String proxy = YarnConfiguration.getProxyHostAndPort(conf);
+      String[] proxyParts = proxy.split(":");
+      builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]);
+
     }
     webApp = builder.start(new RMWebApp(this));
   }

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1296880&r1=1296879&r2=1296880&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
Sun Mar  4 21:15:54 2012
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.CommonConfig
 public class WebAppProxy extends AbstractService {
   public static final String FETCHER_ATTRIBUTE= "AppUrlFetcher";
   public static final String IS_SECURITY_ENABLED_ATTRIBUTE = "IsSecurityEnabled";
+  public static final String PROXY_HOST_ATTRIBUTE = "proxyHost";
   private static final Log LOG = LogFactory.getLog(WebAppProxy.class);
   
   private HttpServer proxyServer = null;
@@ -43,6 +44,7 @@ public class WebAppProxy extends Abstrac
   private AccessControlList acl = null;
   private AppReportFetcher fetcher = null;
   private boolean isSecurityEnabled = false;
+  private String proxyHost = null;
   
   public WebAppProxy() {
     super(WebAppProxy.class.getName());
@@ -60,6 +62,9 @@ public class WebAppProxy extends Abstrac
           CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION +
           " of " + auth);
     }
+    String proxy = YarnConfiguration.getProxyHostAndPort(conf);
+    String[] proxyParts = proxy.split(":");
+    proxyHost = proxyParts[0];
 
     fetcher = new AppReportFetcher(conf);
     bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
@@ -88,6 +93,7 @@ public class WebAppProxy extends Abstrac
           ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
       proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
       proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
+      proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
       proxyServer.start();
     } catch (IOException e) {
       LOG.fatal("Could not start proxy web server",e);

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java?rev=1296880&r1=1296879&r2=1296880&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
Sun Mar  4 21:15:54 2012
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintWriter;
+import java.net.InetAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLEncoder;
@@ -38,6 +39,7 @@ import javax.servlet.http.HttpServletRes
 import org.apache.commons.httpclient.Header;
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.HostConfiguration;
 import org.apache.commons.httpclient.cookie.CookiePolicy;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.params.HttpClientParams;
@@ -123,13 +125,23 @@ public class WebAppProxyServlet extends 
    * @throws IOException on any error.
    */
   private static void proxyLink(HttpServletRequest req, 
-      HttpServletResponse resp, URI link,Cookie c) throws IOException {
+      HttpServletResponse resp, URI link, Cookie c, String proxyHost)
+      throws IOException {
     org.apache.commons.httpclient.URI uri = 
       new org.apache.commons.httpclient.URI(link.toString(), false);
     HttpClientParams params = new HttpClientParams();
     params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
     params.setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true);
     HttpClient client = new HttpClient(params);
+    // Make sure we send the request from the proxy address in the config
+    // since that is what the AM filter checks against. IP aliasing or
+    // similar could cause issues otherwise.
+    HostConfiguration config = new HostConfiguration();
+    InetAddress localAddress = InetAddress.getByName(proxyHost);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("local InetAddress for proxy host: " + localAddress.toString());
+    }
+    config.setLocalAddress(localAddress);
     HttpMethod method = new GetMethod(uri.getEscapedURI());
 
     @SuppressWarnings("unchecked")
@@ -150,7 +162,7 @@ public class WebAppProxyServlet extends 
     }
     OutputStream out = resp.getOutputStream();
     try {
-      resp.setStatus(client.executeMethod(method));
+      resp.setStatus(client.executeMethod(config, method));
       for(Header header : method.getResponseHeaders()) {
         resp.setHeader(header.getName(), header.getValue());
       }
@@ -189,6 +201,11 @@ public class WebAppProxyServlet extends 
         .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE)).getApplicationReport(id);
   }
   
+  private String getProxyHost() throws IOException {
+    return ((String) getServletContext()
+        .getAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE));
+  }
+  
   @Override
   protected void doGet(HttpServletRequest req, HttpServletResponse resp) 
   throws IOException{
@@ -275,7 +292,7 @@ public class WebAppProxyServlet extends 
       if(userWasWarned && userApproved) {
         c = makeCheckCookie(id, true);
       }
-      proxyLink(req, resp, toFetch, c);
+      proxyLink(req, resp, toFetch, c, getProxyHost());
 
     } catch(URISyntaxException e) {
       throw new IOException(e); 

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java?rev=1296880&r1=1296879&r2=1296880&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
Sun Mar  4 21:15:54 2012
@@ -64,6 +64,9 @@ public class AmIpFilter implements Filte
         try {
           proxyAddresses = new HashSet<String>();
           for(InetAddress add : InetAddress.getAllByName(proxyHost)) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("proxy address is: " + add.getHostAddress());
+            }
             proxyAddresses.add(add.getHostAddress());
           }
           lastUpdate = now;
@@ -89,6 +92,9 @@ public class AmIpFilter implements Filte
     
     HttpServletRequest httpReq = (HttpServletRequest)req;
     HttpServletResponse httpResp = (HttpServletResponse)resp;
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Remote address for request is: " + httpReq.getRemoteAddr());
+    }
     if(!getProxyAddresses().contains(httpReq.getRemoteAddr())) {
       String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase + 
           httpReq.getRequestURI());



Mime
View raw message