hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject svn commit: r1619194 - in /hadoop/common/branches/fs-encryption/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/ hadoop-y...
Date Wed, 20 Aug 2014 18:19:14 GMT
Author: wang
Date: Wed Aug 20 18:19:13 2014
New Revision: 1619194

URL: http://svn.apache.org/r1619194
Log:
Merge from trunk to branch

Added:
    hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
      - copied unchanged from r1619192, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
Modified:
    hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
    hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java
    hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
    hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt?rev=1619194&r1=1619193&r2=1619194&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt Wed Aug 20 18:19:13
2014
@@ -50,6 +50,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2411. Support simple user and group mappings to queues. (Ram Venkatesh
     via jianhe)
 
+    YARN-2174. Enable HTTPs for the writer REST API of TimelineServer.
+    (Zhijie Shen via jianhe)
+
   IMPROVEMENTS
 
     YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
@@ -217,6 +220,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2249. Avoided AM release requests being lost on work preserving RM
     restart. (Jian He via zjshen)
 
+    YARN-2034. Description for yarn.nodemanager.localizer.cache.target-size-mb
+    is incorrect (Chen He via jlowe)
+
 Release 2.5.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1619194&r1=1619193&r2=1619194&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Wed Aug 20 18:19:13 2014
@@ -606,7 +606,11 @@ public class YarnConfiguration extends C
   public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 
     10 * 60 * 1000;
   
-  /** Target size of localizer cache in MB, per local directory.*/
+  /**
+   * Target size of localizer cache in MB, per nodemanager. It is a target
+   * retention size that only includes resources with PUBLIC and PRIVATE
+   * visibility and excludes resources with APPLICATION visibility
+   */
   public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB =
     NM_PREFIX + "localizer.cache.target-size-mb";
   public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024;

Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java?rev=1619194&r1=1619193&r2=1619194&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java
(original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java
Wed Aug 20 18:19:13 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.security.authentication.client.Authenticator;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
 import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse;
@@ -53,10 +54,13 @@ import com.google.common.annotations.Vis
 public class TimelineAuthenticator extends KerberosAuthenticator {
 
   private static ObjectMapper mapper;
+  private static TimelineAuthenticator authenticator;
+  private static ConnectionConfigurator connConfigurator;
 
   static {
     mapper = new ObjectMapper();
     YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
+    authenticator = new TimelineAuthenticator();
   }
 
   /**
@@ -98,6 +102,11 @@ public class TimelineAuthenticator exten
     }
   }
 
+  public static void setStaticConnectionConfigurator(
+      ConnectionConfigurator connConfigurator) {
+    TimelineAuthenticator.connConfigurator = connConfigurator;
+  }
+
   public static Token<TimelineDelegationTokenIdentifier> getDelegationToken(
       URL url, AuthenticatedURL.Token token, String renewer) throws IOException {
     TimelineDelegationTokenOperation op =
@@ -107,7 +116,7 @@ public class TimelineAuthenticator exten
     params.put(TimelineAuthenticationConsts.RENEWER_PARAM, renewer);
     url = appendParams(url, params);
     AuthenticatedURL aUrl =
-        new AuthenticatedURL(new TimelineAuthenticator());
+        new AuthenticatedURL(authenticator, connConfigurator);
     try {
       HttpURLConnection conn = aUrl.openConnection(url, token);
       conn.setRequestMethod(op.getHttpMethod());
@@ -137,7 +146,7 @@ public class TimelineAuthenticator exten
         dToken.encodeToUrlString());
     url = appendParams(url, params);
     AuthenticatedURL aUrl =
-        new AuthenticatedURL(new TimelineAuthenticator());
+        new AuthenticatedURL(authenticator, connConfigurator);
     try {
       HttpURLConnection conn = aUrl.openConnection(url, token);
       conn.setRequestMethod(
@@ -164,7 +173,7 @@ public class TimelineAuthenticator exten
         dToken.encodeToUrlString());
     url = appendParams(url, params);
     AuthenticatedURL aUrl =
-        new AuthenticatedURL(new TimelineAuthenticator());
+        new AuthenticatedURL(authenticator, connConfigurator);
     try {
       HttpURLConnection conn = aUrl.openConnection(url, token);
       conn.setRequestMethod(TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN

Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java?rev=1619194&r1=1619193&r2=1619194&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
(original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
Wed Aug 20 18:19:13 2014
@@ -23,10 +23,15 @@ import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URI;
 import java.net.URL;
+import java.net.URLConnection;
+import java.security.GeneralSecurityException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
 import javax.ws.rs.core.MediaType;
 
 import org.apache.commons.cli.CommandLine;
@@ -42,6 +47,8 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
@@ -74,7 +81,10 @@ public class TimelineClientImpl extends 
   private static final String RESOURCE_URI_STR = "/ws/v1/timeline/";
   private static final String URL_PARAM_USER_NAME = "user.name";
   private static final Joiner JOINER = Joiner.on("");
+  public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute
+
   private static Options opts;
+
   static {
     opts = new Options();
     opts.addOption("put", true, "Put the TimelineEntities in a JSON file");
@@ -89,15 +99,6 @@ public class TimelineClientImpl extends 
 
   public TimelineClientImpl() {
     super(TimelineClientImpl.class.getName());
-    ClientConfig cc = new DefaultClientConfig();
-    cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
-    if (UserGroupInformation.isSecurityEnabled()) {
-      urlFactory = new KerberosAuthenticatedURLConnectionFactory();
-      client = new Client(new URLConnectionClientHandler(urlFactory), cc);
-    } else {
-      client = new Client(new URLConnectionClientHandler(
-          new PseudoAuthenticatedURLConnectionFactory()), cc);
-    }
   }
 
   protected void serviceInit(Configuration conf) throws Exception {
@@ -107,6 +108,17 @@ public class TimelineClientImpl extends 
     if (!isEnabled) {
       LOG.info("Timeline service is not enabled");
     } else {
+      ClientConfig cc = new DefaultClientConfig();
+      cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+      ConnectionConfigurator connConfigurator = newConnConfigurator(conf);
+      if (UserGroupInformation.isSecurityEnabled()) {
+        TimelineAuthenticator.setStaticConnectionConfigurator(connConfigurator);
+        urlFactory = new KerberosAuthenticatedURLConnectionFactory(connConfigurator);
+        client = new Client(new URLConnectionClientHandler(urlFactory), cc);
+      } else {
+        client = new Client(new URLConnectionClientHandler(
+            new PseudoAuthenticatedURLConnectionFactory(connConfigurator)), cc);
+      }
       if (YarnConfiguration.useHttps(conf)) {
         resURI = URI
             .create(JOINER.join("https://", conf.get(
@@ -182,6 +194,13 @@ public class TimelineClientImpl extends 
   private static class PseudoAuthenticatedURLConnectionFactory
     implements HttpURLConnectionFactory {
 
+    private ConnectionConfigurator connConfigurator;
+
+    public PseudoAuthenticatedURLConnectionFactory(
+        ConnectionConfigurator connConfigurator) {
+      this.connConfigurator = connConfigurator;
+    }
+
     @Override
     public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
       Map<String, String> params = new HashMap<String, String>();
@@ -191,7 +210,7 @@ public class TimelineClientImpl extends 
       if (LOG.isDebugEnabled()) {
         LOG.debug("URL with delegation token: " + url);
       }
-      return (HttpURLConnection) url.openConnection();
+      return connConfigurator.configure((HttpURLConnection) url.openConnection());
     }
 
   }
@@ -202,10 +221,13 @@ public class TimelineClientImpl extends 
     private TimelineAuthenticator authenticator;
     private Token<TimelineDelegationTokenIdentifier> dToken;
     private Text service;
+    private ConnectionConfigurator connConfigurator;
 
-    public KerberosAuthenticatedURLConnectionFactory() {
+    public KerberosAuthenticatedURLConnectionFactory(
+        ConnectionConfigurator connConfigurator) {
       token = new AuthenticatedURL.Token();
       authenticator = new TimelineAuthenticator();
+      this.connConfigurator = connConfigurator;
     }
 
     @Override
@@ -226,7 +248,8 @@ public class TimelineClientImpl extends 
             LOG.debug("URL with delegation token: " + url);
           }
         }
-        return new AuthenticatedURL(authenticator).openConnection(url, token);
+        return new AuthenticatedURL(
+            authenticator, connConfigurator).openConnection(url, token);
       } catch (AuthenticationException e) {
         LOG.error("Authentication failed when openning connection [" + url
             + "] with token [" + token + "].", e);
@@ -255,6 +278,57 @@ public class TimelineClientImpl extends 
 
   }
 
+  private static ConnectionConfigurator newConnConfigurator(Configuration conf) {
+    try {
+      return newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+    } catch (Exception e) {
+      LOG.debug("Cannot load customized ssl related configuration. " +
+          "Fallback to system-generic settings.", e);
+      return DEFAULT_TIMEOUT_CONN_CONFIGURATOR;
+    }
+  }
+
+  private static final ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR =
+      new ConnectionConfigurator() {
+    @Override
+    public HttpURLConnection configure(HttpURLConnection conn)
+        throws IOException {
+      setTimeouts(conn, DEFAULT_SOCKET_TIMEOUT);
+      return conn;
+    }
+  };
+
+  private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+      Configuration conf) throws IOException, GeneralSecurityException {
+    final SSLFactory factory;
+    final SSLSocketFactory sf;
+    final HostnameVerifier hv;
+
+    factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+    factory.init();
+    sf = factory.createSSLSocketFactory();
+    hv = factory.getHostnameVerifier();
+
+    return new ConnectionConfigurator() {
+      @Override
+      public HttpURLConnection configure(HttpURLConnection conn)
+          throws IOException {
+        if (conn instanceof HttpsURLConnection) {
+          HttpsURLConnection c = (HttpsURLConnection) conn;
+          c.setSSLSocketFactory(sf);
+          c.setHostnameVerifier(hv);
+        }
+        setTimeouts(conn, timeout);
+        return conn;
+      }
+    };
+  }
+
+  private static void setTimeouts(URLConnection connection, int socketTimeout) {
+    connection.setConnectTimeout(socketTimeout);
+    connection.setReadTimeout(socketTimeout);
+  }
+
   public static void main(String[] argv) throws Exception {
     CommandLine cliParser = new GnuParser().parse(opts, argv);
     if (cliParser.hasOption("put")) {

Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1619194&r1=1619193&r2=1619194&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
(original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Wed Aug 20 18:19:13 2014
@@ -757,7 +757,10 @@
   </property>
 
   <property>
-    <description>Target size of localizer cache in MB, per local directory.</description>
+    <description>Target size of localizer cache in MB, per nodemanager. It is
+      a target retention size that only includes resources with PUBLIC and 
+      PRIVATE visibility and excludes resources with APPLICATION visibility
+    </description>
     <name>yarn.nodemanager.localizer.cache.target-size-mb</name>
     <value>10240</value>
   </property>



Mime
View raw message