hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vvasu...@apache.org
Subject [3/5] hadoop git commit: YARN-5309. Fix SSLFactory truststore reloader thread leak in TimelineClientImpl. Contributed by Weiwei Yang.
Date Wed, 20 Jul 2016 08:50:11 GMT
YARN-5309. Fix SSLFactory truststore reloader thread leak in TimelineClientImpl. Contributed
by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/478a2bb1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/478a2bb1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/478a2bb1

Branch: refs/heads/branch-2.8
Commit: 478a2bb1854167ad7ac806cff2b99b15a3b796b8
Parents: f2508d7
Author: Varun Vasudev <vvasudev@apache.org>
Authored: Wed Jul 20 13:02:34 2016 +0530
Committer: Varun Vasudev <vvasudev@apache.org>
Committed: Wed Jul 20 13:02:34 2016 +0530

----------------------------------------------------------------------
 .../hadoop-yarn/hadoop-yarn-common/pom.xml      |  5 ++
 .../client/api/impl/TimelineClientImpl.java     | 21 ++++----
 .../client/api/impl/TestTimelineClient.java     | 53 ++++++++++++++++++++
 3 files changed, 70 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/478a2bb1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 3d74ac0..a982ab2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -145,6 +145,11 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcprov-jdk16</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>com.sun.jersey.jersey-test-framework</groupId>
       <artifactId>jersey-test-framework-grizzly2</artifactId>
       <scope>test</scope>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/478a2bb1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 8c60041..1f662dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -111,6 +111,7 @@ public class TimelineClientImpl extends TimelineClient {
   private Configuration configuration;
   private float timelineServiceVersion;
   private TimelineWriter timelineWriter;
+  private SSLFactory sslFactory;
 
   @Private
   @VisibleForTesting
@@ -269,7 +270,7 @@ public class TimelineClientImpl extends TimelineClient {
     }
     ClientConfig cc = new DefaultClientConfig();
     cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
-    connConfigurator = newConnConfigurator(conf);
+    connConfigurator = initConnConfigurator(conf);
     if (UserGroupInformation.isSecurityEnabled()) {
       authenticator = new KerberosDelegationTokenAuthenticator();
     } else {
@@ -325,6 +326,9 @@ public class TimelineClientImpl extends TimelineClient {
     if (this.timelineWriter != null) {
       this.timelineWriter.close();
     }
+    if (this.sslFactory != null) {
+      this.sslFactory.destroy();
+    }
     super.serviceStop();
   }
 
@@ -477,9 +481,9 @@ public class TimelineClientImpl extends TimelineClient {
 
   }
 
-  private static ConnectionConfigurator newConnConfigurator(Configuration conf) {
+  private ConnectionConfigurator initConnConfigurator(Configuration conf) {
     try {
-      return newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+      return initSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
     } catch (Exception e) {
       LOG.debug("Cannot load customized ssl related configuration. " +
           "Fallback to system-generic settings.", e);
@@ -497,16 +501,15 @@ public class TimelineClientImpl extends TimelineClient {
     }
   };
 
-  private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+  private ConnectionConfigurator initSslConnConfigurator(final int timeout,
       Configuration conf) throws IOException, GeneralSecurityException {
-    final SSLFactory factory;
     final SSLSocketFactory sf;
     final HostnameVerifier hv;
 
-    factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
-    factory.init();
-    sf = factory.createSSLSocketFactory();
-    hv = factory.getHostnameVerifier();
+    sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+    sslFactory.init();
+    sf = sslFactory.createSSLSocketFactory();
+    hv = sslFactory.getHostnameVerifier();
 
     return new ConnectionConfigurator() {
       @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/478a2bb1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index 41b788d..ae9fa46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.ConnectException;
 import java.net.SocketTimeoutException;
@@ -33,8 +34,10 @@ import java.security.PrivilegedExceptionAction;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
@@ -58,9 +61,15 @@ public class TestTimelineClient {
 
   private TimelineClientImpl client;
   private TimelineWriter spyTimelineWriter;
+  private static final File testDir = new File(
+      System.getProperty("test.build.data",
+          System.getProperty("java.io.tmpdir")),
+          "TestTimelineClient");
 
   @Before
   public void setup() {
+    FileUtil.fullyDelete(testDir);
+    testDir.mkdirs();
     YarnConfiguration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.0f);
@@ -69,6 +78,7 @@ public class TestTimelineClient {
 
   @After
   public void tearDown() {
+    FileUtil.fullyDelete(testDir);
     if (client != null) {
       client.stop();
     }
@@ -434,6 +444,49 @@ public class TestTimelineClient {
     return client;
   }
 
+  @Test
+  public void testTimelineClientCleanup() throws Exception {
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 0);
+
+    String sslConfDir =
+        KeyStoreTestUtil.getClasspathDir(TestTimelineClient.class);
+    KeyStoreTestUtil.setupSSLConfig(testDir.getAbsolutePath(),
+        sslConfDir, conf, false);
+    client = createTimelineClient(conf);
+
+    ThreadGroup threadGroup = Thread.currentThread().getThreadGroup();
+
+    while (threadGroup.getParent() != null) {
+      threadGroup = threadGroup.getParent();
+    }
+
+    Thread[] threads = new Thread[threadGroup.activeCount()];
+
+    threadGroup.enumerate(threads);
+    Thread reloaderThread = null;
+    for (Thread thread : threads) {
+      if ((thread.getName() != null)
+          && (thread.getName().contains("Truststore reloader thread"))) {
+        reloaderThread = thread;
+      }
+    }
+    Assert.assertTrue("Reloader is not alive", reloaderThread.isAlive());
+
+    client.close();
+
+    boolean reloaderStillAlive = true;
+    for (int i = 0; i < 10; i++) {
+      reloaderStillAlive = reloaderThread.isAlive();
+      if (!reloaderStillAlive) {
+        break;
+      }
+      Thread.sleep(1000);
+    }
+    Assert.assertFalse("Reloader is still alive", reloaderStillAlive);
+  }
+
   private static class TestTimlineDelegationTokenSecretManager extends
       AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier> {
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message