Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 7C030200B5A for ; Wed, 20 Jul 2016 10:50:12 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 7A99D160A64; Wed, 20 Jul 2016 08:50:12 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A619C160A86 for ; Wed, 20 Jul 2016 10:50:11 +0200 (CEST) Received: (qmail 45928 invoked by uid 500); 20 Jul 2016 08:50:09 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 45831 invoked by uid 99); 20 Jul 2016 08:50:09 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 20 Jul 2016 08:50:09 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 654B3E08E8; Wed, 20 Jul 2016 08:50:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Wed, 20 Jul 2016 08:50:10 -0000 Message-Id: <80fba7185d094d9db3ef3b3a84fb158e@git.apache.org> In-Reply-To: <2133d044902943d3936feedf11601a91@git.apache.org> References: <2133d044902943d3936feedf11601a91@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/5] hadoop git commit: YARN-5309. Fix SSLFactory truststore reloader thread leak in TimelineClientImpl. Contributed by Weiwei Yang. archived-at: Wed, 20 Jul 2016 08:50:12 -0000 YARN-5309. Fix SSLFactory truststore reloader thread leak in TimelineClientImpl. Contributed by Weiwei Yang. (cherry picked from commit 9ccf935b2cba5ac8a3e8124360d2203931397391) Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e326f4e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e326f4e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e326f4e Branch: refs/heads/branch-2 Commit: 9e326f4eeb4f3ca2c385fc5cde12c8e2017545f3 Parents: 586cd0d Author: Varun Vasudev Authored: Wed Jul 20 12:48:32 2016 +0530 Committer: Varun Vasudev Committed: Wed Jul 20 12:55:38 2016 +0530 ---------------------------------------------------------------------- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 5 +++ .../client/api/impl/TimelineClientImpl.java | 21 +++++---- .../client/api/impl/TestTimelineClient.java | 47 ++++++++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e326f4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 2877bcb..f6bd613 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -145,6 +145,11 @@ test + org.bouncycastle + bcprov-jdk16 + test + + com.sun.jersey.jersey-test-framework jersey-test-framework-grizzly2 test http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e326f4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 8c60041..1f662dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -111,6 +111,7 @@ public class TimelineClientImpl extends TimelineClient { private Configuration configuration; private float timelineServiceVersion; private TimelineWriter timelineWriter; + private SSLFactory sslFactory; @Private @VisibleForTesting @@ -269,7 +270,7 @@ public class TimelineClientImpl extends TimelineClient { } ClientConfig cc = new DefaultClientConfig(); cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); - connConfigurator = newConnConfigurator(conf); + connConfigurator = initConnConfigurator(conf); if (UserGroupInformation.isSecurityEnabled()) { authenticator = new KerberosDelegationTokenAuthenticator(); } else { @@ -325,6 +326,9 @@ public class TimelineClientImpl extends TimelineClient { if (this.timelineWriter != null) { this.timelineWriter.close(); } + if (this.sslFactory != null) { + this.sslFactory.destroy(); + } super.serviceStop(); } @@ -477,9 +481,9 @@ public class TimelineClientImpl extends TimelineClient { } - private static ConnectionConfigurator newConnConfigurator(Configuration conf) { + private ConnectionConfigurator initConnConfigurator(Configuration conf) { try { - return newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf); + return initSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf); } catch (Exception e) { LOG.debug("Cannot load customized ssl related configuration. " + "Fallback to system-generic settings.", e); @@ -497,16 +501,15 @@ public class TimelineClientImpl extends TimelineClient { } }; - private static ConnectionConfigurator newSslConnConfigurator(final int timeout, + private ConnectionConfigurator initSslConnConfigurator(final int timeout, Configuration conf) throws IOException, GeneralSecurityException { - final SSLFactory factory; final SSLSocketFactory sf; final HostnameVerifier hv; - factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); - factory.init(); - sf = factory.createSSLSocketFactory(); - hv = factory.getHostnameVerifier(); + sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + sslFactory.init(); + sf = sslFactory.createSSLSocketFactory(); + hv = sslFactory.getHostnameVerifier(); return new ConnectionConfigurator() { @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e326f4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 41b788d..a1d4449 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import java.io.File; import java.io.IOException; import java.net.ConnectException; import java.net.SocketTimeoutException; @@ -35,8 +36,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.test.TestGenericTestUtils; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; @@ -434,6 +437,50 @@ public class TestTimelineClient { return client; } + @Test + public void testTimelineClientCleanup() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 0); + + File testDir = TestGenericTestUtils.getTestDir(); + String sslConfDir = + KeyStoreTestUtil.getClasspathDir(TestTimelineClient.class); + KeyStoreTestUtil.setupSSLConfig(testDir.getAbsolutePath(), + sslConfDir, conf, false); + client = createTimelineClient(conf); + + ThreadGroup threadGroup = Thread.currentThread().getThreadGroup(); + + while (threadGroup.getParent() != null) { + threadGroup = threadGroup.getParent(); + } + + Thread[] threads = new Thread[threadGroup.activeCount()]; + + threadGroup.enumerate(threads); + Thread reloaderThread = null; + for (Thread thread : threads) { + if ((thread.getName() != null) + && (thread.getName().contains("Truststore reloader thread"))) { + reloaderThread = thread; + } + } + Assert.assertTrue("Reloader is not alive", reloaderThread.isAlive()); + + client.close(); + + boolean reloaderStillAlive = true; + for (int i = 0; i < 10; i++) { + reloaderStillAlive = reloaderThread.isAlive(); + if (!reloaderStillAlive) { + break; + } + Thread.sleep(1000); + } + Assert.assertFalse("Reloader is still alive", reloaderStillAlive); + } + private static class TestTimlineDelegationTokenSecretManager extends AbstractDelegationTokenSecretManager { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org