Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 12BE6200D34 for ; Fri, 20 Oct 2017 07:37:42 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 11130160BEC; Fri, 20 Oct 2017 05:37:42 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 307EB1609EE for ; Fri, 20 Oct 2017 07:37:41 +0200 (CEST) Received: (qmail 23556 invoked by uid 500); 20 Oct 2017 05:37:40 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 23547 invoked by uid 99); 20 Oct 2017 05:37:40 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 20 Oct 2017 05:37:40 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A488DDF9FE; Fri, 20 Oct 2017 05:37:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xiao@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HADOOP-14944. Add JvmMetrics to KMS. Date: Fri, 20 Oct 2017 05:37:37 +0000 (UTC) archived-at: Fri, 20 Oct 2017 05:37:42 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2 12c81c67d -> a6370dde9 HADOOP-14944. Add JvmMetrics to KMS. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6370dde Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6370dde Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6370dde Branch: refs/heads/branch-2 Commit: a6370dde90acc944ff466f44143f9e45ad5e7890 Parents: 12c81c6 Author: Xiao Chen Authored: Thu Oct 19 22:36:39 2017 -0700 Committer: Xiao Chen Committed: Thu Oct 19 22:37:35 2017 -0700 ---------------------------------------------------------------------- .../hadoop/metrics2/source/JvmMetrics.java | 16 +++++++++ .../crypto/key/kms/server/KMSConfiguration.java | 9 +++++ .../hadoop/crypto/key/kms/server/KMSWebApp.java | 23 ++++++++++++ .../hadoop/crypto/key/kms/server/TestKMS.java | 38 +++++++++++++++++++- 4 files changed, 85 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6370dde/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java index c6369cd..e3f8754 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java @@ -58,6 +58,11 @@ public class JvmMetrics implements MetricsSource { } return impl; } + + synchronized void shutdown() { + DefaultMetricsSystem.instance().unregisterSource(JvmMetrics.name()); + impl = null; + } } @VisibleForTesting @@ -81,6 +86,7 @@ public class JvmMetrics implements MetricsSource { final ConcurrentHashMap gcInfoCache = new ConcurrentHashMap(); + @VisibleForTesting JvmMetrics(String processName, String sessionId) { this.processName = processName; this.sessionId = sessionId; @@ -104,6 +110,16 @@ public class JvmMetrics implements MetricsSource { return Singleton.INSTANCE.init(processName, sessionId); } + /** + * Shutdown the JvmMetrics singleton. This is not necessary if the JVM itself + * is shutdown, but may be necessary for scenarios where JvmMetrics instance + * needs to be re-created while the JVM is still around. One such scenario + * is unit-testing. + */ + public static void shutdownSingleton() { + Singleton.INSTANCE.shutdown(); + } + @Override public void getMetrics(MetricsCollector collector, boolean all) { MetricsRecordBuilder rb = collector.addRecord(JvmMetrics) http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6370dde/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java index d825b2b..cb89561 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java @@ -63,6 +63,15 @@ public class KMSConfiguration { public static final String KMS_AUDIT_AGGREGATION_WINDOW = CONFIG_PREFIX + "audit.aggregation.window.ms"; + // Process name shown in metrics + public static final String METRICS_PROCESS_NAME_KEY = + CONFIG_PREFIX + "metrics.process.name"; + public static final String METRICS_PROCESS_NAME_DEFAULT = "KMS"; + + // Session id for metrics + public static final String METRICS_SESSION_ID_KEY = + CONFIG_PREFIX + "metrics.session.id"; + // KMS Audit logger classes to use public static final String KMS_AUDIT_LOGGER_KEY = CONFIG_PREFIX + "audit.logger"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6370dde/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java index 5772036..80cb627 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java @@ -28,8 +28,11 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderFactory; import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.VersionInfo; import org.apache.log4j.PropertyConfigurator; import org.slf4j.Logger; @@ -44,6 +47,10 @@ import java.io.IOException; import java.net.URI; import java.net.URL; +import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.METRICS_PROCESS_NAME_DEFAULT; +import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.METRICS_PROCESS_NAME_KEY; +import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.METRICS_SESSION_ID_KEY; + @InterfaceAudience.Private public class KMSWebApp implements ServletContextListener { @@ -78,6 +85,9 @@ public class KMSWebApp implements ServletContextListener { private static Meter decryptEEKCallsMeter; private static Meter generateEEKCallsMeter; private static Meter invalidCallsMeter; + private static String processName; + private static String sessionId; + private static JvmPauseMonitor pauseMonitor; private static KMSAudit kmsAudit; private static KeyProviderCryptoExtension keyProviderCryptoExtension; @@ -149,6 +159,16 @@ public class KMSWebApp implements ServletContextListener { unauthenticatedCallsMeter = metricRegistry.register( UNAUTHENTICATED_CALLS_METER, new Meter()); + processName = + kmsConf.get(METRICS_PROCESS_NAME_KEY, METRICS_PROCESS_NAME_DEFAULT); + sessionId = kmsConf.get(METRICS_SESSION_ID_KEY); + pauseMonitor = new JvmPauseMonitor(); + pauseMonitor.init(kmsConf); + DefaultMetricsSystem.initialize(processName); + final JvmMetrics jm = JvmMetrics.initSingleton(processName, sessionId); + jm.setPauseMonitor(pauseMonitor); + pauseMonitor.start(); + kmsAudit = new KMSAudit(kmsConf); // this is required for the the JMXJsonServlet to work properly. @@ -222,6 +242,9 @@ public class KMSWebApp implements ServletContextListener { } kmsAudit.shutdown(); kmsAcls.stopReloader(); + pauseMonitor.stop(); + JvmMetrics.shutdownSingleton(); + DefaultMetricsSystem.shutdown(); jmxReporter.stop(); jmxReporter.close(); metricRegistry = null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6370dde/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index a6f2bd4..712536f 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -53,7 +53,6 @@ import org.junit.rules.Timeout; import org.slf4j.event.Level; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.slf4j.event.Level; import javax.security.auth.login.AppConfigurationEntry; @@ -62,6 +61,7 @@ import java.io.DataInputStream; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; import java.io.Writer; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -82,6 +82,8 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; +import static org.junit.Assert.assertTrue; + public class TestKMS { private static final Logger LOG = LoggerFactory.getLogger(TestKMS.class); @@ -2380,4 +2382,38 @@ public class TestKMS { }); } + /* + * Test the jmx page can return, and contains the basic JvmMetrics. Only + * testing in simple mode since the page content is the same, kerberized + * or not. + */ + @Test + public void testKMSJMX() throws Exception { + Configuration conf = new Configuration(); + final File confDir = getTestDir(); + conf = createBaseKMSConf(confDir, conf); + final String processName = "testkmsjmx"; + conf.set(KMSConfiguration.METRICS_PROCESS_NAME_KEY, processName); + writeConf(confDir, conf); + + runServer(null, null, confDir, new KMSCallable() { + @Override + public Void call() throws Exception { + final URL jmxUrl = new URL( + getKMSUrl() + "/jmx?user.name=whatever&qry=Hadoop:service=" + + processName + ",name=JvmMetrics"); + LOG.info("Requesting jmx from " + jmxUrl); + final StringBuilder sb = new StringBuilder(); + final InputStream in = jmxUrl.openConnection().getInputStream(); + final byte[] buffer = new byte[64 * 1024]; + int len; + while ((len = in.read(buffer)) > 0) { + sb.append(new String(buffer, 0, len)); + } + LOG.info("jmx returned: " + sb.toString()); + assertTrue(sb.toString().contains("JvmMetrics")); + return null; + } + }); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org