hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject hive git commit: HIVE-13480 : Add hadoop2 metrics reporter for Codahale metrics (Sushanth Sowmyan, reviewed by Thejas Nair)
Date Fri, 22 Apr 2016 20:57:40 GMT
Repository: hive
Updated Branches:
  refs/heads/master fb230f9df -> 0ef92b2c7


HIVE-13480 : Add hadoop2 metrics reporter for Codahale metrics (Sushanth Sowmyan, reviewed
by Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0ef92b2c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0ef92b2c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0ef92b2c

Branch: refs/heads/master
Commit: 0ef92b2c751f644c880e9e6b02cc81e1e6001504
Parents: fb230f9
Author: Sushanth Sowmyan <khorgath@gmail.com>
Authored: Fri Apr 22 13:57:05 2016 -0700
Committer: Sushanth Sowmyan <khorgath@gmail.com>
Committed: Fri Apr 22 13:57:05 2016 -0700

----------------------------------------------------------------------
 common/pom.xml                                      |  5 +++++
 .../common/metrics/metrics2/CodahaleMetrics.java    | 16 ++++++++++++++++
 .../common/metrics/metrics2/MetricsReporting.java   |  3 ++-
 .../java/org/apache/hadoop/hive/conf/HiveConf.java  | 12 +++++++++++-
 pom.xml                                             |  1 +
 5 files changed, 35 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 1b9b4bc..67aab7c 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -188,6 +188,11 @@
       <artifactId>jackson-databind</artifactId>
       <version>${jackson.new.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.github.joshelser</groupId>
+      <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
+      <version>${dropwizard-metrics-hadoop-metrics2-reporter.version}</version>
+    </dependency>
   </dependencies>
 
   <profiles>

http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
index 7dea8de..4c43367 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
@@ -34,6 +34,7 @@ import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
 import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
 import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.joshelser.dropwizard.metrics.hadoop.HadoopMetrics2Reporter;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Splitter;
 import com.google.common.cache.CacheBuilder;
@@ -47,6 +48,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.common.metrics.common.MetricsScope;
 import org.apache.hadoop.hive.common.metrics.common.MetricsVariable;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -372,6 +374,20 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
           jsonFileReporter.start();
           reporters.add(jsonFileReporter);
           break;
+        case HADOOP2:
+          String applicationName = conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_COMPONENT_NAME.varname);
+          long reportingInterval = HiveConf.toTime(
+              conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_INTERVAL.varname),
+              TimeUnit.SECONDS, TimeUnit.SECONDS);
+          final HadoopMetrics2Reporter metrics2Reporter = HadoopMetrics2Reporter.forRegistry(metricRegistry)
+              .convertRatesTo(TimeUnit.SECONDS)
+              .convertDurationsTo(TimeUnit.MILLISECONDS)
+              .build(DefaultMetricsSystem.initialize(applicationName), // The application-level
name
+                  applicationName, // Component name
+                  applicationName, // Component description
+                  "General"); // Name for each metric record
+          metrics2Reporter.start(reportingInterval, TimeUnit.SECONDS);
+          break;
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java
b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java
index 643246f..e6d1f1b 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java
@@ -23,5 +23,6 @@ package org.apache.hadoop.hive.common.metrics.metrics2;
 public enum MetricsReporting {
   JMX,
   CONSOLE,
-  JSON_FILE
+  JSON_FILE,
+  HADOOP2
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index c52b9d9..926806b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1961,7 +1961,8 @@ public class HiveConf extends Configuration {
             "org.apache.hadoop.hive.common.metrics.LegacyMetrics"),
         "Hive metrics subsystem implementation class."),
     HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "JSON_FILE, JMX",
-        "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics,
comma separated list of JMX, CONSOLE, JSON_FILE"),
+        "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics,
" +
+        "comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"),
     HIVE_METRICS_JSON_FILE_LOCATION("hive.service.metrics.file.location", "/tmp/report.json",
         "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics
JSON_FILE reporter, the location of local JSON metrics file.  " +
         "This file will get overwritten at every interval."),
@@ -1969,6 +1970,15 @@ public class HiveConf extends Configuration {
         new TimeValidator(TimeUnit.MILLISECONDS),
         "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics
JSON_FILE reporter, " +
         "the frequency of updating JSON metrics file."),
+    HIVE_METRICS_HADOOP2_INTERVAL("hive.service.metrics.hadoop2.frequency", "30s",
+        new TimeValidator(TimeUnit.SECONDS),
+        "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics
HADOOP2 reporter, " +
+        "the frequency of updating the HADOOP2 metrics system."),
+    HIVE_METRICS_HADOOP2_COMPONENT_NAME("hive.service.metrics.hadoop2.component",
+        "hive",
+        "Component name to provide to Hadoop2 Metrics system. Ideally 'hivemetastore' for
the MetaStore " +
+        " and and 'hiveserver2' for HiveServer2."
+        ),
     HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger",
         "The class responsible for logging client side performance metrics. \n" +
         "Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger"),

http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 77cfaeb..5de35c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -128,6 +128,7 @@
     <commons-dbcp.version>1.4</commons-dbcp.version>
     <derby.version>10.10.2.0</derby.version>
     <dropwizard.version>3.1.0</dropwizard.version>
+    <dropwizard-metrics-hadoop-metrics2-reporter.version>0.1.0</dropwizard-metrics-hadoop-metrics2-reporter.version>
     <guava.version>14.0.1</guava.version>
     <groovy.version>2.4.4</groovy.version>
     <hadoop.version>2.6.0</hadoop.version>


Mime
View raw message