hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject hive git commit: HIVE-12499 : Add HMS metrics for number of tables and partitions (Szehon, reviewed by Jimmy Xiang)
Date Wed, 09 Dec 2015 19:32:58 GMT
Repository: hive
Updated Branches:
  refs/heads/master 695bde3ac -> 546c75c59


HIVE-12499 : Add HMS metrics for number of tables and partitions (Szehon, reviewed by Jimmy
Xiang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/546c75c5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/546c75c5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/546c75c5

Branch: refs/heads/master
Commit: 546c75c598617d79eae66bdb33c92106b486ad24
Parents: 695bde3
Author: Szehon Ho <szehon@cloudera.com>
Authored: Wed Dec 9 11:32:21 2015 -0800
Committer: Szehon Ho <szehon@cloudera.com>
Committed: Wed Dec 9 11:32:21 2015 -0800

----------------------------------------------------------------------
 .../common/metrics/common/MetricsConstant.java  |   8 ++
 .../hive/common/metrics/MetricsTestUtils.java   |   4 +-
 .../hive/metastore/TestMetaStoreMetrics.java    |  83 +++++++++--
 .../hbase/TestHBaseMetastoreMetrics.java        | 136 +++++++++++++++++++
 .../hive/metastore/HMSMetricsListener.java      | 113 +++++++++++++++
 .../hadoop/hive/metastore/HiveMetaStore.java    |  51 +++++++
 .../hadoop/hive/metastore/ObjectStore.java      |  38 +++++-
 .../apache/hadoop/hive/metastore/RawStore.java  |  19 +++
 .../hive/metastore/hbase/HBaseReadWrite.java    |  20 +++
 .../hadoop/hive/metastore/hbase/HBaseStore.java |  30 ++++
 .../DummyRawStoreControlledCommit.java          |  15 ++
 .../DummyRawStoreForJdoConnection.java          |  15 ++
 12 files changed, 520 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
index 95e2bcf..3477187 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
@@ -39,4 +39,12 @@ public class MetricsConstant {
 
   public static final String OPERATION_PREFIX = "hs2_operation_";
   public static final String COMPLETED_OPERATION_PREFIX = "hs2_completed_operation_";
+
+  public static final String INIT_TOTAL_DATABASES = "init_total_count_dbs";
+  public static final String INIT_TOTAL_TABLES = "init_total_count_tables";
+  public static final String INIT_TOTAL_PARTITIONS = "init_total_count_partitions";
+
+  public static final String DELTA_TOTAL_DATABASES = "delta_total_count_dbs";
+  public static final String DELTA_TOTAL_TABLES = "delta_total_count_tables";
+  public static final String DELTA_TOTAL_PARTITIONS = "delta_total_count_partitions";
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
index fd420f7..f21b431 100644
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
@@ -45,9 +45,9 @@ public class MetricsTestUtils {
   }
 
   public static void verifyMetricFile(File jsonReportFile, MetricsCategory category, String
metricsName,
-    Object value) throws Exception {
+    Object expectedValue) throws Exception {
     JsonNode jsonNode = getJsonNode(jsonReportFile, category, metricsName);
-    Assert.assertEquals(jsonNode.asText(), value.toString());
+    Assert.assertEquals(expectedValue.toString(), jsonNode.asText());
   }
 
   private static JsonNode getJsonNode(File jsonReportFile, MetricsCategory category, String
metricsName) throws Exception {

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
index f571c7c..bbfee1d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
@@ -19,26 +19,32 @@ package org.apache.hadoop.hive.metastore;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import junit.framework.TestCase;
 import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
 import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.server.HiveServer2;
-import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.File;
-import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -71,19 +77,18 @@ public class TestMetaStoreMetrics {
     hiveConf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
 
     MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf);
-
     SessionState.start(new CliSessionState(hiveConf));
     driver = new Driver(hiveConf);
   }
 
+
   @Test
-  public void testMetricsFile() throws Exception {
+  public void testMethodCounts() throws Exception {
     driver.run("show databases");
 
     //give timer thread a chance to print the metrics
     Thread.sleep(2000);
 
-    //As the file is being written, try a few times.
     //This can be replaced by CodahaleMetrics's JsonServlet reporter once it is exposed.
     byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath()));
     ObjectMapper objectMapper = new ObjectMapper();
@@ -95,6 +100,66 @@ public class TestMetaStoreMetrics {
     Assert.assertTrue(methodCountNode.asInt() > 0);
   }
 
+  @Test
+  public void testMetaDataCounts() throws Exception {
+    //1 databases created
+    driver.run("create database testdb1");
+
+    //4 tables
+    driver.run("create table testtbl1 (key string)");
+    driver.run("create table testtblpart (key string) partitioned by (partkey string)");
+    driver.run("use testdb1");
+    driver.run("create table testtbl2 (key string)");
+    driver.run("create table testtblpart2 (key string) partitioned by (partkey string)");
+
+    //6 partitions
+    driver.run("alter table default.testtblpart add partition (partkey='a')");
+    driver.run("alter table default.testtblpart add partition (partkey='b')");
+    driver.run("alter table default.testtblpart add partition (partkey='c')");
+    driver.run("alter table testdb1.testtblpart2 add partition (partkey='a')");
+    driver.run("alter table testdb1.testtblpart2 add partition (partkey='b')");
+    driver.run("alter table testdb1.testtblpart2 add partition (partkey='c')");
+
+
+    //create and drop some additional metadata, to test drop counts.
+    driver.run("create database tempdb");
+    driver.run("use tempdb");
+
+    driver.run("create table delete_by_table (key string) partitioned by (partkey string)");
+    driver.run("alter table delete_by_table add partition (partkey='temp')");
+    driver.run("drop table delete_by_table");
+
+    driver.run("create table delete_by_part (key string) partitioned by (partkey string)");
+    driver.run("alter table delete_by_part add partition (partkey='temp')");
+    driver.run("alter table delete_by_part drop partition (partkey='temp')");
+
+    driver.run("create table delete_by_db (key string) partitioned by (partkey string)");
+    driver.run("alter table delete_by_db add partition (partkey='temp')");
+    driver.run("use default");
+    driver.run("drop database tempdb cascade");
+
+    //give timer thread a chance to print the metrics
+    Thread.sleep(2000);
+
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_DATABASES,
1);
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_TABLES,
4);
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_PARTITIONS,
6);
+
+
+    //to test initial metadata count metrics.
+    hiveConf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, ObjectStore.class.getName());
+    HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("test", hiveConf,
false);
+    baseHandler.init();
+    baseHandler.updateMetrics();
+
+    Thread.sleep(2000);
+
+    //1 new db + default
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_DATABASES,
2);
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_TABLES,
4);
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_PARTITIONS,
6);
+  }
+
 
   @Test
   public void testConnections() throws Exception {

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
new file mode 100644
index 0000000..b528376
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.hbase;
+
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Test HMS Metrics on HBase Metastore
+ */
+public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {
+
+  private static File jsonReportFile;
+
+  @BeforeClass
+  public static void startup() throws Exception {
+    File workDir = new File(System.getProperty("test.tmp.dir"));
+    jsonReportFile = new File(workDir, "json_reporting");
+    jsonReportFile.delete();
+
+    HBaseIntegrationTests.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void shutdown() throws Exception {
+    HBaseIntegrationTests.shutdownMiniCluster();
+  }
+
+  @Before
+  public void before() throws IOException {
+    HBaseReadWrite.setConf(conf);
+    conf = new HiveConf();
+    conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
+      "org.apache.hadoop.hive.metastore.hbase.HBaseStore");
+    conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true);
+    conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+
+    conf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true);
+    conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name()
+ "," + MetricsReporting.JMX.name());
+    conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+    conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+    SessionState.start(new CliSessionState(conf));
+    driver = new Driver(conf);
+  }
+
+  @Test
+  public void testMetaDataCounts() throws Exception {
+    //1 databases created
+    driver.run("create database testdb1");
+
+    //4 tables
+    driver.run("create table testtbl1 (key string)");
+    driver.run("create table testtblpart (key string) partitioned by (partkey string)");
+    driver.run("use testdb1");
+    driver.run("create table testtbl2 (key string)");
+    driver.run("create table testtblpart2 (key string) partitioned by (partkey string)");
+
+    //6 partitions
+    driver.run("alter table default.testtblpart add partition (partkey='a')");
+    driver.run("alter table default.testtblpart add partition (partkey='b')");
+    driver.run("alter table default.testtblpart add partition (partkey='c')");
+    driver.run("alter table testdb1.testtblpart2 add partition (partkey='a')");
+    driver.run("alter table testdb1.testtblpart2 add partition (partkey='b')");
+    driver.run("alter table testdb1.testtblpart2 add partition (partkey='c')");
+
+
+    //create and drop some additional metadata, to test drop counts.
+    driver.run("create database tempdb");
+    driver.run("use tempdb");
+
+    driver.run("create table delete_by_table (key string) partitioned by (partkey string)");
+    driver.run("alter table delete_by_table add partition (partkey='temp')");
+    driver.run("drop table delete_by_table");
+
+    driver.run("create table delete_by_part (key string) partitioned by (partkey string)");
+    driver.run("alter table delete_by_part add partition (partkey='temp')");
+    driver.run("alter table delete_by_part drop partition (partkey='temp')");
+
+    driver.run("create table delete_by_db (key string) partitioned by (partkey string)");
+    driver.run("alter table delete_by_db add partition (partkey='temp')");
+    driver.run("use default");
+    driver.run("drop database tempdb cascade");
+
+    //give timer thread a chance to print the metrics
+    Thread.sleep(2000);
+
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_DATABASES,
1);
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_TABLES,
4);
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_PARTITIONS,
6);
+
+
+    //to test initial metadata count metrics.
+    conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, ObjectStore.class.getName());
+    HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("test", conf, false);
+    baseHandler.init();
+    baseHandler.updateMetrics();
+
+    Thread.sleep(2000);
+
+    //1 new db + default
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_DATABASES,
2);
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_TABLES,
4);
+    MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_PARTITIONS,
6);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
new file mode 100644
index 0000000..fa5922f
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.metrics.common.Metrics;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Report metrics of metadata added, deleted by this Hive Metastore.
+ */
+public class HMSMetricsListener extends MetaStoreEventListener {
+
+  public static final Logger LOGGER = LoggerFactory.getLogger(HMSMetricsListener.class);
+  private Metrics metrics;
+
+  public HMSMetricsListener(Configuration config, Metrics metrics) {
+    super(config);
+    this.metrics = metrics;
+  }
+
+  @Override
+  public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException {
+    if (metrics != null) {
+      try {
+        metrics.incrementCounter(MetricsConstant.DELTA_TOTAL_DATABASES);
+      } catch (IOException e) {
+        LOGGER.warn("Error updating metadata metrics", e);
+      }
+    }
+  }
+
+  @Override
+  public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
+    if (metrics != null) {
+      try {
+        metrics.decrementCounter(MetricsConstant.DELTA_TOTAL_DATABASES);
+      } catch (IOException e) {
+        LOGGER.warn("Error updating metadata metrics", e);
+      }
+    }
+  }
+
+  @Override
+  public void onCreateTable(CreateTableEvent tableEvent) throws MetaException {
+    if (metrics != null) {
+      try {
+        metrics.incrementCounter(MetricsConstant.DELTA_TOTAL_TABLES);
+      } catch (IOException e) {
+        LOGGER.warn("Error updating metadata metrics", e);
+      }
+    }
+  }
+
+  @Override
+  public void onDropTable(DropTableEvent tableEvent) throws MetaException {
+    if (metrics != null) {
+      try {
+        metrics.decrementCounter(MetricsConstant.DELTA_TOTAL_TABLES);
+      } catch (IOException e) {
+        LOGGER.warn("Error updating metadata metrics", e);
+      }
+    }
+  }
+
+  @Override
+  public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException {
+    if (metrics != null) {
+      try {
+        metrics.decrementCounter(MetricsConstant.DELTA_TOTAL_PARTITIONS);
+      } catch (IOException e) {
+        LOGGER.warn("Error updating metadata metrics", e);
+      }
+    }
+  }
+
+  @Override
+  public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException {
+    if (metrics != null) {
+      try {
+        metrics.incrementCounter(MetricsConstant.DELTA_TOTAL_PARTITIONS);
+      } catch (IOException e) {
+        LOGGER.warn("Error updating metadata metrics", e);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 00602e1..fec8ea0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -28,6 +28,7 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Multimaps;
 
 import org.apache.commons.cli.OptionBuilder;
+import org.apache.hadoop.hive.common.metrics.common.MetricsVariable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -203,6 +204,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
 
     private static String currentUrl;
 
+    //For Metrics
+    private int initDatabaseCount, initTableCount, initPartCount;
+
     private Warehouse wh; // hdfs warehouse
     private static final ThreadLocal<RawStore> threadLocalMS =
         new ThreadLocal<RawStore>() {
@@ -385,12 +389,43 @@ public class HiveMetaStore extends ThriftHiveMetastore {
         }
       }
 
+      Metrics metrics = MetricsFactory.getInstance();
+      if (metrics != null) {
+        LOG.info("Begin calculating metadata count metrics.");
+        updateMetrics();
+        LOG.info("Finished metadata count metrics: " + initDatabaseCount + " databases, "
+ initTableCount +
+          " tables, " + initPartCount + " partitions.");
+        metrics.addGauge(MetricsConstant.INIT_TOTAL_DATABASES, new MetricsVariable() {
+          @Override
+          public Object getValue() {
+            return initDatabaseCount;
+          }
+        });
+        metrics.addGauge(MetricsConstant.INIT_TOTAL_TABLES, new MetricsVariable() {
+          @Override
+          public Object getValue() {
+            return initTableCount;
+          }
+        });
+        metrics.addGauge(MetricsConstant.INIT_TOTAL_PARTITIONS, new MetricsVariable() {
+          @Override
+          public Object getValue() {
+            return initPartCount;
+          }
+        });
+      }
+
       preListeners = MetaStoreUtils.getMetaStoreListeners(MetaStorePreEventListener.class,
           hiveConf,
           hiveConf.getVar(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS));
       listeners = MetaStoreUtils.getMetaStoreListeners(MetaStoreEventListener.class, hiveConf,
           hiveConf.getVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS));
       listeners.add(new SessionPropertiesListener(hiveConf));
+
+      if (metrics != null) {
+        listeners.add(new HMSMetricsListener(hiveConf, metrics));
+      }
+
       endFunctionListeners = MetaStoreUtils.getMetaStoreListeners(
           MetaStoreEndFunctionListener.class, hiveConf,
           hiveConf.getVar(HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS));
@@ -1633,6 +1668,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
           }
           partNames.add(Warehouse.makePartName(tbl.getPartitionKeys(), part.getValues()));
         }
+        for (MetaStoreEventListener listener : listeners) {
+          //No drop part listener events fired for public listeners historically, for drop
table case.
+          //Limiting to internal listeners for now, to avoid unexpected calls for public
listeners.
+          if (listener instanceof HMSMetricsListener) {
+            for (Partition part : partsToDelete) {
+              listener.onDropPartition(null);
+            }
+          }
+        }
         ms.dropPartitions(dbName, tableName, partNames);
       }
 
@@ -5730,6 +5774,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
       getMS().putFileMetadata(req.getFileIds(), null);
       return new ClearFileMetadataResult();
     }
+
+    @VisibleForTesting
+    public void updateMetrics() throws MetaException {
+      initTableCount = getMS().getTableCount();
+      initPartCount = getMS().getPartitionCount();
+      initDatabaseCount = getMS().getDatabaseCount();
+    }
   }
 
 

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 1c0ab6d..abfe2b8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -37,8 +37,11 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.regex.Pattern;
@@ -213,7 +216,6 @@ public class ObjectStore implements RawStore, Configurable {
   private volatile int openTrasactionCalls = 0;
   private Transaction currentTransaction = null;
   private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE;
-
   private Pattern partitionValidationPattern;
 
   /**
@@ -1049,6 +1051,40 @@ public class ObjectStore implements RawStore, Configurable {
     return tbls;
   }
 
+  public int getDatabaseCount() throws MetaException {
+    return getObjectCount("name", MDatabase.class.getName());
+  }
+
+  public int getPartitionCount() throws MetaException {
+    return getObjectCount("partitionName", MPartition.class.getName());
+  }
+
+  public int getTableCount() throws MetaException {
+    return getObjectCount("tableName", MTable.class.getName());
+  }
+
+  private int getObjectCount(String fieldName, String objName) {
+    Long result = 0L;
+    boolean commited = false;
+    Query query = null;
+    try {
+      openTransaction();
+      String queryStr =
+        "select count(" + fieldName + ") from " + objName;
+      query = pm.newQuery(queryStr);
+      result = (Long) query.execute();
+      commited = commitTransaction();
+    } finally {
+      if (!commited) {
+        rollbackTransaction();
+      }
+      if (query != null) {
+        query.closeAll();
+      }
+    }
+    return result.intValue();
+  }
+
   @Override
   public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String>
tableTypes)
       throws MetaException {

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
index 5b36b03..e118a3b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
 import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
 import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
@@ -633,4 +634,22 @@ public interface RawStore extends Configurable {
   void getFileMetadataByExpr(List<Long> fileIds, FileMetadataExprType type, byte[]
expr,
       ByteBuffer[] metadatas, ByteBuffer[] exprResults, boolean[] eliminated)
           throws MetaException;
+
+  /**
+   * Gets total number of tables.
+   */
+  @InterfaceStability.Evolving
+  int getTableCount() throws MetaException;
+
+  /**
+   * Gets total number of partitions.
+   */
+  @InterfaceStability.Evolving
+  int getPartitionCount() throws MetaException;
+
+  /**
+   * Gets total number of databases.
+   */
+  @InterfaceStability.Evolving
+  int getDatabaseCount() throws MetaException;
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
index 2fb3e8f..287394e 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
@@ -19,8 +19,10 @@
 package org.apache.hadoop.hive.metastore.hbase;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterators;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -421,6 +423,12 @@ public class HBaseReadWrite {
     }
   }
 
+  int getDatabaseCount() throws IOException {
+    Filter fil = new FirstKeyOnlyFilter();
+    Iterator<Result> iter = scan(DB_TABLE, fil);
+    return Iterators.size(iter);
+  }
+
   /**********************************************************************************************
    * Function related methods
    *********************************************************************************************/
@@ -865,6 +873,12 @@ public class HBaseReadWrite {
     return lines;
   }
 
+  int getPartitionCount() throws IOException {
+    Filter fil = new FirstKeyOnlyFilter();
+    Iterator<Result> iter = scan(PART_TABLE, fil);
+    return Iterators.size(iter);
+  }
+
   private String printOnePartition(Result result) throws IOException, TException {
     byte[] key = result.getRow();
     HBaseUtils.StorageDescriptorParts sdParts =
@@ -1671,6 +1685,12 @@ public class HBaseReadWrite {
     return lines;
   }
 
+  int getTableCount() throws IOException {
+    Filter fil = new FirstKeyOnlyFilter();
+    Iterator<Result> iter = scan(TABLE_TABLE, fil);
+    return Iterators.size(iter);
+  }
+
   private String printOneTable(Result result) throws IOException, TException {
     byte[] key = result.getRow();
     HBaseUtils.StorageDescriptorParts sdParts =

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
index 98e6c75..b9509ab 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
@@ -240,6 +240,16 @@ public class HBaseStore implements RawStore {
   }
 
   @Override
+  public int getDatabaseCount() throws MetaException {
+    try {
+      return getHBase().getDatabaseCount();
+    } catch (IOException e) {
+      LOG.error("Unable to get database count", e);
+      throw new MetaException("Error scanning databases");
+    }
+  }
+
+  @Override
   public boolean createType(Type type) {
     throw new UnsupportedOperationException();
   }
@@ -561,6 +571,16 @@ public class HBaseStore implements RawStore {
   }
 
   @Override
+  public int getTableCount() throws MetaException {
+    try {
+      return getHBase().getTableCount();
+    } catch (IOException e) {
+      LOG.error("Unable to get table count", e);
+      throw new MetaException("Error scanning tables");
+    }
+  }
+
+  @Override
   public List<String> listTableNamesByFilter(String dbName, String filter, short max_tables)
throws
       MetaException, UnknownDBException {
     // TODO needs to wait until we support pushing filters into HBase.
@@ -883,6 +903,16 @@ public class HBaseStore implements RawStore {
    */
 
   @Override
+  public int getPartitionCount() throws MetaException {
+    try {
+      return getHBase().getPartitionCount();
+    } catch (IOException e) {
+      LOG.error("Unable to get partition count", e);
+      throw new MetaException("Error scanning partitions");
+    }
+  }
+
+  @Override
   public boolean addRole(String roleName, String ownerName) throws InvalidObjectException,
       MetaException, NoSuchObjectException {
     int now = (int)(System.currentTimeMillis()/1000);

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index 9a1d159..c1156b3 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -788,4 +788,19 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable
{
   public void getFileMetadataByExpr(List<Long> fileIds, FileMetadataExprType type,
byte[] expr,
       ByteBuffer[] metadatas, ByteBuffer[] stripeBitsets, boolean[] eliminated) {
   }
+
+  @Override
+  public int getTableCount() throws MetaException {
+    return objectStore.getTableCount();
+  }
+
+  @Override
+  public int getPartitionCount() throws MetaException {
+    return objectStore.getPartitionCount();
+  }
+
+  @Override
+  public int getDatabaseCount() throws MetaException {
+    return objectStore.getDatabaseCount();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 8dde0af..bf20e99 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -804,6 +804,21 @@ public class DummyRawStoreForJdoConnection implements RawStore {
   public void getFileMetadataByExpr(List<Long> fileIds, FileMetadataExprType type,
byte[] expr,
       ByteBuffer[] metadatas, ByteBuffer[] stripeBitsets, boolean[] eliminated) {
   }
+
+  @Override
+  public int getTableCount() throws MetaException {
+    return 0;
+  }
+
+  @Override
+  public int getPartitionCount() throws MetaException {
+    return 0;
+  }
+
+  @Override
+  public int getDatabaseCount() throws MetaException {
+    return 0;
+  }
 }
 
 


Mime
View raw message