hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-18652 Expose individual cache stats in a CombinedCache through JMX
Date Tue, 05 Sep 2017 15:13:29 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 52d4b2575 -> 36371acee


HBASE-18652 Expose individual cache stats in a CombinedCache through JMX

Signed-off-by: tedyu <yuzhihong@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/36371ace
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/36371ace
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/36371ace

Branch: refs/heads/master
Commit: 36371acee79b08dd7b1e069b46cf94bf78763db1
Parents: 52d4b25
Author: Biju Nair <gs.biju@gmail.com>
Authored: Fri Aug 25 17:13:16 2017 -0400
Committer: tedyu <yuzhihong@gmail.com>
Committed: Tue Sep 5 08:13:19 2017 -0700

----------------------------------------------------------------------
 .../regionserver/MetricsRegionServerSource.java | 17 ++++-
 .../MetricsRegionServerWrapper.java             | 40 +++++++++++
 .../MetricsRegionServerSourceImpl.java          | 16 +++++
 .../hadoop/hbase/io/hfile/CacheConfig.java      | 25 +++++--
 .../MetricsRegionServerWrapperImpl.java         | 74 +++++++++++++++++++-
 .../MetricsRegionServerWrapperStub.java         | 40 +++++++++++
 .../regionserver/TestMetricsRegionServer.java   |  8 +++
 7 files changed, 213 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/36371ace/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 9656894..7ed9cbb 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -348,7 +348,22 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String BLOCK_CACHE_GENERAL_BLOOM_META_HIT_COUNT = "blockCacheGeneralBloomMetaHitCount";
   String BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT = "blockCacheDeleteFamilyBloomHitCount";
   String BLOCK_CACHE_TRAILER_HIT_COUNT = "blockCacheTrailerHitCount";
-
+  String L1_CACHE_HIT_COUNT = "l1CacheHitCount";
+  String L1_CACHE_HIT_COUNT_DESC = "L1 cache hit count.";
+  String L1_CACHE_MISS_COUNT = "l1CacheMissCount";
+  String L1_CACHE_MISS_COUNT_DESC = "L1 cache miss count.";
+  String L1_CACHE_HIT_RATIO = "l1CacheHitRatio";
+  String L1_CACHE_HIT_RATIO_DESC = "L1 cache hit ratio.";
+  String L1_CACHE_MISS_RATIO = "l1CacheMissRatio";
+  String L1_CACHE_MISS_RATIO_DESC = "L1 cache miss ratio.";
+  String L2_CACHE_HIT_COUNT = "l2CacheHitCount";
+  String L2_CACHE_HIT_COUNT_DESC = "L2 cache hit count.";
+  String L2_CACHE_MISS_COUNT = "l2CacheMissCount";
+  String L2_CACHE_MISS_COUNT_DESC = "L2 cache miss count.";
+  String L2_CACHE_HIT_RATIO = "l2CacheHitRatio";
+  String L2_CACHE_HIT_RATIO_DESC = "L2 cache hit ratio.";
+  String L2_CACHE_MISS_RATIO = "l2CacheMissRatio";
+  String L2_CACHE_MISS_RATIO_DESC = "L2 cache miss ratio.";
   String RS_START_TIME_NAME = "regionServerStartTime";
   String ZOOKEEPER_QUORUM_NAME = "zookeeperQuorum";
   String SERVER_NAME_NAME = "serverName";

http://git-wip-us.apache.org/repos/asf/hbase/blob/36371ace/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index ccb9de2..86a30d5 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -269,6 +269,46 @@ public interface MetricsRegionServerWrapper {
   long getBlockCacheFailedInsertions();
 
   /**
+   * Hit count of L1 cache.
+   */
+  public long getL1CacheHitCount();
+
+  /**
+   * Miss count of L1 cache.
+   */
+  public long getL1CacheMissCount();
+
+  /**
+   * Hit ratio of L1 cache.
+   */
+  public double getL1CacheHitRatio();
+
+  /**
+   * Miss ratio of L1 cache.
+   */
+  public double getL1CacheMissRatio();
+
+  /**
+   * Hit count of L2 cache.
+   */
+  public long getL2CacheHitCount();
+
+  /**
+   * Miss count of L2 cache.
+   */
+  public long getL2CacheMissCount();
+
+  /**
+   * Hit ratio of L2 cache.
+   */
+  public double getL2CacheHitRatio();
+
+  /**
+   * Miss ratio of L2 cache.
+   */
+  public double getL2CacheMissRatio();
+
+  /**
    * Force a re-computation of the metrics.
    */
   void forceRecompute();

http://git-wip-us.apache.org/repos/asf/hbase/blob/36371ace/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 77643ca..923ec78 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -460,6 +460,22 @@ public class MetricsRegionServerSourceImpl
           .addCounter(Interns.info(BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT, ""),
               rsWrap.getDeleteFamilyBloomHitCount())
           .addCounter(Interns.info(BLOCK_CACHE_TRAILER_HIT_COUNT, ""), rsWrap.getTrailerHitCount())
+          .addGauge(Interns.info(L1_CACHE_HIT_COUNT, L1_CACHE_HIT_COUNT_DESC),
+                  rsWrap.getL1CacheHitCount())
+          .addGauge(Interns.info(L1_CACHE_MISS_COUNT, L1_CACHE_MISS_COUNT_DESC),
+                  rsWrap.getL1CacheMissCount())
+          .addGauge(Interns.info(L1_CACHE_HIT_RATIO, L1_CACHE_HIT_RATIO_DESC),
+                  rsWrap.getL1CacheHitRatio())
+          .addGauge(Interns.info(L1_CACHE_MISS_RATIO, L1_CACHE_MISS_RATIO_DESC),
+                  rsWrap.getL1CacheMissRatio())
+          .addGauge(Interns.info(L2_CACHE_HIT_COUNT, L2_CACHE_HIT_COUNT_DESC),
+                  rsWrap.getL2CacheHitCount())
+          .addGauge(Interns.info(L2_CACHE_MISS_COUNT, L2_CACHE_MISS_COUNT_DESC),
+                  rsWrap.getL2CacheMissCount())
+          .addGauge(Interns.info(L2_CACHE_HIT_RATIO, L2_CACHE_HIT_RATIO_DESC),
+                  rsWrap.getL2CacheHitRatio())
+          .addGauge(Interns.info(L2_CACHE_MISS_RATIO, L2_CACHE_MISS_RATIO_DESC),
+                  rsWrap.getL2CacheMissRatio())
           .addCounter(Interns.info(UPDATES_BLOCKED_TIME, UPDATES_BLOCKED_DESC),
               rsWrap.getUpdatesBlockedTime())
           .addCounter(Interns.info(FLUSHED_CELLS, FLUSHED_CELLS_DESC),

http://git-wip-us.apache.org/repos/asf/hbase/blob/36371ace/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 13f048e..09774a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -547,7 +547,8 @@ public class CacheConfig {
   // Clear this if in tests you'd make more than one block cache instance.
   @VisibleForTesting
   static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
-  private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE;
+  private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE = null;
+  private static BlockCache GLOBAL_L2_CACHE_INSTANCE = null;
 
   /** Boolean whether we have disabled the block cache entirely. */
   @VisibleForTesting
@@ -561,6 +562,20 @@ public class CacheConfig {
     return getL1Internal(c);
   }
 
+  public CacheStats getL1Stats() {
+    if (GLOBAL_L1_CACHE_INSTANCE != null) {
+      return GLOBAL_L1_CACHE_INSTANCE.getStats();
+    }
+    return null;
+  }
+
+  public CacheStats getL2Stats() {
+    if (GLOBAL_L2_CACHE_INSTANCE != null) {
+      return GLOBAL_L2_CACHE_INSTANCE.getStats();
+    }
+    return null;
+  }
+
   /**
    * @param c Configuration to use.
    * @return An L1 instance.  Currently an instance of LruBlockCache.
@@ -593,10 +608,12 @@ public class CacheConfig {
 
     // If we want to use an external block cache then create that.
     if (useExternal) {
-      return getExternalBlockcache(c);
+      GLOBAL_L2_CACHE_INSTANCE = getExternalBlockcache(c);
+    } else {
+      // otherwise use the bucket cache.
+      GLOBAL_L2_CACHE_INSTANCE = getBucketCache(c);
     }
-    // otherwise use the bucket cache.
-    return getBucketCache(c);
+    return GLOBAL_L2_CACHE_INSTANCE;
   }
 
   private static BlockCache getExternalBlockcache(Configuration c) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/36371ace/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index f43db93..85ac097 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -107,6 +107,8 @@ class MetricsRegionServerWrapperImpl
   private volatile long averageRegionSize = 0L;
 
   private CacheStats cacheStats;
+  private CacheStats l1Stats = null;
+  private CacheStats l2Stats = null;
   private ScheduledExecutorService executor;
   private Runnable runnable;
   private long period;
@@ -148,8 +150,12 @@ class MetricsRegionServerWrapperImpl
    */
   private synchronized  void initBlockCache() {
     CacheConfig cacheConfig = this.regionServer.cacheConfig;
-    if (cacheConfig != null && this.blockCache == null) {
-      this.blockCache = cacheConfig.getBlockCache();
+    if (cacheConfig != null) {
+      l1Stats = cacheConfig.getL1Stats();
+      l2Stats = cacheConfig.getL2Stats();
+      if (this.blockCache == null) {
+        this.blockCache = cacheConfig.getBlockCache();
+      }
     }
 
     if (this.blockCache != null && this.cacheStats == null) {
@@ -373,6 +379,70 @@ class MetricsRegionServerWrapperImpl
     return this.cacheStats.getFailedInserts();
   }
 
+  @Override
+  public long getL1CacheHitCount() {
+    if (this.l1Stats == null) {
+      return 0;
+    }
+    return this.l1Stats.getHitCount();
+  }
+
+  @Override
+  public long getL1CacheMissCount() {
+    if (this.l1Stats == null) {
+      return 0;
+    }
+    return this.l1Stats.getMissCount();
+  }
+
+  @Override
+  public double getL1CacheHitRatio() {
+    if (this.l1Stats == null) {
+      return 0;
+    }
+    return this.l1Stats.getHitRatio();
+  }
+
+  @Override
+  public double getL1CacheMissRatio() {
+    if (this.l1Stats == null) {
+      return 0;
+    }
+    return this.l1Stats.getMissRatio();
+  }
+
+  @Override
+  public long getL2CacheHitCount() {
+    if (this.l2Stats == null) {
+      return 0;
+    }
+    return this.l2Stats.getHitCount();
+  }
+
+  @Override
+  public long getL2CacheMissCount() {
+    if (this.l2Stats == null) {
+      return 0;
+    }
+    return this.l2Stats.getMissCount();
+  }
+
+  @Override
+  public double getL2CacheHitRatio() {
+    if (this.l2Stats == null) {
+      return 0;
+    }
+    return this.l2Stats.getHitRatio();
+  }
+
+  @Override
+  public double getL2CacheMissRatio() {
+    if (this.l2Stats == null) {
+      return 0;
+    }
+    return this.l2Stats.getMissRatio();
+  }
+
   @Override public void forceRecompute() {
     this.runnable.run();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/36371ace/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 1c3f6f3..39188f3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -266,6 +266,46 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
   }
 
   @Override
+  public long getL1CacheHitCount() {
+    return 200;
+  }
+
+  @Override
+  public long getL1CacheMissCount() {
+    return 100;
+  }
+
+  @Override
+  public double getL1CacheHitRatio() {
+    return 80;
+  }
+
+  @Override
+  public double getL1CacheMissRatio() {
+    return 20;
+  }
+
+  @Override
+  public long getL2CacheHitCount() {
+    return 800;
+  }
+
+  @Override
+  public long getL2CacheMissCount() {
+    return 200;
+  }
+
+  @Override
+  public double getL2CacheHitRatio() {
+    return 90;
+  }
+
+  @Override
+  public double getL2CacheMissRatio() {
+    return 10;
+  }
+
+  @Override
   public long getUpdatesBlockedTime() {
     return 419;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/36371ace/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
index 7ce7701..c2c420a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
@@ -99,6 +99,14 @@ public class TestMetricsRegionServer {
     HELPER.assertGauge("blockCacheCountHitPercent", 98, serverSource);
     HELPER.assertGauge("blockCacheExpressHitPercent", 97, serverSource);
     HELPER.assertCounter("blockCacheFailedInsertionCount", 36, serverSource);
+    HELPER.assertGauge("l1CacheHitCount", 200, serverSource);
+    HELPER.assertGauge("l1CacheMissCount", 100, serverSource);
+    HELPER.assertGauge("l1CacheHitRatio", 80, serverSource);
+    HELPER.assertGauge("l1CacheMissRatio", 20, serverSource);
+    HELPER.assertGauge("l2CacheHitCount", 800, serverSource);
+    HELPER.assertGauge("l2CacheMissCount", 200, serverSource);
+    HELPER.assertGauge("l2CacheHitRatio", 90, serverSource);
+    HELPER.assertGauge("l2CacheMissRatio", 10, serverSource);
     HELPER.assertCounter("updatesBlockedTime", 419, serverSource);
   }
 


Mime
View raw message