hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject hbase git commit: HBASE-17017 Remove the current per-region latency histogram metrics
Date Wed, 09 Nov 2016 02:39:11 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 3b629d632 -> 03bc884ea


HBASE-17017 Remove the current per-region latency histogram metrics


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/03bc884e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/03bc884e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/03bc884e

Branch: refs/heads/master
Commit: 03bc884ea085197a651b50ddc21f575560854f1c
Parents: 3b629d6
Author: Enis Soztutar <enis@apache.org>
Authored: Tue Nov 8 15:43:41 2016 -0800
Committer: Enis Soztutar <enis@apache.org>
Committed: Tue Nov 8 18:31:53 2016 -0800

----------------------------------------------------------------------
 .../regionserver/MetricsRegionServerSource.java |  5 +-
 .../hbase/regionserver/MetricsRegionSource.java | 12 -----
 .../regionserver/MetricsRegionSourceImpl.java   | 52 ++++++--------------
 .../hadoop/hbase/regionserver/HRegion.java      | 11 +----
 .../hbase/regionserver/MetricsRegion.java       |  8 ---
 .../hbase/regionserver/RSRpcServices.java       |  1 -
 .../regionserver/TestRegionServerMetrics.java   | 11 ++---
 7 files changed, 25 insertions(+), 75 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/03bc884e/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 1e9b7ef..0e7dd8d 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -321,7 +321,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String BLOCK_CACHE_GENERAL_BLOOM_META_HIT_COUNT = "blockCacheGeneralBloomMetaHitCount";
   String BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT = "blockCacheDeleteFamilyBloomHitCount";
   String BLOCK_CACHE_TRAILER_HIT_COUNT = "blockCacheTrailerHitCount";
-  
+
   String RS_START_TIME_NAME = "regionServerStartTime";
   String ZOOKEEPER_QUORUM_NAME = "zookeeperQuorum";
   String SERVER_NAME_NAME = "serverName";
@@ -340,6 +340,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String MUTATE_KEY = "mutate";
   String APPEND_KEY = "append";
   String REPLAY_KEY = "replay";
+  String SCAN_KEY = "scan";
   String SCAN_SIZE_KEY = "scanSize";
   String SCAN_TIME_KEY = "scanTime";
 
@@ -489,6 +490,6 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String RPC_MUTATE_REQUEST_COUNT_DESC =
       "Number of rpc mutation requests this region server has answered.";
   String AVERAGE_REGION_SIZE = "averageRegionSize";
-  String AVERAGE_REGION_SIZE_DESC = 
+  String AVERAGE_REGION_SIZE_DESC =
       "Average region size over the region server including memstore and storefile sizes.";
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/03bc884e/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
index 12ef07c..decf841 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
@@ -60,24 +60,12 @@ public interface MetricsRegionSource extends Comparable<MetricsRegionSource>
{
   void updateDelete();
 
   /**
-   * Update count and sizes of gets.
-   * @param getSize size in bytes of the resulting key values for a get
-   */
-  void updateGetSize(long getSize);
-
-  /**
    * Update time of gets
    * @param mills time for this get operation.
    */
   void updateGet(long mills);
 
   /**
-   * Update the count and sizes of resultScanner.next()
-   * @param scanSize Size in bytes of the resulting key values for a next()
-   */
-  void updateScanSize(long scanSize);
-
-  /**
    * Update time used of resultScanner.next().
    * */
   void updateScanTime(long mills);

http://git-wip-us.apache.org/repos/asf/hbase/blob/03bc884e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index 24064ad..17710e1 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -24,7 +24,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.metrics.Interns;
-import org.apache.hadoop.metrics2.MetricHistogram;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
 import org.apache.hadoop.metrics2.lib.MutableFastCounter;
@@ -48,21 +47,22 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
   private final String regionNamePrefix;
   private final String regionPutKey;
   private final String regionDeleteKey;
-  private final String regionGetSizeKey;
   private final String regionGetKey;
   private final String regionIncrementKey;
   private final String regionAppendKey;
-  private final String regionScanSizeKey;
-  private final String regionScanTimeKey;
+  private final String regionScanKey;
 
+  /*
+   * Implementation note: Do not put histograms per region. With hundreds of regions in a
server
+   * histograms allocate too many counters. See HBASE-17016.
+   */
   private final MutableFastCounter regionPut;
   private final MutableFastCounter regionDelete;
   private final MutableFastCounter regionIncrement;
   private final MutableFastCounter regionAppend;
-  private final MetricHistogram regionGetSize;
-  private final MetricHistogram regionGet;
-  private final MetricHistogram regionScanSize;
-  private final MetricHistogram regionScanTime;
+  private final MutableFastCounter regionGet;
+  private final MutableFastCounter regionScan;
+
   private final int hashCode;
 
   public MetricsRegionSourceImpl(MetricsRegionWrapper regionWrapper,
@@ -95,17 +95,11 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
     regionAppendKey = regionNamePrefix + MetricsRegionServerSource.APPEND_KEY + suffix;
     regionAppend = registry.getCounter(regionAppendKey, 0L);
 
-    regionGetSizeKey = regionNamePrefix + MetricsRegionServerSource.GET_SIZE_KEY;
-    regionGetSize = registry.newSizeHistogram(regionGetSizeKey);
-
-    regionGetKey = regionNamePrefix + MetricsRegionServerSource.GET_KEY;
-    regionGet = registry.newTimeHistogram(regionGetKey);
-
-    regionScanSizeKey = regionNamePrefix + MetricsRegionServerSource.SCAN_SIZE_KEY;
-    regionScanSize = registry.newSizeHistogram(regionScanSizeKey);
+    regionGetKey = regionNamePrefix + MetricsRegionServerSource.GET_KEY + suffix;
+    regionGet = registry.getCounter(regionGetKey, 0L);
 
-    regionScanTimeKey = regionNamePrefix + MetricsRegionServerSource.SCAN_TIME_KEY;
-    regionScanTime = registry.newTimeHistogram(regionScanTimeKey);
+    regionScanKey = regionNamePrefix + MetricsRegionServerSource.SCAN_KEY + suffix;
+    regionScan = registry.getCounter(regionScanKey, 0L);
 
     hashCode = regionWrapper.getRegionHashCode();
   }
@@ -134,14 +128,8 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
       registry.removeMetric(regionDeleteKey);
       registry.removeMetric(regionIncrementKey);
       registry.removeMetric(regionAppendKey);
-      registry.removeMetric(regionGetSizeKey);
       registry.removeMetric(regionGetKey);
-      registry.removeMetric(regionScanSizeKey);
-      registry.removeMetric(regionScanTimeKey);
-      registry.removeHistogramMetrics(regionGetSizeKey);
-      registry.removeHistogramMetrics(regionGetKey);
-      registry.removeHistogramMetrics(regionScanSizeKey);
-      registry.removeHistogramMetrics(regionScanTimeKey);
+      registry.removeMetric(regionScanKey);
 
       regionWrapper = null;
     }
@@ -158,23 +146,13 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource
{
   }
 
   @Override
-  public void updateGetSize(long getSize) {
-    regionGetSize.add(getSize);
-  }
-
-  @Override
   public void updateGet(long mills) {
-    regionGet.add(mills);
-  }
-
-  @Override
-  public void updateScanSize(long scanSize) {
-    regionScanSize.add(scanSize);
+    regionGet.incr();
   }
 
   @Override
   public void updateScanTime(long mills) {
-    regionScanTime.add(mills);
+    regionScan.incr();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/03bc884e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 3423473..b444a1c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -237,7 +237,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
    */
   protected volatile long lastReplayedOpenRegionSeqId = -1L;
   protected volatile long lastReplayedCompactionSeqId = -1L;
-  
+
   // collects Map(s) of Store to sequence Id when handleFileNotFound() is involved
   protected List<Map> storeSeqIds = new ArrayList<>();
 
@@ -6953,13 +6953,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
 
   void metricsUpdateForGet(List<Cell> results, long before) {
     if (this.metricsRegion != null) {
-      long totalSize = 0L;
-      for (Cell cell : results) {
-        // This should give an estimate of the cell in the result. Why do we need
-        // to know the serialization of how the codec works with it??
-        totalSize += CellUtil.estimatedSerializedSizeOf(cell);
-      }
-      this.metricsRegion.updateGetSize(totalSize);
       this.metricsRegion.updateGet(EnvironmentEdgeManager.currentTime() - before);
     }
   }
@@ -7304,7 +7297,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
         writeEntry = null;
       } finally {
         this.updatesLock.readLock().unlock();
-        // For increment/append, a region scanner for doing a get operation could throw 
+        // For increment/append, a region scanner for doing a get operation could throw
         // FileNotFoundException. So we call dropMemstoreContents() in finally block
         // after releasing read lock
         dropMemstoreContents();

http://git-wip-us.apache.org/repos/asf/hbase/blob/03bc884e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java
index 94be034..0364e91 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java
@@ -49,18 +49,10 @@ public class MetricsRegion {
     source.updateDelete();
   }
 
-  public void updateGetSize(final long getSize) {
-    source.updateGetSize(getSize);
-  }
-
   public void updateGet(final long t) {
     source.updateGet(t);
   }
 
-  public void updateScanSize(final long scanSize) {
-    source.updateScanSize(scanSize);
-  }
-
   public void updateScanTime(final long t) {
     source.updateScanTime(t);
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/03bc884e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 9f13939..b0165f0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -2922,7 +2922,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
               region.updateReadRequestsCount(i);
               long end = EnvironmentEdgeManager.currentTime();
               long responseCellSize = context != null ? context.getResponseCellSize() : 0;
-              region.getMetrics().updateScanSize(responseCellSize);
               region.getMetrics().updateScanTime(end - before);
               if (regionServer.metricsRegionServer != null) {
                 regionServer.metricsRegionServer.updateScanSize(responseCellSize);

http://git-wip-us.apache.org/repos/asf/hbase/blob/03bc884e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index d76df56..e2a9bee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -231,7 +231,7 @@ public class TestRegionServerMetrics {
     assertCounter("readRequestCount", readRequests + 10);
     assertCounter("writeRequestCount", writeRequests + 30);
 
-    assertRegionMetrics("getNumOps", 10);
+    assertRegionMetrics("getCount", 10);
     assertRegionMetrics("mutateCount", 31);
 
     doNGets(10, true);  // true = batch
@@ -254,8 +254,7 @@ public class TestRegionServerMetrics {
     // Do a first put to be sure that the connection is established, meta is there and so
on.
     doNPuts(1, false);
     doNGets(10, false);
-    assertRegionMetrics("getNumOps", 10);
-    assertRegionMetrics("getSizeNumOps", 10);
+    assertRegionMetrics("getCount", 10);
     metricsHelper.assertCounterGt("Get_num_ops", 10, serverSource);
   }
 
@@ -360,7 +359,7 @@ public class TestRegionServerMetrics {
       assertEquals(1, result.size());
     }
     numScanNext += NUM_SCAN_NEXT;
-    assertRegionMetrics("scanSizeNumOps", NUM_SCAN_NEXT);
+    assertRegionMetrics("scanCount", NUM_SCAN_NEXT);
     assertCounter("ScanSize_num_ops", numScanNext);
   }
 
@@ -378,7 +377,7 @@ public class TestRegionServerMetrics {
       assertEquals(1, result.size());
     }
     numScanNext += NUM_SCAN_NEXT;
-    assertRegionMetrics("scanTimeNumOps", NUM_SCAN_NEXT);
+    assertRegionMetrics("scanCount", NUM_SCAN_NEXT);
     assertCounter("ScanTime_num_ops", numScanNext);
   }
 
@@ -396,7 +395,7 @@ public class TestRegionServerMetrics {
       assertEquals(1, result.size());
     }
     numScanNext += NUM_SCAN_NEXT;
-    assertRegionMetrics("scanSizeNumOps", NUM_SCAN_NEXT);
+    assertRegionMetrics("scanCount", NUM_SCAN_NEXT);
     assertCounter("ScanSize_num_ops", numScanNext);
   }
 


Mime
View raw message