hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [2/4] hbase git commit: HBASE-18469 Correct RegionServer metric of totalRequestCount
Date Mon, 14 Aug 2017 15:53:36 GMT
HBASE-18469 Correct RegionServer metric of totalRequestCount


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c9f38bc7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c9f38bc7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c9f38bc7

Branch: refs/heads/branch-1.4
Commit: c9f38bc7f31057a723effd5b59d36760623cffb9
Parents: 2d06c1f7
Author: Yu Li <liyu@apache.org>
Authored: Fri Aug 11 14:13:18 2017 +0800
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Mon Aug 14 08:41:46 2017 -0700

----------------------------------------------------------------------
 .../regionserver/MetricsRegionServerSource.java |  3 ++
 .../MetricsRegionServerWrapper.java             |  2 +
 .../MetricsRegionServerSourceImpl.java          |  2 +
 .../MetricsRegionServerWrapperImpl.java         |  5 +++
 .../hbase/regionserver/RSRpcServices.java       | 15 +++++--
 .../MetricsRegionServerWrapperStub.java         |  5 +++
 .../regionserver/TestMetricsRegionServer.java   |  4 ++
 .../regionserver/TestRegionServerMetrics.java   | 41 ++++++++++++++++++--
 8 files changed, 71 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 06bdee6..3ac678e 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -244,6 +244,9 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String TOTAL_REQUEST_COUNT = "totalRequestCount";
   String TOTAL_REQUEST_COUNT_DESC =
       "Total number of requests this RegionServer has answered.";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT = "totalRowActionRequestCount";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT_DESC =
+      "Total number of region requests this RegionServer has answered, count by row-level
action";
   String READ_REQUEST_COUNT = "readRequestCount";
   String READ_REQUEST_COUNT_DESC =
       "Number of read requests this region server has answered.";

http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 7232063..7d7f66d 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -367,4 +367,6 @@ public interface MetricsRegionServerWrapper {
   long getDeleteFamilyBloomHitCount();
 
   long getTrailerHitCount();
+
+  long getTotalRowActionRequestCount();
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 9b7f80a..cec122b 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -354,6 +354,8 @@ public class MetricsRegionServerSourceImpl
           .addGauge(Interns.info(AVERAGE_REGION_SIZE, AVERAGE_REGION_SIZE_DESC), rsWrap.getAverageRegionSize())
           .addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC),
               rsWrap.getTotalRequestCount())
+          .addCounter(Interns.info(TOTAL_ROW_ACTION_REQUEST_COUNT, TOTAL_ROW_ACTION_REQUEST_COUNT_DESC),
+              rsWrap.getTotalRowActionRequestCount())
           .addCounter(Interns.info(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC),
               rsWrap.getReadRequestsCount())
           .addCounter(Interns.info(WRITE_REQUEST_COUNT, WRITE_REQUEST_COUNT_DESC),

http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 35fd1ae..e805e75 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -174,6 +174,11 @@ class MetricsRegionServerWrapperImpl
   }
 
   @Override
+  public long getTotalRowActionRequestCount() {
+    return regionServer.rpcServices.requestRowActionCount.get();
+  }
+
+  @Override
   public int getSplitQueueSize() {
     if (this.regionServer.compactSplitThread == null) {
       return 0;

http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 8317139..6afc3cc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -237,9 +237,13 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
    */
   static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000;
 
-  // Request counter. (Includes requests that are not serviced by regions.)
+  // Count only once for requests with multiple actions like multi/caching-scan/replayBatch
   final Counter requestCount = new Counter();
 
+  // Request counter. (Excludes requests that are not serviced by regions.)
+  // Count rows for requests with multiple actions like multi/caching-scan/replayBatch
+  final Counter requestRowActionCount = new Counter();
+
   // Request counter for rpc get
   final Counter rpcGetRequestCount = new Counter();
 
@@ -990,7 +994,8 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
           it.remove();
         }
       }
-      requestCount.add(mutations.size());
+      requestCount.increment();
+      requestRowActionCount.add(mutations.size());
       if (!region.getRegionInfo().isMetaTable()) {
         regionServer.cacheFlusher.reclaimMemStoreMemory();
       }
@@ -2193,6 +2198,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
     try {
       checkOpen();
       requestCount.increment();
+      requestRowActionCount.increment();
       rpcGetRequestCount.increment();
       Region region = getRegion(request.getRegion());
 
@@ -2307,10 +2313,11 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
     Boolean processed = null;
 
     this.rpcMultiRequestCount.increment();
+    this.requestCount.increment();
     Map<RegionSpecifier, ClientProtos.RegionLoadStats> regionStats = new HashMap<>(request
       .getRegionActionCount());
     for (RegionAction regionAction : request.getRegionActionList()) {
-      this.requestCount.add(regionAction.getActionCount());
+      this.requestRowActionCount.add(regionAction.getActionCount());
       OperationQuota quota;
       Region region;
       regionActionResultBuilder.clear();
@@ -2435,6 +2442,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
     try {
       checkOpen();
       requestCount.increment();
+      requestRowActionCount.increment();
       rpcMutateRequestCount.increment();
       Region region = getRegion(request.getRegion());
       MutateResponse.Builder builder = MutateResponse.newBuilder();
@@ -2872,6 +2880,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
         }
       }
       region.updateReadRequestsCount(numOfResults);
+      requestRowActionCount.add(numOfResults);
       long end = EnvironmentEdgeManager.currentTime();
       long responseCellSize = context != null ? context.getResponseCellSize() : 0;
       region.getMetrics().updateScanTime(end - before);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 32971a4..b573323 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -101,6 +101,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
   }
 
   @Override
+  public long getTotalRowActionRequestCount() {
+    return getReadRequestsCount() + getWriteRequestsCount();
+  }
+
+  @Override
   public long getReadRequestsCount() {
     return 997;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
index ff83c02..e1ef971 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
@@ -71,6 +71,10 @@ public class TestMetricsRegionServer {
     HELPER.assertGauge("memstoreSize", 1025, serverSource);
     HELPER.assertGauge("storeFileSize", 1900, serverSource);
     HELPER.assertCounter("totalRequestCount", 899, serverSource);
+    HELPER.assertCounter("totalRowActionRequestCount",
+      HELPER.getCounter("readRequestCount", serverSource)
+          + HELPER.getCounter("writeRequestCount", serverSource),
+      serverSource);
     HELPER.assertCounter("readRequestCount", 997, serverSource);
     HELPER.assertCounter("writeRequestCount", 707, serverSource);
     HELPER.assertCounter("checkMutateFailedCount", 401, serverSource);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 19041d0..ba12b9c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -124,6 +124,7 @@ public class TestRegionServerMetrics {
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
     long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
+    long rowActionRequests = metricsHelper.getCounter("totalRowActionRequestCount", serverSource);
     long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
     long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
 
@@ -133,6 +134,7 @@ public class TestRegionServerMetrics {
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
     metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
+    metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 30, serverSource);
     metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
 
@@ -143,6 +145,7 @@ public class TestRegionServerMetrics {
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
     metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
+    metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 40, serverSource);
     metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
 
@@ -170,27 +173,59 @@ public class TestRegionServerMetrics {
     // By default, master doesn't host meta now.
     // Adding some meta related requests
     requests += 1;
+    rowActionRequests += 1;
     readRequests ++;
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
-    metricsHelper.assertCounter("totalRequestCount", requests + 50, serverSource);
+    metricsHelper.assertCounter("totalRequestCount", requests + 41, serverSource);
+    metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 50, serverSource);
     metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
 
     List<Put> puts = new ArrayList<>();
     for (int i=0; i< 30; i++) {
-      puts.add(p);
+      // put multiple lines for latter scan
+      Put put = new Put(Bytes.toBytes("" + i + "row")).addColumn(cfName, qualifier, initValue);
+      puts.add(put);
     }
     table.put(puts);
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
-    metricsHelper.assertCounter("totalRequestCount", requests + 80, serverSource);
+    metricsHelper.assertCounter("totalRequestCount", requests + 42, serverSource);
+    metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 80, serverSource);
     metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
     metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
 
+    doScan(table, 10, false);
+    metricsRegionServer.getRegionServerWrapper().forceRecompute();
+    metricsHelper.assertCounter("totalRequestCount", requests + 52, serverSource);
+    metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 90, serverSource);
+    metricsHelper.assertCounter("readRequestCount", readRequests + 30, serverSource);
+    metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
+
+    doScan(table, 10, true);
+    metricsRegionServer.getRegionServerWrapper().forceRecompute();
+    metricsHelper.assertCounter("totalRequestCount", requests + 53, serverSource);
+    metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 100, serverSource);
+    metricsHelper.assertCounter("readRequestCount", readRequests + 40, serverSource);
+    metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
+
     table.close();
   }
 
+  public void doScan(Table table, int n, boolean caching) throws IOException {
+    Scan scan = new Scan();
+    if (caching) {
+      scan.setCaching(n);
+    } else {
+      scan.setCaching(1);
+    }
+    ResultScanner scanner = table.getScanner(scan);
+    for (int i = 0; i < n; i++) {
+      scanner.next();
+    }
+  }
+
   @Test
   public void testGet() throws Exception {
     String tableNameString = "testGet";


Mime
View raw message