hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wangzh...@apache.org
Subject [hbase] branch master updated: HBASE-25799 add clusterReadRequests and clusterWriteRequests jmx (#3188)
Date Thu, 03 Jun 2021 07:48:32 GMT
This is an automated email from the ASF dual-hosted git repository.

wangzheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
     new 426c3c1  HBASE-25799 add clusterReadRequests and clusterWriteRequests jmx (#3188)
426c3c1 is described below

commit 426c3c16f30c15f9c5cbb198674e11aaa047b60a
Author: xijiawen <15891721997@163.com>
AuthorDate: Thu Jun 3 15:48:03 2021 +0800

    HBASE-25799 add clusterReadRequests and clusterWriteRequests jmx (#3188)
    
    * HBASE-25799 add clusterReadRequests and clusterWriteRequests jmx
    
    Co-authored-by: stevenxi <stevenxi@tencent.com>
---
 .../org/apache/hadoop/hbase/ServerMetrics.java     | 10 ++++++
 .../apache/hadoop/hbase/ServerMetricsBuilder.java  | 38 ++++++++++++++++++++--
 .../hadoop/hbase/master/MetricsMasterSource.java   | 18 ++++++++++
 .../hbase/master/MetricsMasterSourceImpl.java      | 14 ++++++++
 .../src/main/protobuf/server/ClusterStatus.proto   | 10 ++++++
 .../hadoop/hbase/master/MasterRpcServices.java     |  7 +++-
 .../apache/hadoop/hbase/master/MetricsMaster.java  | 14 ++++++++
 .../hadoop/hbase/regionserver/HRegionServer.java   |  2 ++
 .../hadoop/hbase/master/TestMasterMetrics.java     | 14 ++++++--
 .../hbase/master/TestRegionsRecoveryChore.java     | 10 ++++++
 10 files changed, 130 insertions(+), 7 deletions(-)

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetrics.java
index 21fad92..9b2dc40 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetrics.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetrics.java
@@ -58,6 +58,16 @@ public interface ServerMetrics {
   long getRequestCount();
 
   /**
+   * @return total Number of read requests from the start of the region server.
+   */
+  long getReadRequestsCount();
+
+  /**
+   * @return total Number of write requests from the start of the region server.
+   */
+  long getWriteRequestsCount();
+
+  /**
    * @return the amount of used heap
    */
   Size getUsedHeapSize();
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java
index c97e094..05d118f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java
@@ -72,6 +72,8 @@ public final class ServerMetricsBuilder {
       .setRequestCountPerSecond(serverLoadPB.getNumberOfRequests())
       .setRequestCount(serverLoadPB.getTotalNumberOfRequests())
       .setInfoServerPort(serverLoadPB.getInfoServerPort())
+      .setReadRequestCount(serverLoadPB.getReadRequestsCount())
+      .setWriteRequestCount(serverLoadPB.getWriteRequestsCount())
       .setMaxHeapSize(new Size(serverLoadPB.getMaxHeapMB(), Size.Unit.MEGABYTE))
       .setUsedHeapSize(new Size(serverLoadPB.getUsedHeapMB(), Size.Unit.MEGABYTE))
       .setCoprocessorNames(serverLoadPB.getCoprocessorsList().stream()
@@ -128,6 +130,8 @@ public final class ServerMetricsBuilder {
   private String version = "0.0.0";
   private long requestCountPerSecond;
   private long requestCount;
+  private long readRequestCount;
+  private long writeRequestCount;
   private Size usedHeapSize = Size.ZERO;
   private Size maxHeapSize = Size.ZERO;
   private int infoServerPort;
@@ -163,6 +167,17 @@ public final class ServerMetricsBuilder {
     return this;
   }
 
+  public ServerMetricsBuilder setReadRequestCount(long value) {
+    this.readRequestCount = value;
+    return this;
+  }
+
+  public ServerMetricsBuilder setWriteRequestCount(long value) {
+    this.writeRequestCount = value;
+    return this;
+  }
+
+
   public ServerMetricsBuilder setUsedHeapSize(Size value) {
     this.usedHeapSize = value;
     return this;
@@ -220,6 +235,8 @@ public final class ServerMetricsBuilder {
         version,
         requestCountPerSecond,
         requestCount,
+        readRequestCount,
+        writeRequestCount,
         usedHeapSize,
         maxHeapSize,
         infoServerPort,
@@ -238,6 +255,8 @@ public final class ServerMetricsBuilder {
     private final String version;
     private final long requestCountPerSecond;
     private final long requestCount;
+    private final long readRequestsCount;
+    private final long writeRequestsCount;
     private final Size usedHeapSize;
     private final Size maxHeapSize;
     private final int infoServerPort;
@@ -251,15 +270,18 @@ public final class ServerMetricsBuilder {
     private final Map<byte[], UserMetrics> userMetrics;
 
     ServerMetricsImpl(ServerName serverName, int versionNumber, String version,
-        long requestCountPerSecond, long requestCount, Size usedHeapSize, Size maxHeapSize,
+        long requestCountPerSecond, long requestCount, long readRequestsCount,
+        long writeRequestsCount, Size usedHeapSize, Size maxHeapSize,
         int infoServerPort, List<ReplicationLoadSource> sources, ReplicationLoadSink
sink,
-        Map<byte[], RegionMetrics> regionStatus, Set<String> coprocessorNames,
long reportTimestamp,
-        long lastReportTimestamp, Map<byte[], UserMetrics> userMetrics) {
+        Map<byte[], RegionMetrics> regionStatus, Set<String> coprocessorNames,
+        long reportTimestamp, long lastReportTimestamp, Map<byte[], UserMetrics> userMetrics)
{
       this.serverName = Preconditions.checkNotNull(serverName);
       this.versionNumber = versionNumber;
       this.version = version;
       this.requestCountPerSecond = requestCountPerSecond;
       this.requestCount = requestCount;
+      this.readRequestsCount = readRequestsCount;
+      this.writeRequestsCount = writeRequestsCount;
       this.usedHeapSize = Preconditions.checkNotNull(usedHeapSize);
       this.maxHeapSize = Preconditions.checkNotNull(maxHeapSize);
       this.infoServerPort = infoServerPort;
@@ -297,6 +319,16 @@ public final class ServerMetricsBuilder {
     }
 
     @Override
+    public long getReadRequestsCount() {
+      return readRequestsCount;
+    }
+
+    @Override
+    public long getWriteRequestsCount() {
+      return writeRequestsCount;
+    }
+
+    @Override
     public Size getUsedHeapSize() {
       return usedHeapSize;
     }
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java
index 71eb1b2..5f275e8 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java
@@ -69,6 +69,8 @@ public interface MetricsMasterSource extends BaseSource {
   String OFFLINE_REGION_COUNT_NAME = "offlineRegionCount";
 
   String CLUSTER_REQUESTS_NAME = "clusterRequests";
+  String CLUSTER_READ_REQUESTS_NAME = "clusterReadRequests";
+  String CLUSTER_WRITE_REQUESTS_NAME = "clusterWriteRequests";
   String MASTER_ACTIVE_TIME_DESC = "Master Active Time";
   String MASTER_START_TIME_DESC = "Master Start Time";
   String MASTER_FINISHED_INITIALIZATION_TIME_DESC =
@@ -99,6 +101,22 @@ public interface MetricsMasterSource extends BaseSource {
   void incRequests(final long inc);
 
   /**
+   * Increment the number of read requests the cluster has seen.
+   *
+   * @param inc Ammount to increment the total by.
+   */
+  void incReadRequests(final long inc);
+
+
+  /**
+   * Increment the number of write requests the cluster has seen.
+   *
+   * @param inc Ammount to increment the total by.
+   */
+  void incWriteRequests(final long inc);
+
+
+  /**
    * @return {@link OperationMetrics} containing common metrics for server crash operation
    */
   OperationMetrics getServerCrashMetrics();
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
index dacb7a5..4072d8d 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
@@ -38,6 +38,8 @@ public class MetricsMasterSourceImpl
 
   private final MetricsMasterWrapper masterWrapper;
   private MutableFastCounter clusterRequestsCounter;
+  private MutableFastCounter clusterReadRequestsCounter;
+  private MutableFastCounter clusterWriteRequestsCounter;
 
   private OperationMetrics serverCrashMetrics;
 
@@ -63,6 +65,8 @@ public class MetricsMasterSourceImpl
   public void init() {
     super.init();
     clusterRequestsCounter = metricsRegistry.newCounter(CLUSTER_REQUESTS_NAME, "", 0L);
+    clusterReadRequestsCounter = metricsRegistry.newCounter(CLUSTER_READ_REQUESTS_NAME, "",
0L);
+    clusterWriteRequestsCounter = metricsRegistry.newCounter(CLUSTER_WRITE_REQUESTS_NAME,
"", 0L);
 
     /*
      * NOTE: Please refer to HBASE-9774 and HBASE-14282. Based on these two issues, HBase
is
@@ -78,6 +82,16 @@ public class MetricsMasterSourceImpl
   }
 
   @Override
+  public void incReadRequests(final long inc) {
+    this.clusterReadRequestsCounter.incr(inc);
+  }
+
+  @Override
+  public void incWriteRequests(final long inc) {
+    this.clusterWriteRequestsCounter.incr(inc);
+  }
+
+  @Override
   public void getMetrics(MetricsCollector metricsCollector, boolean all) {
 
     MetricsRecordBuilder metricsRecordBuilder = metricsCollector.addRecord(metricsName);
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
index dc875da..fdfa2bc 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
@@ -285,6 +285,16 @@ message ServerLoad {
    * The metrics for each user on this region server
    */
   repeated UserLoad userLoads = 12;
+
+  /**
+   * The metrics for read requests count on this region server
+   */
+  optional uint64 read_requests_count = 13;
+
+  /**
+   * The metrics for write requests on this region server
+   */
+  optional uint64 write_requests_count = 14;
 }
 
 message LiveServerInfo {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 0a29627..e7bf96d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -599,7 +599,12 @@ public class MasterRpcServices extends RSRpcServices implements
       if (sl != null && master.metricsMaster != null) {
         // Up our metrics.
         master.metricsMaster.incrementRequests(
-          sl.getTotalNumberOfRequests() - (oldLoad != null ? oldLoad.getRequestCount() :
0));
+          sl.getTotalNumberOfRequests() -
+                  (oldLoad != null ? oldLoad.getRequestCount() : 0));
+        master.metricsMaster.incrementReadRequests(sl.getReadRequestsCount() -
+                (oldLoad != null ? oldLoad.getReadRequestsCount() : 0));
+        master.metricsMaster.incrementWriteRequests(sl.getWriteRequestsCount() -
+                (oldLoad != null ? oldLoad.getWriteRequestsCount() : 0));
       }
     } catch (IOException ioe) {
       throw new ServiceException(ioe);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
index 83a6988..53030c7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
@@ -76,6 +76,20 @@ public class MetricsMaster {
   }
 
   /**
+   * @param inc How much to add to read requests.
+   */
+  public void incrementReadRequests(final long inc) {
+    masterSource.incReadRequests(inc);
+  }
+
+  /**
+   * @param inc How much to add to write requests.
+   */
+  public void incrementWriteRequests(final long inc) {
+    masterSource.incWriteRequests(inc);
+  }
+
+  /**
    * Sets the number of space quotas defined.
    *
    * @see MetricsMasterQuotaSource#updateNumSpaceQuotas(long)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 330c714..7783bff 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1402,6 +1402,8 @@ public class HRegionServer extends Thread implements
     serverLoad.setTotalNumberOfRequests(regionServerWrapper.getTotalRequestCount());
     serverLoad.setUsedHeapMB((int)(usedMemory / 1024 / 1024));
     serverLoad.setMaxHeapMB((int) (maxMemory / 1024 / 1024));
+    serverLoad.setReadRequestsCount(this.metricsRegionServerImpl.getReadRequestsCount());
+    serverLoad.setWriteRequestsCount(this.metricsRegionServerImpl.getWriteRequestsCount());
     Set<String> coprocessors = getWAL(null).getCoprocessorHost().getCoprocessors();
     Builder coprocessorBuilder = Coprocessor.newBuilder();
     for (String coprocessor : coprocessors) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
index 69247ef..a1b0a69 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
@@ -149,20 +149,28 @@ public class TestMasterMetrics {
 
     MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
     ClusterStatusProtos.ServerLoad sl = ClusterStatusProtos.ServerLoad.newBuilder()
-      .setTotalNumberOfRequests(expectedRequestNumber).build();
+       .setTotalNumberOfRequests(expectedRequestNumber)
+       .setReadRequestsCount(expectedRequestNumber)
+       .setWriteRequestsCount(expectedRequestNumber).build();
     request.setLoad(sl);
 
     master.getMasterRpcServices().regionServerReport(null, request.build());
     metricsHelper.assertCounter("cluster_requests", expectedRequestNumber, masterSource);
+    metricsHelper.assertCounter("cluster_read_requests", expectedRequestNumber, masterSource);
+    metricsHelper.assertCounter("cluster_write_requests", expectedRequestNumber, masterSource);
 
     expectedRequestNumber = 15000;
 
-    sl = ClusterStatusProtos.ServerLoad.newBuilder().setTotalNumberOfRequests(expectedRequestNumber)
-      .build();
+    sl = ClusterStatusProtos.ServerLoad.newBuilder()
+            .setTotalNumberOfRequests(expectedRequestNumber)
+            .setReadRequestsCount(expectedRequestNumber)
+            .setWriteRequestsCount(expectedRequestNumber).build();
     request.setLoad(sl);
 
     master.getMasterRpcServices().regionServerReport(null, request.build());
     metricsHelper.assertCounter("cluster_requests", expectedRequestNumber, masterSource);
+    metricsHelper.assertCounter("cluster_read_requests", expectedRequestNumber, masterSource);
+    metricsHelper.assertCounter("cluster_write_requests", expectedRequestNumber, masterSource);
   }
 
   @Test
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
index 2208f5a..3d5cfac 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
@@ -316,6 +316,16 @@ public class TestRegionsRecoveryChore {
       }
 
       @Override
+      public long getReadRequestsCount() {
+        return 0;
+      }
+
+      @Override
+      public long getWriteRequestsCount() {
+        return 0;
+      }
+
+      @Override
       public Size getUsedHeapSize() {
         return null;
       }

Mime
View raw message