hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-13213 Split out locality metrics among primary and secondary region
Date Tue, 31 Mar 2015 22:16:19 GMT
Repository: hbase
Updated Branches:
  refs/heads/master f1f4b6618 -> 619d34487


HBASE-13213 Split out locality metrics among primary and secondary region


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/619d3448
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/619d3448
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/619d3448

Branch: refs/heads/master
Commit: 619d34487e4426e9405c7c479b89b93869f283dc
Parents: f1f4b66
Author: tedyu <yuzhihong@gmail.com>
Authored: Tue Mar 31 15:16:08 2015 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Tue Mar 31 15:16:08 2015 -0700

----------------------------------------------------------------------
 .../regionserver/MetricsRegionServerSource.java |  3 +++
 .../MetricsRegionServerWrapper.java             |  5 +++++
 .../MetricsRegionServerSourceImpl.java          |  5 ++++-
 .../tmpl/regionserver/ServerMetricsTmpl.jamon   |  2 ++
 .../MetricsRegionServerWrapperImpl.java         | 20 +++++++++++++++++++-
 .../MetricsRegionServerWrapperStub.java         |  5 +++++
 .../regionserver/TestMetricsRegionServer.java   |  1 +
 .../regionserver/TestRegionServerMetrics.java   |  1 +
 8 files changed, 40 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/619d3448/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index b86a8eb..518709e 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -180,6 +180,9 @@ public interface MetricsRegionServerSource extends BaseSource {
   String PERCENT_FILES_LOCAL = "percentFilesLocal";
   String PERCENT_FILES_LOCAL_DESC =
       "The percent of HFiles that are stored on the local hdfs data node.";
+  String PERCENT_FILES_LOCAL_SECONDARY_REGIONS = "percentFilesLocalSecondaryRegions";
+  String PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC =
+    "The percent of HFiles used by secondary regions that are stored on the local hdfs data
node.";
   String SPLIT_QUEUE_LENGTH = "splitQueueLength";
   String SPLIT_QUEUE_LENGTH_DESC = "Length of the queue for splits.";
   String COMPACTION_QUEUE_LENGTH = "compactionQueueLength";

http://git-wip-us.apache.org/repos/asf/hbase/blob/619d3448/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 9634be7..9a66da0 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -154,6 +154,11 @@ public interface MetricsRegionServerWrapper {
   int getPercentFileLocal();
 
   /**
+   * Get the percent of HFiles' that are local for secondary region replicas.
+   */
+  int getPercentFileLocalSecondaryRegions();
+
+  /**
    * Get the size of the split queue
    */
   int getSplitQueueSize();

http://git-wip-us.apache.org/repos/asf/hbase/blob/619d3448/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 4cd73a9..0c9e7dc 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -76,7 +76,7 @@ public class MetricsRegionServerSourceImpl
     slowGet = getMetricsRegistry().newCounter(SLOW_GET_KEY, SLOW_GET_DESC, 0l);
 
     incrementHisto = getMetricsRegistry().newHistogram(INCREMENT_KEY);
-    slowIncrement = getMetricsRegistry().newCounter(SLOW_INCREMENT_KEY, SLOW_INCREMENT_DESC,
0l);
+    slowIncrement = getMetricsRegistry().newCounter(SLOW_INCREMENT_KEY, SLOW_INCREMENT_DESC,
0L);
 
     appendHisto = getMetricsRegistry().newHistogram(APPEND_KEY);
     slowAppend = getMetricsRegistry().newCounter(SLOW_APPEND_KEY, SLOW_APPEND_DESC, 0l);
@@ -212,6 +212,9 @@ public class MetricsRegionServerSourceImpl
               rsWrap.getDataInMemoryWithoutWAL())
           .addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC),
               rsWrap.getPercentFileLocal())
+          .addGauge(Interns.info(PERCENT_FILES_LOCAL_SECONDARY_REGIONS,
+              PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC),
+              rsWrap.getPercentFileLocalSecondaryRegions())
           .addGauge(Interns.info(SPLIT_QUEUE_LENGTH, SPLIT_QUEUE_LENGTH_DESC),
               rsWrap.getSplitQueueSize())
           .addGauge(Interns.info(COMPACTION_QUEUE_LENGTH, COMPACTION_QUEUE_LENGTH_DESC),

http://git-wip-us.apache.org/repos/asf/hbase/blob/619d3448/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
index e4e7e89..a86a4ea 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
@@ -76,12 +76,14 @@ java.lang.management.ManagementFactory;
         <th>Requests Per Second</th>
         <th>Num. Regions</th>
         <th>Block locality</th>
+        <th>Block locality (Secondary replicas)</th>
         <th>Slow WAL Append Count</th>
     </tr>
     <tr>
         <td><% String.format("%.0f", mWrap.getRequestsPerSecond()) %></td>
         <td><% mWrap.getNumOnlineRegions() %></td>
         <td><% mWrap.getPercentFileLocal() %></td>
+        <td><% mWrap.getPercentFileLocalSecondaryRegions() %></td>
         <td><% 0 %></td>
     </tr>
 </table>

http://git-wip-us.apache.org/repos/asf/hbase/blob/619d3448/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 3111661..8f7dee4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -71,6 +72,7 @@ class MetricsRegionServerWrapperImpl
   private volatile long numMutationsWithoutWAL = 0;
   private volatile long dataInMemoryWithoutWAL = 0;
   private volatile int percentFileLocal = 0;
+  private volatile int percentFileLocalSecondaryRegions = 0;
   private volatile long flushedCellsCount = 0;
   private volatile long compactedCellsCount = 0;
   private volatile long majorCompactedCellsCount = 0;
@@ -376,6 +378,11 @@ class MetricsRegionServerWrapperImpl
   }
 
   @Override
+  public int getPercentFileLocalSecondaryRegions() {
+    return percentFileLocalSecondaryRegions;
+  }
+
+  @Override
   public long getUpdatesBlockedTime() {
     if (this.regionServer.cacheFlusher == null) {
       return 0;
@@ -430,6 +437,8 @@ class MetricsRegionServerWrapperImpl
 
       HDFSBlocksDistribution hdfsBlocksDistribution =
           new HDFSBlocksDistribution();
+      HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
+          new HDFSBlocksDistribution();
 
       long tempNumStores = 0;
       long tempNumStoreFiles = 0;
@@ -445,6 +454,7 @@ class MetricsRegionServerWrapperImpl
       long tempNumMutationsWithoutWAL = 0;
       long tempDataInMemoryWithoutWAL = 0;
       int tempPercentFileLocal = 0;
+      int tempPercentFileLocalSecondaryRegions = 0;
       long tempFlushedCellsCount = 0;
       long tempCompactedCellsCount = 0;
       long tempMajorCompactedCellsCount = 0;
@@ -478,13 +488,20 @@ class MetricsRegionServerWrapperImpl
           tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize();
         }
 
-        hdfsBlocksDistribution.add(r.getHDFSBlocksDistribution());
+        HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
+        hdfsBlocksDistribution.add(distro);
+        if (r.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+          hdfsBlocksDistributionSecondaryRegions.add(distro);
+        }
       }
 
       float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(
           regionServer.getServerName().getHostname());
       tempPercentFileLocal = (int) (localityIndex * 100);
 
+      float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions
+          .getBlockLocalityIndex(regionServer.getServerName().getHostname());
+      tempPercentFileLocalSecondaryRegions = (int) (localityIndexSecondaryRegions * 100);
 
       //Compute the number of requests per second
       long currentTime = EnvironmentEdgeManager.currentTime();
@@ -522,6 +539,7 @@ class MetricsRegionServerWrapperImpl
       numMutationsWithoutWAL = tempNumMutationsWithoutWAL;
       dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL;
       percentFileLocal = tempPercentFileLocal;
+      percentFileLocalSecondaryRegions = tempPercentFileLocalSecondaryRegions;
       flushedCellsCount = tempFlushedCellsCount;
       compactedCellsCount = tempCompactedCellsCount;
       majorCompactedCellsCount = tempMajorCompactedCellsCount;

http://git-wip-us.apache.org/repos/asf/hbase/blob/619d3448/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 852d319..35718d8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -131,6 +131,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
   }
 
   @Override
+  public int getPercentFileLocalSecondaryRegions() {
+    return 99;
+  }
+
+  @Override
   public int getCompactionQueueSize() {
     return 411;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/619d3448/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
index e777c1d..ffaae0c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
@@ -76,6 +76,7 @@ public class TestMetricsRegionServer {
     HELPER.assertGauge("mutationsWithoutWALCount", 409, serverSource);
     HELPER.assertGauge("mutationsWithoutWALSize", 410, serverSource);
     HELPER.assertGauge("percentFilesLocal", 99, serverSource);
+    HELPER.assertGauge("percentFilesLocalSecondaryRegions", 99, serverSource);
     HELPER.assertGauge("compactionQueueLength", 411, serverSource);
     HELPER.assertGauge("flushQueueLength", 412, serverSource);
     HELPER.assertGauge("blockCacheFreeSize", 413, serverSource);

http://git-wip-us.apache.org/repos/asf/hbase/blob/619d3448/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index aa071ef..3767513 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -97,6 +97,7 @@ public class TestRegionServerMetrics {
   @Test
   public void testLocalFiles() throws Exception {
     metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
+    metricsHelper.assertGauge("percentFilesLocalSecondaryRegions", 0, serverSource);
   }
 
   @Test


Mime
View raw message