hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-13863 Multi-wal feature breaks reported number and size of HLogs (Abhilash)
Date Fri, 26 Jun 2015 15:49:18 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 a1b696d70 -> 939cc56aa


HBASE-13863 Multi-wal feature breaks reported number and size of HLogs (Abhilash)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/939cc56a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/939cc56a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/939cc56a

Branch: refs/heads/branch-1.1
Commit: 939cc56aaf7b6f63742fbc0f38131fe46399348a
Parents: a1b696d
Author: tedyu <yuzhihong@gmail.com>
Authored: Fri Jun 26 08:49:24 2015 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Fri Jun 26 08:49:24 2015 -0700

----------------------------------------------------------------------
 .../MetricsRegionServerWrapperImpl.java         | 11 ++--
 .../wal/BoundedRegionGroupingProvider.java      | 57 +++++++++++++++++++-
 .../hadoop/hbase/wal/DefaultWALProvider.java    |  2 +-
 3 files changed, 63 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/939cc56a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 1955c2b..5315cef 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.CacheStats;
+import org.apache.hadoop.hbase.wal.BoundedRegionGroupingProvider;
 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -503,14 +504,16 @@ class MetricsRegionServerWrapperImpl
       //If we've time traveled keep the last requests per second.
       if ((currentTime - lastRan) > 0) {
         long currentRequestCount = getTotalRequestCount();
-        requestsPerSecond = (currentRequestCount - lastRequestCount) / ((currentTime - lastRan)
/ 1000.0);
+        requestsPerSecond = (currentRequestCount - lastRequestCount) /
+            ((currentTime - lastRan) / 1000.0);
         lastRequestCount = currentRequestCount;
       }
       lastRan = currentTime;
 
-      numWALFiles = DefaultWALProvider.getNumLogFiles(regionServer.walFactory);
-      walFileSize = DefaultWALProvider.getLogFileSize(regionServer.walFactory);
-
+      numWALFiles = DefaultWALProvider.getNumLogFiles(regionServer.walFactory) +
+          BoundedRegionGroupingProvider.getNumLogFiles(regionServer.walFactory);
+      walFileSize = DefaultWALProvider.getLogFileSize(regionServer.walFactory) +
+          BoundedRegionGroupingProvider.getLogFileSize(regionServer.walFactory);
       //Copy over computed values so that no thread sees half computed values.
       numStores = tempNumStores;
       numStoreFiles = tempNumStoreFiles;

http://git-wip-us.apache.org/repos/asf/hbase/blob/939cc56a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
index 478d5c3..e1417b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
@@ -26,7 +26,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-
+import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
 // imports for classes still in regionserver.wal
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
 
@@ -37,7 +37,7 @@ import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
  * {@link RegionGroupingProvider}.
  */
 @InterfaceAudience.Private
-class BoundedRegionGroupingProvider extends RegionGroupingProvider {
+public class BoundedRegionGroupingProvider extends RegionGroupingProvider {
   private static final Log LOG = LogFactory.getLog(BoundedRegionGroupingProvider.class);
 
   static final String NUM_REGION_GROUPS = "hbase.wal.regiongrouping.numgroups";
@@ -103,4 +103,57 @@ class BoundedRegionGroupingProvider extends RegionGroupingProvider {
       throw failure;
     }
   }
+
+  /**
+   * iff the given WALFactory is using the BoundedRegionGroupingProvider for meta and/or
non-meta,
+   * count the number of files (rolled and active). if either of them isn't, count 0
+   * for that provider.
+   * @param walFactory may not be null.
+   */
+  public static long getNumLogFiles(WALFactory walFactory) {
+    long result = 0;
+    if (walFactory.provider instanceof BoundedRegionGroupingProvider) {
+      BoundedRegionGroupingProvider groupProviders =
+          (BoundedRegionGroupingProvider)walFactory.provider;
+      for (int i = 0; i < groupProviders.delegates.length; i++) {
+        result +=
+            ((FSHLog)((DefaultWALProvider)(groupProviders.delegates[i])).log).getNumLogFiles();
+      }
+    }
+    WALProvider meta = walFactory.metaProvider.get();
+    if (meta instanceof BoundedRegionGroupingProvider) {
+      for (int i = 0; i < ((BoundedRegionGroupingProvider)meta).delegates.length; i++)
{
+        result += ((FSHLog)
+            ((DefaultWALProvider)(((BoundedRegionGroupingProvider)meta).delegates[i])).log)
+            .getNumLogFiles();      }
+    }
+    return result;
+  }
+
+  /**
+   * iff the given WALFactory is using the BoundedRegionGroupingProvider for meta and/or
non-meta,
+   * count the size of files (rolled and active). if either of them isn't, count 0
+   * for that provider.
+   * @param walFactory may not be null.
+   */
+  public static long getLogFileSize(WALFactory walFactory) {
+    long result = 0;
+    if (walFactory.provider instanceof BoundedRegionGroupingProvider) {
+      BoundedRegionGroupingProvider groupProviders =
+          (BoundedRegionGroupingProvider)walFactory.provider;
+      for (int i = 0; i < groupProviders.delegates.length; i++) {
+        result +=
+            ((FSHLog)((DefaultWALProvider)(groupProviders.delegates[i])).log).getLogFileSize();
+      }
+    }
+    WALProvider meta = walFactory.metaProvider.get();
+    if (meta instanceof BoundedRegionGroupingProvider) {
+      for (int i = 0; i < ((BoundedRegionGroupingProvider)meta).delegates.length; i++)
{
+        result += ((FSHLog)
+            ((DefaultWALProvider)(((BoundedRegionGroupingProvider)meta).delegates[i])).log)
+            .getLogFileSize();
+      }
+    }
+    return result;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/939cc56a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
index b710059..8d8f731 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
@@ -75,7 +75,7 @@ public class DefaultWALProvider implements WALProvider {
     void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
   }
 
-  private FSHLog log = null;
+  protected FSHLog log = null;
 
   /**
    * @param factory factory that made us, identity used for FS layout. may not be null


Mime
View raw message