hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-18532 Improve cache related stats rendered on RS UI
Date Wed, 23 Aug 2017 20:26:40 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.4 3071dc2fd -> 40dedb8df


HBASE-18532 Improve cache related stats rendered on RS UI

Signed-off-by: tedyu <yuzhihong@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/40dedb8d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/40dedb8d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/40dedb8d

Branch: refs/heads/branch-1.4
Commit: 40dedb8df204b12e602860ea1029fcfb596dd679
Parents: 3071dc2
Author: Biju Nair <gs.biju@gmail.com>
Authored: Wed Aug 23 16:19:31 2017 -0400
Committer: tedyu <yuzhihong@gmail.com>
Committed: Wed Aug 23 13:26:32 2017 -0700

----------------------------------------------------------------------
 .../hbase/io/hfile/MemcachedBlockCache.java     | 10 +++++++
 .../tmpl/regionserver/BlockCacheTmpl.jamon      |  9 +++---
 .../hadoop/hbase/io/hfile/BlockCache.java       | 13 +++++++++
 .../hbase/io/hfile/CombinedBlockCache.java      | 10 +++++++
 .../hadoop/hbase/io/hfile/LruBlockCache.java    | 30 +++++++++++++++++++-
 .../hbase/io/hfile/bucket/BucketCache.java      | 10 +++++++
 .../regionserver/TestHeapMemoryManager.java     | 10 +++++++
 7 files changed, 86 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/40dedb8d/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
b/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
index 54cb8b6..f50a117 100644
--- a/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
+++ b/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
@@ -205,11 +205,21 @@ public class MemcachedBlockCache implements BlockCache {
   }
 
   @Override
+  public long getCurrentDataSize() {
+    return 0;
+  }
+
+  @Override
   public long getBlockCount() {
     return 0;
   }
 
   @Override
+  public long getDataBlockCount() {
+    return 0;
+  }
+
+  @Override
   public Iterator<CachedBlock> iterator() {
     return new Iterator<CachedBlock>() {
       @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/40dedb8d/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
index daa5d76..3d2606b 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
@@ -327,25 +327,25 @@ are combined counts. Request count is sum of hits and misses.</p>
 </%if>
     <tr>
         <td>Count</td>
-        <td><% String.format("%,d", cbsbf.getCount()) %></td>
+        <td><% String.format("%,d", bc.getBlockCount()) %></td>
         <td>Count of Blocks</td>
     </tr>
 <%if !bucketCache %>
     <tr>
         <td>Count</td>
-        <td><% String.format("%,d", cbsbf.getDataCount()) %></td>
+        <td><% String.format("%,d", bc.getDataBlockCount()) %></td>
         <td>Count of DATA Blocks</td>
     </tr>
 </%if>
     <tr>
         <td>Size</td>
-        <td><% TraditionalBinaryPrefix.long2String(cbsbf.getSize(), "B", 1) %></td>
+        <td><% TraditionalBinaryPrefix.long2String(bc.getCurrentSize(), "B", 1)
%></td>
         <td>Size of Blocks</td>
     </tr>
 <%if !bucketCache %>
     <tr>
         <td>Size</td>
-        <td><% TraditionalBinaryPrefix.long2String(cbsbf.getDataSize(), "B", 1)
%></td>
+        <td><% TraditionalBinaryPrefix.long2String(bc.getCurrentDataSize(), "B",
1) %></td>
         <td>Size of DATA Blocks</td>
     </tr>
 </%if> 
@@ -371,4 +371,3 @@ are combined counts. Request count is sum of hits and misses.</p>
 cbsbf = null;
 </%java>
 </%def>
-

http://git-wip-us.apache.org/repos/asf/hbase/blob/40dedb8d/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
index 57c4be9..35cec26 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
@@ -101,6 +101,13 @@ public interface BlockCache extends Iterable<CachedBlock> {
    */
   long getCurrentSize();
 
+ 
+  /**
+   * Returns the occupied size of data blocks, in bytes.
+   * @return occupied space in cache, in bytes
+   */
+  long getCurrentDataSize();
+
   /**
    * Returns the number of blocks currently cached in the block cache.
    * @return number of blocks in the cache
@@ -108,6 +115,12 @@ public interface BlockCache extends Iterable<CachedBlock> {
   long getBlockCount();
 
   /**
+   * Returns the number of data blocks currently cached in the block cache.
+   * @return number of blocks in the cache
+   */
+  long getDataBlockCount();
+
+  /**
    * @return Iterator over the blocks in the cache.
    */
   Iterator<CachedBlock> iterator();

http://git-wip-us.apache.org/repos/asf/hbase/blob/40dedb8d/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
index 4a1c2c7..391dcae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
@@ -122,10 +122,20 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize
{
   }
 
   @Override
+  public long getCurrentDataSize() {
+    return lruCache.getCurrentDataSize() + l2Cache.getCurrentDataSize();
+  }
+
+  @Override
   public long getBlockCount() {
     return lruCache.getBlockCount() + l2Cache.getBlockCount();
   }
 
+  @Override
+  public long getDataBlockCount() {
+    return lruCache.getDataBlockCount() + l2Cache.getDataBlockCount();
+  }
+
   public static class CombinedCacheStats extends CacheStats {
     private final CacheStats lruCacheStats;
     private final CacheStats bucketCacheStats;

http://git-wip-us.apache.org/repos/asf/hbase/blob/40dedb8d/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 334e848..5375e42 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -173,9 +173,15 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
   /** Current size of cache */
   private final AtomicLong size;
 
+  /** Current size of data blocks */
+  private final AtomicLong dataBlockSize;
+
   /** Current number of cached elements */
   private final AtomicLong elements;
 
+  /** Current number of cached data block elements */
+  private final AtomicLong dataBlockElements;
+
   /** Cache access count (sequential ID) */
   private final AtomicLong count;
 
@@ -309,6 +315,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
     this.stats = new CacheStats(this.getClass().getSimpleName());
     this.count = new AtomicLong(0);
     this.elements = new AtomicLong(0);
+    this.dataBlockElements = new AtomicLong(0);
+    this.dataBlockSize = new AtomicLong(0);
     this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
     this.size = new AtomicLong(this.overhead);
     this.hardCapacityLimitFactor = hardLimitFactor;
@@ -394,6 +402,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
     long newSize = updateSizeMetrics(cb, false);
     map.put(cacheKey, cb);
     long val = elements.incrementAndGet();
+    if (buf.getBlockType().isData()) {
+      dataBlockElements.incrementAndGet();
+    }
     if (LOG.isTraceEnabled()) {
       long size = map.size();
       assertCounterSanity(size, val);
@@ -451,9 +462,13 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
    */
   protected long updateSizeMetrics(LruCachedBlock cb, boolean evict) {
     long heapsize = cb.heapSize();
+    BlockType bt = cb.getBuffer().getBlockType();
     if (evict) {
       heapsize *= -1;
     }
+    if (bt != null && bt.isData()) {
+      dataBlockSize.addAndGet(heapsize);
+    }
     return size.addAndGet(heapsize);
   }
 
@@ -553,6 +568,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
       long size = map.size();
       assertCounterSanity(size, val);
     }
+    if (block.getBuffer().getBlockType().isData()) {
+      dataBlockElements.decrementAndGet();
+    }
     stats.evicted(block.getCachedTime(), block.getCacheKey().isPrimary());
     if (evictedByEvictionProcess && victimHandler != null) {
       if (victimHandler instanceof BucketCache) {
@@ -822,6 +840,11 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
   }
 
   @Override
+  public long getCurrentDataSize() {
+    return this.dataBlockSize.get();
+  }
+
+  @Override
   public long getFreeSize() {
     return getMaxSize() - getCurrentSize();
   }
@@ -836,6 +859,11 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
     return this.elements.get();
   }
 
+  @Override
+  public long getDataBlockCount() {
+    return this.dataBlockElements.get();
+  }
+
   EvictionThread getEvictionThread() {
     return this.evictionThread;
   }
@@ -947,7 +975,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
   }
 
   public final static long CACHE_FIXED_OVERHEAD = ClassSize.align(
-      (4 * Bytes.SIZEOF_LONG) + (9 * ClassSize.REFERENCE) +
+      (4 * Bytes.SIZEOF_LONG) + (11 * ClassSize.REFERENCE) +
       (6 * Bytes.SIZEOF_FLOAT) + (2 * Bytes.SIZEOF_BOOLEAN)
       + ClassSize.OBJECT);
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/40dedb8d/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 0c8e62b..c12a826 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -1122,6 +1122,11 @@ public class BucketCache implements BlockCache, HeapSize {
   }
 
   @Override
+  public long getCurrentDataSize() {
+    return size();
+  }
+
+  @Override
   public long getFreeSize() {
     return this.bucketAllocator.getFreeSize();
   }
@@ -1132,6 +1137,11 @@ public class BucketCache implements BlockCache, HeapSize {
   }
 
   @Override
+  public long getDataBlockCount() {
+    return getBlockCount();
+  }
+
+  @Override
   public long getCurrentSize() {
     return this.bucketAllocator.getUsedSize();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/40dedb8d/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
index 0e72d0d..24a995e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
@@ -596,11 +596,21 @@ public class TestHeapMemoryManager {
     }
 
     @Override
+    public long getCurrentDataSize() {
+      return 0;
+    }
+
+    @Override
     public long getBlockCount() {
       return 0;
     }
 
     @Override
+    public long getDataBlockCount() {
+      return 0;
+    }
+
+    @Override
     public void setMaxSize(long size) {
       this.maxSize = size;
     }


Mime
View raw message