hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ndimi...@apache.org
Subject svn commit: r1549935 - in /hbase/trunk/hbase-server/src/main: jamon/org/apache/hadoop/hbase/tmpl/regionserver/ java/org/apache/hadoop/hbase/io/hfile/ java/org/apache/hadoop/hbase/io/hfile/bucket/ java/org/apache/hadoop/hbase/io/hfile/slab/ java/org/apa...
Date Tue, 10 Dec 2013 18:58:48 GMT
Author: ndimiduk
Date: Tue Dec 10 18:58:47 2013
New Revision: 1549935

URL: http://svn.apache.org/r1549935
Log:
HBASE-10116 SlabCache metrics improvements

Modified:
    hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java

Modified: hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon?rev=1549935&r1=1549934&r2=1549935&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
(original)
+++ hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
Tue Dec 10 18:58:47 2013
@@ -31,6 +31,7 @@ org.apache.hadoop.hbase.protobuf.Protobu
 org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
 org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
 org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
+org.apache.hadoop.hbase.util.DirectMemoryUtils;
 org.apache.hadoop.util.StringUtils;
 com.yammer.metrics.stats.Snapshot;
 java.lang.management.ManagementFactory;
@@ -100,6 +101,8 @@ MetricsRegionServerWrapper mWrap;
     <tr>
         <th>Used Heap</th>
         <th>Max Heap</th>
+        <th>Direct Memory Used</th>
+        <th>Direct Memory Configured</th>
         <th>Memstore Size</th>
     </tr>
 </tr>
@@ -110,6 +113,12 @@ MetricsRegionServerWrapper mWrap;
     <td>
         <% StringUtils.humanReadableInt(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax())
%>
     </td>
+    <td>
+        <% StringUtils.humanReadableInt(DirectMemoryUtils.getDirectMemoryUsage()) %>
+    </td>
+    <td>
+        <% StringUtils.humanReadableInt(DirectMemoryUtils.getDirectMemorySize()) %>
+    </td>
     <td><% StringUtils.humanReadableInt(mWrap.getMemstoreSize()) %></td>
 </tr>
 </table>

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
Tue Dec 10 18:58:47 2013
@@ -100,8 +100,8 @@ public interface BlockCache {
   long getCurrentSize();
 
   /**
-   * Returns the number of evictions that have occurred.
-   * @return number of evictions
+   * Returns the number of blocks that have been evicted.
+   * @return number of evicted blocks
    */
   long getEvictedCount();
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
Tue Dec 10 18:58:47 2013
@@ -55,7 +55,6 @@ public class CombinedBlockCache implemen
     return lruCache.heapSize() + bucketCache.heapSize();
   }
 
-
   @Override
   public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
     boolean isMetaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA;
@@ -66,7 +65,6 @@ public class CombinedBlockCache implemen
     }
   }
 
-
   @Override
   public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
     cacheBlock(cacheKey, buf, false);
@@ -79,7 +77,6 @@ public class CombinedBlockCache implemen
       return lruCache.getBlock(cacheKey, caching, repeat);
     }
     return bucketCache.getBlock(cacheKey, caching, repeat);
-
   }
 
   @Override
@@ -102,7 +99,6 @@ public class CombinedBlockCache implemen
   public void shutdown() {
     lruCache.shutdown();
     bucketCache.shutdown();
-    
   }
 
   @Override
@@ -209,7 +205,5 @@ public class CombinedBlockCache implemen
           .getSumRequestCachingCountsPastNPeriods()));
       return Double.isNaN(ratio) ? 0 : ratio;
     }
-
   }
-
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
Tue Dec 10 18:58:47 2013
@@ -603,26 +603,17 @@ public class LruBlockCache implements Bl
     return this.maxSize;
   }
 
-  /**
-   * Get the current size of this cache.
-   * @return current size in bytes
-   */
+  @Override
   public long getCurrentSize() {
     return this.size.get();
   }
 
-  /**
-   * Get the current size of this cache.
-   * @return current size in bytes
-   */
+  @Override
   public long getFreeSize() {
     return getMaxSize() - getCurrentSize();
   }
 
-  /**
-   * Get the size of this cache (number of cached blocks)
-   * @return number of cached blocks
-   */
+  @Override
   public long size() {
     return this.elements.get();
   }
@@ -639,10 +630,7 @@ public class LruBlockCache implements Bl
     return this.stats.getEvictionCount();
   }
 
-  /**
-   * Get the number of blocks that have been evicted during the lifetime
-   * of this cache.
-   */
+  @Override
   public long getEvictedCount() {
     return this.stats.getEvictedCount();
   }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
Tue Dec 10 18:58:47 2013
@@ -893,14 +893,11 @@ public class BucketCache implements Bloc
     return this.bucketAllocator;
   }
 
+  @Override
   public long heapSize() {
     return this.heapSize.get();
   }
 
-  /**
-   * Returns the total size of the block cache, in bytes.
-   * @return size of cache, in bytes
-   */
   @Override
   public long size() {
     return this.realCacheSize.get();
@@ -916,10 +913,6 @@ public class BucketCache implements Bloc
     return this.blockNumber.get();
   }
 
-  /**
-   * Returns the occupied size of the block cache, in bytes.
-   * @return occupied space in cache, in bytes
-   */
   @Override
   public long getCurrentSize() {
     return this.bucketAllocator.getUsedSize();
@@ -931,10 +924,10 @@ public class BucketCache implements Bloc
   }
 
   /**
-   * Evicts all blocks for a specific HFile. 
+   * Evicts all blocks for a specific HFile.
    * <p>
    * This is used for evict-on-close to remove all blocks of a specific HFile.
-   * 
+   *
    * @return the number of blocks evicted
    */
   @Override

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
Tue Dec 10 18:58:47 2013
@@ -91,8 +91,7 @@ public class SingleSizeCache implements 
     this.timeSinceLastAccess = new AtomicLong();
 
     // This evictionListener is called whenever the cache automatically
-    // evicts
-    // something.
+    // evicts something.
     RemovalListener<BlockCacheKey, CacheablePair> listener =
       new RemovalListener<BlockCacheKey, CacheablePair>() {
         @Override
@@ -116,8 +115,6 @@ public class SingleSizeCache implements 
         .removalListener(listener)
         .<BlockCacheKey, CacheablePair>build()
         .asMap();
-
-
   }
 
   @Override
@@ -138,7 +135,6 @@ public class SingleSizeCache implements 
 
     synchronized (this) {
       CacheablePair alreadyCached = backingMap.putIfAbsent(blockName, newEntry);
-    
 
       if (alreadyCached != null) {
         backingStore.free(storedBlock);
@@ -193,7 +189,6 @@ public class SingleSizeCache implements 
       doEviction(key, evictedBlock);
     }
     return evictedBlock != null;
-
   }
 
   private void doEviction(BlockCacheKey key, CacheablePair evictedBlock) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
Tue Dec 10 18:58:47 2013
@@ -179,7 +179,7 @@ public class SlabCache implements SlabIt
 
   private void addSlab(int blockSize, int numBlocks) {
     LOG.info("Creating a slab of blockSize " + blockSize + " with " + numBlocks
-        + " blocks.");
+        + " blocks, " + StringUtils.humanReadableInt(blockSize * numBlocks) + "bytes.");
     sizer.put(blockSize, new SingleSizeCache(blockSize, numBlocks, this));
   }
 
@@ -229,8 +229,6 @@ public class SlabCache implements SlabIt
 
   /**
    * Get the buffer of the block with the specified name.
-   * @param caching
-   * @param key
    *
    * @return buffer of specified block name, or null if not in cache
    */
@@ -301,13 +299,17 @@ public class SlabCache implements SlabIt
   }
 
   public long getFreeSize() {
-    return 0; // this cache, by default, allocates all its space.
+    long childFreeSize = 0;
+    for (SingleSizeCache s : sizer.values()) {
+      childFreeSize += s.getFreeSize();
+    }
+    return childFreeSize;
   }
 
   @Override
   public long getBlockCount() {
     long count = 0;
-    for (SingleSizeCache cache : backingStore.values()) {
+    for (SingleSizeCache cache : sizer.values()) {
       count += cache.getBlockCount();
     }
     return count;

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java?rev=1549935&r1=1549934&r2=1549935&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
Tue Dec 10 18:58:47 2013
@@ -198,7 +198,7 @@ class MetricsRegionServerWrapperImpl
     if (this.blockCache == null) {
       return 0;
     }
-    return this.blockCache.size();
+    return this.blockCache.getBlockCount();
   }
 
   @Override

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java?rev=1549935&r1=1549934&r2=1549935&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java
Tue Dec 10 18:58:47 2013
@@ -26,22 +26,53 @@ import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 import com.google.common.base.Preconditions;
 
+import javax.management.JMException;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+
+/**
+ * Utilities for interacting with and monitoring DirectByteBuffer allocations.
+ */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class DirectMemoryUtils {
+  private static final Log LOG = LogFactory.getLog(DirectMemoryUtils.class);
+  private static final MBeanServer beanServer;
+  private static final ObjectName nioDirectPool;
+
+  static {
+    // initialize singletons. Only maintain a reference to the MBeanServer if
+    // we're able to consume it -- hence convoluted logic.
+    ObjectName n = null;
+    MBeanServer s = null;
+    try {
+      n = new ObjectName("java.nio:type=BufferPool,name=direct");
+    } catch (MalformedObjectNameException e) {
+      LOG.warn("Unable to initialize ObjectName for DirectByteBuffer allocations.");
+    } finally {
+      nioDirectPool = n;
+    }
+    if (nioDirectPool != null) {
+      s = ManagementFactory.getPlatformMBeanServer();
+    }
+    beanServer = s;
+  }
+
   /**
    * @return the setting of -XX:MaxDirectMemorySize as a long. Returns 0 if
    *         -XX:MaxDirectMemorySize is not set.
    */
-
   public static long getDirectMemorySize() {
-    RuntimeMXBean RuntimemxBean = ManagementFactory.getRuntimeMXBean();
-    List<String> arguments = RuntimemxBean.getInputArguments();
+    RuntimeMXBean runtimemxBean = ManagementFactory.getRuntimeMXBean();
+    List<String> arguments = runtimemxBean.getInputArguments();
     long multiplier = 1; //for the byte case.
     for (String s : arguments) {
       if (s.contains("-XX:MaxDirectMemorySize=")) {
@@ -64,12 +95,25 @@ public class DirectMemoryUtils {
         long retValue = Long.parseLong(memSize);
         return retValue * multiplier;
       }
-
     }
     return 0;
   }
 
   /**
+   * @return the current amount of direct memory used.
+   */
+  public static long getDirectMemoryUsage() {
+    if (beanServer == null || nioDirectPool == null) return 0;
+    try {
+      Long value = (Long) beanServer.getAttribute(nioDirectPool, "MemoryUsed");
+      return value == null ? 0 : value;
+    } catch (JMException e) {
+      LOG.debug("Failed to retrieve nio.BufferPool direct MemoryUsed");
+      return 0;
+    }
+  }
+
+  /**
    * DirectByteBuffers are garbage collected by using a phantom reference and a
    * reference queue. Every once a while, the JVM checks the reference queue and
    * cleans the DirectByteBuffers. However, as this doesn't happen
@@ -94,6 +138,5 @@ public class DirectMemoryUtils {
     Method cleanMethod = cleaner.getClass().getMethod("clean");
     cleanMethod.setAccessible(true);
     cleanMethod.invoke(cleaner);
-
   }
 }



Mime
View raw message