Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DF5711067E for ; Tue, 10 Dec 2013 18:59:10 +0000 (UTC) Received: (qmail 91049 invoked by uid 500); 10 Dec 2013 18:59:10 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 91019 invoked by uid 500); 10 Dec 2013 18:59:10 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 91012 invoked by uid 99); 10 Dec 2013 18:59:10 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Dec 2013 18:59:10 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Dec 2013 18:59:09 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id C43802388831; Tue, 10 Dec 2013 18:58:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1549935 - in /hbase/trunk/hbase-server/src/main: jamon/org/apache/hadoop/hbase/tmpl/regionserver/ java/org/apache/hadoop/hbase/io/hfile/ java/org/apache/hadoop/hbase/io/hfile/bucket/ java/org/apache/hadoop/hbase/io/hfile/slab/ java/org/apa... Date: Tue, 10 Dec 2013 18:58:48 -0000 To: commits@hbase.apache.org From: ndimiduk@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131210185848.C43802388831@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: ndimiduk Date: Tue Dec 10 18:58:47 2013 New Revision: 1549935 URL: http://svn.apache.org/r1549935 Log: HBASE-10116 SlabCache metrics improvements Modified: hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java Modified: hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon?rev=1549935&r1=1549934&r2=1549935&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon (original) +++ hbase/trunk/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon Tue Dec 10 18:58:47 2013 @@ -31,6 +31,7 @@ org.apache.hadoop.hbase.protobuf.Protobu org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad; org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram; +org.apache.hadoop.hbase.util.DirectMemoryUtils; org.apache.hadoop.util.StringUtils; com.yammer.metrics.stats.Snapshot; java.lang.management.ManagementFactory; @@ -100,6 +101,8 @@ MetricsRegionServerWrapper mWrap; Used Heap Max Heap + Direct Memory Used + Direct Memory Configured Memstore Size @@ -110,6 +113,12 @@ MetricsRegionServerWrapper mWrap; <% StringUtils.humanReadableInt(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) %> + + <% StringUtils.humanReadableInt(DirectMemoryUtils.getDirectMemoryUsage()) %> + + + <% StringUtils.humanReadableInt(DirectMemoryUtils.getDirectMemorySize()) %> + <% StringUtils.humanReadableInt(mWrap.getMemstoreSize()) %> Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java Tue Dec 10 18:58:47 2013 @@ -100,8 +100,8 @@ public interface BlockCache { long getCurrentSize(); /** - * Returns the number of evictions that have occurred. - * @return number of evictions + * Returns the number of blocks that have been evicted. + * @return number of evicted blocks */ long getEvictedCount(); Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java Tue Dec 10 18:58:47 2013 @@ -55,7 +55,6 @@ public class CombinedBlockCache implemen return lruCache.heapSize() + bucketCache.heapSize(); } - @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { boolean isMetaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA; @@ -66,7 +65,6 @@ public class CombinedBlockCache implemen } } - @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { cacheBlock(cacheKey, buf, false); @@ -79,7 +77,6 @@ public class CombinedBlockCache implemen return lruCache.getBlock(cacheKey, caching, repeat); } return bucketCache.getBlock(cacheKey, caching, repeat); - } @Override @@ -102,7 +99,6 @@ public class CombinedBlockCache implemen public void shutdown() { lruCache.shutdown(); bucketCache.shutdown(); - } @Override @@ -209,7 +205,5 @@ public class CombinedBlockCache implemen .getSumRequestCachingCountsPastNPeriods())); return Double.isNaN(ratio) ? 0 : ratio; } - } - } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Tue Dec 10 18:58:47 2013 @@ -603,26 +603,17 @@ public class LruBlockCache implements Bl return this.maxSize; } - /** - * Get the current size of this cache. - * @return current size in bytes - */ + @Override public long getCurrentSize() { return this.size.get(); } - /** - * Get the current size of this cache. - * @return current size in bytes - */ + @Override public long getFreeSize() { return getMaxSize() - getCurrentSize(); } - /** - * Get the size of this cache (number of cached blocks) - * @return number of cached blocks - */ + @Override public long size() { return this.elements.get(); } @@ -639,10 +630,7 @@ public class LruBlockCache implements Bl return this.stats.getEvictionCount(); } - /** - * Get the number of blocks that have been evicted during the lifetime - * of this cache. - */ + @Override public long getEvictedCount() { return this.stats.getEvictedCount(); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java Tue Dec 10 18:58:47 2013 @@ -893,14 +893,11 @@ public class BucketCache implements Bloc return this.bucketAllocator; } + @Override public long heapSize() { return this.heapSize.get(); } - /** - * Returns the total size of the block cache, in bytes. - * @return size of cache, in bytes - */ @Override public long size() { return this.realCacheSize.get(); @@ -916,10 +913,6 @@ public class BucketCache implements Bloc return this.blockNumber.get(); } - /** - * Returns the occupied size of the block cache, in bytes. - * @return occupied space in cache, in bytes - */ @Override public long getCurrentSize() { return this.bucketAllocator.getUsedSize(); @@ -931,10 +924,10 @@ public class BucketCache implements Bloc } /** - * Evicts all blocks for a specific HFile. + * Evicts all blocks for a specific HFile. *

* This is used for evict-on-close to remove all blocks of a specific HFile. - * + * * @return the number of blocks evicted */ @Override Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java Tue Dec 10 18:58:47 2013 @@ -91,8 +91,7 @@ public class SingleSizeCache implements this.timeSinceLastAccess = new AtomicLong(); // This evictionListener is called whenever the cache automatically - // evicts - // something. + // evicts something. RemovalListener listener = new RemovalListener() { @Override @@ -116,8 +115,6 @@ public class SingleSizeCache implements .removalListener(listener) .build() .asMap(); - - } @Override @@ -138,7 +135,6 @@ public class SingleSizeCache implements synchronized (this) { CacheablePair alreadyCached = backingMap.putIfAbsent(blockName, newEntry); - if (alreadyCached != null) { backingStore.free(storedBlock); @@ -193,7 +189,6 @@ public class SingleSizeCache implements doEviction(key, evictedBlock); } return evictedBlock != null; - } private void doEviction(BlockCacheKey key, CacheablePair evictedBlock) { Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java?rev=1549935&r1=1549934&r2=1549935&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java Tue Dec 10 18:58:47 2013 @@ -179,7 +179,7 @@ public class SlabCache implements SlabIt private void addSlab(int blockSize, int numBlocks) { LOG.info("Creating a slab of blockSize " + blockSize + " with " + numBlocks - + " blocks."); + + " blocks, " + StringUtils.humanReadableInt(blockSize * numBlocks) + "bytes."); sizer.put(blockSize, new SingleSizeCache(blockSize, numBlocks, this)); } @@ -229,8 +229,6 @@ public class SlabCache implements SlabIt /** * Get the buffer of the block with the specified name. - * @param caching - * @param key * * @return buffer of specified block name, or null if not in cache */ @@ -301,13 +299,17 @@ public class SlabCache implements SlabIt } public long getFreeSize() { - return 0; // this cache, by default, allocates all its space. + long childFreeSize = 0; + for (SingleSizeCache s : sizer.values()) { + childFreeSize += s.getFreeSize(); + } + return childFreeSize; } @Override public long getBlockCount() { long count = 0; - for (SingleSizeCache cache : backingStore.values()) { + for (SingleSizeCache cache : sizer.values()) { count += cache.getBlockCount(); } return count; Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java?rev=1549935&r1=1549934&r2=1549935&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java Tue Dec 10 18:58:47 2013 @@ -198,7 +198,7 @@ class MetricsRegionServerWrapperImpl if (this.blockCache == null) { return 0; } - return this.blockCache.size(); + return this.blockCache.getBlockCount(); } @Override Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java?rev=1549935&r1=1549934&r2=1549935&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java Tue Dec 10 18:58:47 2013 @@ -26,22 +26,53 @@ import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import com.google.common.base.Preconditions; +import javax.management.JMException; +import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; + +/** + * Utilities for interacting with and monitoring DirectByteBuffer allocations. + */ @InterfaceAudience.Private @InterfaceStability.Evolving public class DirectMemoryUtils { + private static final Log LOG = LogFactory.getLog(DirectMemoryUtils.class); + private static final MBeanServer beanServer; + private static final ObjectName nioDirectPool; + + static { + // initialize singletons. Only maintain a reference to the MBeanServer if + // we're able to consume it -- hence convoluted logic. + ObjectName n = null; + MBeanServer s = null; + try { + n = new ObjectName("java.nio:type=BufferPool,name=direct"); + } catch (MalformedObjectNameException e) { + LOG.warn("Unable to initialize ObjectName for DirectByteBuffer allocations."); + } finally { + nioDirectPool = n; + } + if (nioDirectPool != null) { + s = ManagementFactory.getPlatformMBeanServer(); + } + beanServer = s; + } + /** * @return the setting of -XX:MaxDirectMemorySize as a long. Returns 0 if * -XX:MaxDirectMemorySize is not set. */ - public static long getDirectMemorySize() { - RuntimeMXBean RuntimemxBean = ManagementFactory.getRuntimeMXBean(); - List arguments = RuntimemxBean.getInputArguments(); + RuntimeMXBean runtimemxBean = ManagementFactory.getRuntimeMXBean(); + List arguments = runtimemxBean.getInputArguments(); long multiplier = 1; //for the byte case. for (String s : arguments) { if (s.contains("-XX:MaxDirectMemorySize=")) { @@ -64,12 +95,25 @@ public class DirectMemoryUtils { long retValue = Long.parseLong(memSize); return retValue * multiplier; } - } return 0; } /** + * @return the current amount of direct memory used. + */ + public static long getDirectMemoryUsage() { + if (beanServer == null || nioDirectPool == null) return 0; + try { + Long value = (Long) beanServer.getAttribute(nioDirectPool, "MemoryUsed"); + return value == null ? 0 : value; + } catch (JMException e) { + LOG.debug("Failed to retrieve nio.BufferPool direct MemoryUsed"); + return 0; + } + } + + /** * DirectByteBuffers are garbage collected by using a phantom reference and a * reference queue. Every once a while, the JVM checks the reference queue and * cleans the DirectByteBuffers. However, as this doesn't happen @@ -94,6 +138,5 @@ public class DirectMemoryUtils { Method cleanMethod = cleaner.getClass().getMethod("clean"); cleanMethod.setAccessible(true); cleanMethod.invoke(cleaner); - } }