hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject git commit: HBASE-11514 Fix findbugs warnings in blockcache
Date Tue, 15 Jul 2014 14:52:30 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 11fccd94c -> 2a20143f7


HBASE-11514 Fix findbugs warnings in blockcache


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2a20143f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2a20143f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2a20143f

Branch: refs/heads/branch-1
Commit: 2a20143f72c32ededd90bcdbb2e162174498914c
Parents: 11fccd9
Author: stack <stack@apache.org>
Authored: Tue Jul 15 07:52:22 2014 -0700
Committer: stack <stack@apache.org>
Committed: Tue Jul 15 07:52:22 2014 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/io/hfile/CacheConfig.java      |  50 +++++----
 .../hadoop/hbase/io/hfile/LruBlockCache.java    |  22 +++-
 .../hbase/io/hfile/bucket/BucketCache.java      |  24 ++++-
 .../hadoop/hbase/io/hfile/package-info.java     | 102 ++++++-------------
 .../hbase/io/hfile/TestBlockCacheReporting.java |   1 -
 .../hadoop/hbase/io/hfile/TestCacheConfig.java  |  21 +++-
 src/main/docbkx/book.xml                        |  43 +++-----
 7 files changed, 136 insertions(+), 127 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/2a20143f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 2e9a41d..d119c85 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -117,7 +117,7 @@ public class CacheConfig {
    * A float which designates how much of the overall cache to give to bucket cache
    * and how much to on-heap lru cache when {@link #BUCKET_CACHE_COMBINED_KEY} is set.
    */
-  public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY = 
+  public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY =
       "hbase.bucketcache.percentage.in.combinedcache";
 
   public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
@@ -451,21 +451,12 @@ public class CacheConfig {
   @VisibleForTesting
   static boolean blockCacheDisabled = false;
 
-  /**
-   * Returns the block cache or <code>null</code> in case none should be used.
-   *
-   * @param conf  The current configuration.
-   * @return The block cache or <code>null</code>.
-   */
-  public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
-    if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
-    if (blockCacheDisabled) return null;
-
+  private static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
     float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
       HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
     if (cachePercentage <= 0.0001f) {
       blockCacheDisabled = true;
-      return null;
+      return -1;
     }
     if (cachePercentage > 1.0) {
       throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
@@ -473,8 +464,20 @@ public class CacheConfig {
     }
 
     // Calculate the amount of heap to give the heap.
+    return (long) (mu.getMax() * cachePercentage);
+  }
+
+  /**
+   * Returns the block cache or <code>null</code> in case none should be used.
+   *
+   * @param conf  The current configuration.
+   * @return The block cache or <code>null</code>.
+   */
+  public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
+    if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
+    if (blockCacheDisabled) return null;
     MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
-    long lruCacheSize = (long) (mu.getMax() * cachePercentage);
+    long lruCacheSize = getLruCacheSize(conf, mu);
     int blockSize = conf.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE);
 
     String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
@@ -492,26 +495,23 @@ public class CacheConfig {
       int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
         DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
       String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
-      float combinedPercentage = conf.getFloat(
-        BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
+      float combinedPercentage = conf.getFloat(BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
         DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
       String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
-      int[] bucketSizes = null;
+      int [] bucketSizes = null;
       if (configuredBucketSizes != null) {
         bucketSizes = new int[configuredBucketSizes.length];
         for (int i = 0; i < configuredBucketSizes.length; i++) {
           bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i]);
         }
       }
+      if (combinedWithLru) {
+        lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
+        bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
+      }
       LOG.info("Allocating LruBlockCache size=" +
         StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
       LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
-      lruCache.setVictimCache(bucketCache);
-      if (bucketCache != null && combinedWithLru) {
-        GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache);
-      } else {
-        GLOBAL_BLOCK_CACHE_INSTANCE = lruCache;
-      }
       try {
         int ioErrorsTolerationDuration = conf.getInt(
           "hbase.bucketcache.ioengine.errors.tolerated.duration",
@@ -523,6 +523,12 @@ public class CacheConfig {
         LOG.error("Can't instantiate bucket cache", ioex);
         throw new RuntimeException(ioex);
       }
+      if (combinedWithLru) {
+        GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache);
+      } else {
+        GLOBAL_BLOCK_CACHE_INSTANCE = lruCache;
+      }
+      lruCache.setVictimCache(bucketCache);
     }
     LOG.info("Allocating LruBlockCache size=" +
       StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));

http://git-wip-us.apache.org/repos/asf/hbase/blob/2a20143f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index dd2503c..5595439 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -698,7 +698,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
    */
   static class EvictionThread extends HasThread {
     private WeakReference<LruBlockCache> cache;
-    private boolean go = true;
+    private volatile boolean go = true;
     // flag set after enter the run method, used for test
     private boolean enteringRun = false;
 
@@ -861,7 +861,25 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
 
           @Override
           public int compareTo(CachedBlock other) {
-            return (int)(other.getOffset() - this.getOffset());
+            int diff = this.getFilename().compareTo(other.getFilename());
+            if (diff != 0) return diff;
+            diff = (int)(this.getOffset() - other.getOffset());
+            if (diff != 0) return diff;
+            if (other.getCachedTime() < 0 || this.getCachedTime() < 0) {
+              throw new IllegalStateException("" + this.getCachedTime() + ", " +
+                other.getCachedTime());
+            }
+            return (int)(other.getCachedTime() - this.getCachedTime());
+          }
+
+          @Override
+          public boolean equals(Object obj) {
+            if (obj instanceof CachedBlock) {
+              CachedBlock cb = (CachedBlock)obj;
+              return compareTo(cb) == 0;
+            } else {
+              return false;
+            }
           }
         };
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/2a20143f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index a1f9a50..e3a8118 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -259,6 +259,10 @@ public class BucketCache implements BlockCache, HeapSize {
       persistencePath + ", bucketAllocator=" + this.bucketAllocator);
   }
 
+  public long getMaxSize() {
+    return this.cacheCapacity;
+  }
+
   public String getIoEngine() {
     return ioEngine.toString();
   }
@@ -1248,7 +1252,25 @@ public class BucketCache implements BlockCache, HeapSize {
 
           @Override
           public int compareTo(CachedBlock other) {
-            return (int)(this.getOffset() - other.getOffset());
+            int diff = this.getFilename().compareTo(other.getFilename());
+            if (diff != 0) return diff;
+            diff = (int)(this.getOffset() - other.getOffset());
+            if (diff != 0) return diff;
+            if (other.getCachedTime() < 0 || this.getCachedTime() < 0) {
+              throw new IllegalStateException("" + this.getCachedTime() + ", " +
+                other.getCachedTime());
+            }
+            return (int)(other.getCachedTime() - this.getCachedTime());
+          }
+
+          @Override
+          public boolean equals(Object obj) {
+            if (obj instanceof CachedBlock) {
+              CachedBlock cb = (CachedBlock)obj;
+              return compareTo(cb) == 0;
+            } else {
+              return false;
+            }
           }
         };
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/2a20143f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java
index 74d2603..362181c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java
@@ -20,89 +20,48 @@
  * {@link org.apache.hadoop.hbase.io.hfile.BlockCache}.  Caches are configured (and instantiated)
  * by {@link org.apache.hadoop.hbase.io.hfile.CacheConfig}.  See head of the
  * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig} class for constants that define
- * cache options and configuration keys to use setting cache options.  Cache implementations
- * include the default, native on-heap {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache},
- * a {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache} that can serve as an L2 for
- * {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache} (hosted inside the class
- * {@link org.apache.hadoop.hbase.io.hfile.DoubleBlockCache} that caches blocks in BOTH L1
and L2,
- * and on evict, moves from L1 to L2, etc), and a
+ * cache options and configuration keys to use setting cache options. Cache implementations
+ * include the default, native on-heap {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache}
and a
  * {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache} that has a bunch of deploy
formats
  * including acting as a L2 for LruBlockCache -- when a block is evicted from LruBlockCache,
it
- * goes to the BucketCache and when we search a block, we look in both places -- or using
- * {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache}, as
- * a host for data blocks with meta blocks in the LRUBlockCache as well as onheap, offheap,
and
- * file options.
+ * goes to the BucketCache and when we search a block, we look in both places -- or
+ * using {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache}, BucketCache is used
as
+ * a host for data blocks with meta blocks in an instance of LruBlockCache.  BucketCache
+ * can also be onheap, offheap, and file-backed.
  * 
  * <h1>Which BlockCache should I use?</h1>
- * BucketCache has seen more production deploys and has more deploy options.  Fetching will
always
+ * By default LruBlockCache is on.  If you would like to cache more, and offheap, try enabling
+ * BucketCache. Fetching will always
  * be slower when fetching from BucketCache but latencies tend to be less erratic over time
- * (roughly because GC is less).  SlabCache tends to do more GCs as blocks are moved between
L1
- * and L2 always, at least given the way {@link org.apache.hadoop.hbase.io.hfile.DoubleBlockCache}
- * currently works. It is tough doing an apples to apples compare since their hosting classes,
- * {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache} for BucketCache vs
- * {@link org.apache.hadoop.hbase.io.hfile.DoubleBlockCache} operate so differently.
- * See Nick Dimiduk's
- * <a href="http://www.n10k.com/blog/blockcache-101/">BlockCache 101</a> for
some numbers. See
- * also the description of <a href="https://issues.apache.org/jira/browse/HBASE-7404">HBASE-7404</a>
- * where Chunhui Shen lists issues he found with BlockCache (inefficent use of memory, doesn't
- * help w/ GC).
- *
- * <h1>Enabling {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache}</h2>
- * {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache} is the original offheap block
cache
- * but unfortunately has seen little use. It is originally described in
- * <a href="http://blog.cloudera.com/blog/2012/01/caching-in-hbase-slabcache/">Caching
- * in Apache HBase: SlabCache</a>.To enable it,
- * set the float <code>hbase.offheapcache.percentage</code>
- * ({@link org.apache.hadoop.hbase.io.hfile.CacheConfig#SLAB_CACHE_OFFHEAP_PERCENTAGE_KEY})
to some
- * value between 0 and 1 in
- * your <code>hbase-site.xml</code> file. This
- * enables {@link org.apache.hadoop.hbase.io.hfile.DoubleBlockCache}, a facade over
- * {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache} and
- * {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache}.  DoubleBlockCache works as follows.
- * When caching, it  
- * "...attempts to cache the block in both caches, while readblock reads first from the faster
- * onheap cache before looking for the block in the off heap cache. Metrics are the
- * combined size and hits and misses of both caches." The value set in
- * <code>hbase.offheapcache.percentage</code> will be
- * multiplied by whatever the setting for <code>-XX:MaxDirectMemorySize</code>
is in
- * your <code>hbase-env.sh</code> configuration file and this is what
- * will be used by {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache} as its offheap
store.
- * Onheap store will be whatever the float
- * {@link org.apache.hadoop.hbase.HConstants#HFILE_BLOCK_CACHE_SIZE_KEY} setting is
- * (some value between 0 and 1) times the size of the allocated java heap.
- * 
- * <p>Restart (or rolling restart) your cluster for the configs to take effect.  Check
logs to
- * ensure your configurations came out as expected.
+ * (roughly because GC is less). See Nick Dimiduk's
+ * <a href="http://www.n10k.com/blog/blockcache-101/">BlockCache 101</a> for
some numbers.
  *
  * <h1>Enabling {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache}</h2>
- * Ensure the SlabCache config <code>hbase.offheapcache.percentage</code> is
not set (or set to 0).
- * At this point, it is probably best to read the code to learn the list of bucket cache
options
- * and how they combine (to be fixed).  Read the options and defaults for BucketCache in
the
- * head of the {@link org.apache.hadoop.hbase.io.hfile.CacheConfig}.
+ * Read the options and defaults for BucketCache in the head of the
+ * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig}.
  * 
- * <p>Here is a simple example of how to enable a <code>4G</code>
- * offheap bucket cache with 1G onheap cache.
- * The onheap/offheap caches
- * are managed by {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache} by default.
For the
- * CombinedBlockCache (from the class comment), "The smaller lruCache is used
+ * <p>Here is a simple example of how to enable a <code>4G</code> offheap
bucket cache with 1G
+ * onheap cache managed by {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache}.
+ * CombinedBlockCache will put DATA blocks in the BucketCache and META blocks -- INDEX and
BLOOMS
+ * -- in an instance of the LruBlockCache. For the
+ * CombinedBlockCache (from the class comment), "[t]he smaller lruCache is used
  * to cache bloom blocks and index blocks, the larger bucketCache is used to
  * cache data blocks. getBlock reads first from the smaller lruCache before
  * looking for the block in the bucketCache. Metrics are the combined size and
  * hits and misses of both caches."  To disable CombinedBlockCache and have the BucketCache
act
  * as a strict L2 cache to the L1 LruBlockCache (i.e. on eviction from L1, blocks go to L2),
set
- * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#BUCKET_CACHE_COMBINED_KEY} to false.
- * Also by default, unless you change it,
- * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#BUCKET_CACHE_COMBINED_PERCENTAGE_KEY}
- * defaults to <code>0.9</code> (see
- * the top of the CacheConfig in the BucketCache defaults section).  This means that whatever
- * size you set for the bucket cache with
- * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#BUCKET_CACHE_SIZE_KEY},
- * <code>90%</code> will be used for offheap and <code>10%</code>
of the size will be used
- * by the onheap {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache}.
- * <p>Back to the example of setting an onheap cache of 1G and ofheap of 4G, in 
- * <code>hbase-env.sh</code> ensure the java option <code>-XX:MaxDirectMemorySize</code>
is
- * enabled and 5G in size: e.g. <code>-XX:MaxDirectMemorySize=5G</code>.  Then
in
- * <code>hbase-site.xml</code> add the following configurations:
+ * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#BUCKET_CACHE_COMBINED_KEY} to false.
By
+ * default, hbase.bucketcache.combinedcache.enabled (BUCKET_CACHE_COMBINED_KEY) is true.
+ * 
+ * <p>Back to the example of setting an onheap cache of 1G and offheap of 4G with the
BlockCache
+ * deploy managed by CombinedBlockCache. Setting hbase.bucketcache.ioengine and
+ * hbase.bucketcache.size > 0 enables CombinedBlockCache.
+ * In <code>hbase-env.sh</code> ensure the environment
+ * variable <code>-XX:MaxDirectMemorySize</code> is enabled and is bigger than
4G, say 5G in size:
+ * e.g. <code>-XX:MaxDirectMemorySize=5G</code>. This setting allows the JVM
use offheap memory
+ * up to this upper limit.  Allocate more than you need because there are other consumers
of
+ * offheap memory other than BlockCache (for example DFSClient in the RegionServer uses offheap).
+ * In <code>hbase-site.xml</code> add the following configurations:
 <pre>&lt;property>
   &lt;name>hbase.bucketcache.ioengine&lt;/name>
   &lt;value>offheap&lt;/value>
@@ -114,7 +73,8 @@
 &lt;property>
   &lt;name>hbase.bucketcache.size&lt;/name>
   &lt;value>5120&lt;/value>
-&lt;/property></pre>.  Above we set a cache of 5G, 80% of which will be offheap
(4G) and 1G onheap.
+&lt;/property></pre>.  Above we set a cache of 5G, 80% of which will be offheap
(4G) and 1G onheap
+ * (with DATA blocks in BucketCache and INDEX blocks in the onheap LruBlockCache).
  * Restart (or rolling restart) your cluster for the configs to take effect.  Check logs
to ensure
  * your configurations came out as expected.
  *

http://git-wip-us.apache.org/repos/asf/hbase/blob/2a20143f/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
index fa1f10f..cfe9078 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
@@ -73,7 +73,6 @@ public class TestBlockCacheReporting {
   public void testBucketCache() throws JsonGenerationException, JsonMappingException, IOException
{
     this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
     this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100);
-    this.conf.setFloat(CacheConfig.BUCKET_CACHE_COMBINED_PERCENTAGE_KEY, 0.8f);
     CacheConfig cc = new CacheConfig(this.conf);
     assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
     logPerBlock(cc.getBlockCache());

http://git-wip-us.apache.org/repos/asf/hbase/blob/2a20143f/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
index 8f7ed2b..dfba277 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
@@ -30,6 +30,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -200,12 +201,27 @@ public class TestCacheConfig {
   @Test
   public void testBucketCacheConfig() {
     this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
-    this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100);
-    this.conf.setFloat(CacheConfig.BUCKET_CACHE_COMBINED_PERCENTAGE_KEY, 0.8f);
+    final float percent = 0.8f;
+    this.conf.setFloat(CacheConfig.BUCKET_CACHE_COMBINED_PERCENTAGE_KEY, percent);
+    final int bcSize = 100;
+    this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, bcSize);
     CacheConfig cc = new CacheConfig(this.conf);
     basicBlockCacheOps(cc, false, false);
     assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
     // TODO: Assert sizes allocated are right and proportions.
+    CombinedBlockCache cbc = (CombinedBlockCache)cc.getBlockCache();
+    BlockCache [] bcs = cbc.getBlockCaches();
+    assertTrue(bcs[0] instanceof LruBlockCache);
+    LruBlockCache lbc = (LruBlockCache)bcs[0];
+    long expectedBCSize = (long)(bcSize * (1.0f - percent));
+    long actualBCSize = lbc.getMaxSize() / (1024 * 1024);
+    assertTrue(expectedBCSize == actualBCSize);
+    assertTrue(bcs[1] instanceof BucketCache);
+    BucketCache bc = (BucketCache)bcs[1];
+    // getMaxSize comes back in bytes but we specified size in MB
+    expectedBCSize = (long)(bcSize * percent);
+    actualBCSize = (long)(bc.getMaxSize() / (1024 * 1024));
+    assertTrue(expectedBCSize == actualBCSize);
   }
 
   /**
@@ -216,7 +232,6 @@ public class TestCacheConfig {
   public void testCacheDataInL1() {
     this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
     this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100);
-    this.conf.setFloat(CacheConfig.BUCKET_CACHE_COMBINED_PERCENTAGE_KEY, 0.8f);
     CacheConfig cc = new CacheConfig(this.conf);
     assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
     CombinedBlockCache cbc = (CombinedBlockCache)cc.getBlockCache();

http://git-wip-us.apache.org/repos/asf/hbase/blob/2a20143f/src/main/docbkx/book.xml
----------------------------------------------------------------------
diff --git a/src/main/docbkx/book.xml b/src/main/docbkx/book.xml
index a1b7991..7e58c1c 100644
--- a/src/main/docbkx/book.xml
+++ b/src/main/docbkx/book.xml
@@ -1945,7 +1945,7 @@ rs.close();
         <title>Block Cache</title>
 
         <para>HBase provides three different BlockCache implementations: the default
onheap
-          LruBlockCache, and BucketCache, and SlabCache, which are both (usually) offheap.
This section
+          LruBlockCache, BucketCache, and SlabCache, which are both (usually) offheap. This
section
           discusses benefits and drawbacks of each implementation, how to choose the appropriate
           option, and configuration options for each.</para>
         <section>
@@ -1980,10 +1980,7 @@ rs.close();
               works so differently, it is difficult to do a fair comparison between BucketCache
and SlabCache.
             See Nick Dimiduk's <link
               xlink:href="http://www.n10k.com/blog/blockcache-101/">BlockCache 101</link>
for some
-            numbers. See also the description of <link
-              xlink:href="https://issues.apache.org/jira/browse/HBASE-7404">HBASE-7404</link>
where
-            Chunhui Shen lists issues he found with BlockCache, such as inefficient use of
memory
-            and garbage-collection overhead.</para>
+          numbers.</para>
           <para>For more information about the off heap cache options, see <xref
               linkend="offheap.blockcache" />.</para>
         </section>
@@ -2164,16 +2161,11 @@ rs.close();
           </section>
           <section>
             <title>Enable BucketCache</title>
-            <para> To enable BucketCache, set the value of
-                <varname>hbase.offheapcache.percentage</varname> to 0 in the
RegionServer's
-                <filename>hbase-site.xml</filename> file. This disables SlabCache.</para>
-
-                <para>Just as for SlabCache, the usual deploy of BucketCache is via
a
+                <para>The usual deploy of BucketCache is via a
                     managing class that sets up two caching tiers: an L1 onheap cache
                     implemented by LruBlockCache and a second L2 cache implemented
                     with BucketCache. The managing class is <link
-                xlink:href="http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.html">CombinedBlockCache</link>
-            by default. The just-previous link describes the mechanism of CombinedBlockCache.
In short, it works
+                xlink:href="http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.html">CombinedBlockCache</link>
by default. The just-previous link describes the mechanism of CombinedBlockCache. In short,
it works
             by keeping meta blocks -- INDEX and BLOOM in the L1, onheap LruBlockCache tier
-- and DATA
             blocks are kept in the L2, BucketCache tier. It is possible to amend this behavior
in
             HBase since version 1.0 and ask that a column family have both its meta and DATA
blocks hosted onheap in the L1 tier by
@@ -2189,32 +2181,29 @@ rs.close();
             (Useful in particular if you have some fast i/o attached to the box such
             as SSDs).
         </para>
-        <para>To disable
-              CombinedBlockCache, and use the BucketCache as a strict L2 cache to the L1
+        <para>To disable CombinedBlockCache, and use the BucketCache as a strict L2
cache to the L1
               LruBlockCache, set <varname>CacheConfig.BUCKET_CACHE_COMBINED_KEY</varname>
to
                 <literal>false</literal>. In this mode, on eviction from L1,
blocks go to L2.</para>
 
-            <para> By default, <varname>CacheConfig.BUCKET_CACHE_COMBINED_PERCENTAGE_KEY</varname>
-              defaults to <literal>0.9</literal>. This means that whatever size
you set for the
-              bucket cache with <varname>CacheConfig.BUCKET_CACHE_SIZE_KEY</varname>,
90% will be
-              used for offheap and 10% will be used by the onheap LruBlockCache. </para>
-          <para>
-          </para>
             <procedure>
               <title>BucketCache Example Configuration</title>
-              <para> This sample provides a configuration for a 4 GB offheap BucketCache
with a 1 GB
-                onheap cache. Configuration is performed on the RegionServer.</para>
+              <para>This sample provides a configuration for a 4 GB offheap BucketCache
with a 1 GB
+                  onheap cache. Configuration is performed on the RegionServer.  Setting
+                  <varname>hbase.bucketcache.ioengine</varname> and 
+                  <varname>hbase.bucketcache.size</varname> &gt; 0 enables
CombinedBlockCache.
+              </para>
               <step>
                 <para>First, edit the RegionServer's <filename>hbase-env.sh</filename>
and set
-                  -XX:MaxDirectMemorySize to the total size of the desired onheap plus offheap,
in
-                  this case, 5 GB (but expressed as 5G).</para>
+                  -XX:MaxDirectMemorySize to a value greater than the offheap size wanted,
in
+                  this case, 4 GB (expressed as 4G).  Lets set it to 5G.  That'll be 4G
+                  for our offheap cache and 1G for any other uses of offheap memory (there
are
+                  other users of offheap memory other than BlockCache; e.g. DFSClient 
+                  in RegionServer can make use of offheap memory).</para>
                 <programlisting>-XX:MaxDirectMemorySize=5G</programlisting>
               </step>
               <step>
                 <para>Next, add the following configuration to the RegionServer's
-                    <filename>hbase-site.xml</filename>. This configuration uses
80% of the
-                  -XX:MaxDirectMemorySize (4 GB) for offheap, and the remainder (1 GB) for
-                  onheap.</para>
+                    <filename>hbase-site.xml</filename>.</para>
                 <programlisting>
 <![CDATA[<property>
   <name>hbase.bucketcache.ioengine</name>


Mime
View raw message