Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id BDDD3200B5A for ; Thu, 4 Aug 2016 15:40:40 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id BC1BC160AAB; Thu, 4 Aug 2016 13:40:40 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D8953160A6A for ; Thu, 4 Aug 2016 15:40:39 +0200 (CEST) Received: (qmail 55475 invoked by uid 500); 4 Aug 2016 13:40:39 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 55464 invoked by uid 99); 4 Aug 2016 13:40:39 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Aug 2016 13:40:39 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D2EEEE0A7D; Thu, 4 Aug 2016 13:40:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: chenheng@apache.org To: commits@hbase.apache.org Message-Id: <83405e18cde342e2b7368fc709720416@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun) Date: Thu, 4 Aug 2016 13:40:38 +0000 (UTC) archived-at: Thu, 04 Aug 2016 13:40:40 -0000 Repository: hbase Updated Branches: refs/heads/branch-1.2 dfc9616ae -> 3bce24dee HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3bce24de Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3bce24de Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3bce24de Branch: refs/heads/branch-1.2 Commit: 3bce24dee6edc3af9d3190a506e9d6df4c115108 Parents: dfc9616 Author: chenheng Authored: Thu Aug 4 21:13:42 2016 +0800 Committer: chenheng Committed: Thu Aug 4 21:37:48 2016 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/io/hfile/LruBlockCache.java | 36 ++++++++++++++++++-- .../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 2 ++ .../hbase/io/hfile/TestLruBlockCache.java | 5 +++ 3 files changed, 40 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/3bce24de/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 2781833..52ecf00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -112,6 +112,10 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { */ static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor"; + /** + * Hard capacity limit of cache, will reject any put if size > this * acceptable + */ + static final String LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.hard.capacity.limit.factor"; static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.single.percentage"; static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.multi.percentage"; static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.memory.percentage"; @@ -138,6 +142,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { static final float DEFAULT_MULTI_FACTOR = 0.50f; static final float DEFAULT_MEMORY_FACTOR = 0.25f; + /** default hard capacity limit */ + static final float DEFAULT_HARD_CAPACITY_LIMIT_FACTOR = 1.2f; + static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false; /** Statistics thread */ @@ -171,6 +178,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { /** Cache access count (sequential ID) */ private final AtomicLong count; + /** hard capacity limit */ + private float hardCapacityLimitFactor; + /** Cache statistics */ private final CacheStats stats; @@ -228,6 +238,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { DEFAULT_SINGLE_FACTOR, DEFAULT_MULTI_FACTOR, DEFAULT_MEMORY_FACTOR, + DEFAULT_HARD_CAPACITY_LIMIT_FACTOR, false, DEFAULT_MAX_BLOCK_SIZE ); @@ -243,6 +254,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, DEFAULT_SINGLE_FACTOR), conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR), conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR), + conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, DEFAULT_HARD_CAPACITY_LIMIT_FACTOR), conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE), conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE) ); @@ -269,7 +281,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel, float minFactor, float acceptableFactor, float singleFactor, - float multiFactor, float memoryFactor, boolean forceInMemory, long maxBlockSize) { + float multiFactor, float memoryFactor, float hardLimitFactor, + boolean forceInMemory, long maxBlockSize) { this.maxBlockSize = maxBlockSize; if(singleFactor + multiFactor + memoryFactor != 1 || singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) { @@ -297,6 +310,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { this.elements = new AtomicLong(0); this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel); this.size = new AtomicLong(this.overhead); + this.hardCapacityLimitFactor = hardLimitFactor; if(evictionThread) { this.evictionThread = new EvictionThread(this); this.evictionThread.start(); // FindBugs SC_START_IN_CTOR @@ -359,6 +373,22 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { LOG.warn(msg); return; } + long currentSize = size.get(); + long currentAcceptableSize = acceptableSize(); + long hardLimitSize = (long) (hardCapacityLimitFactor * currentAcceptableSize); + if (currentSize >= hardLimitSize) { + stats.failInsert(); + if (LOG.isTraceEnabled()) { + LOG.trace("LruBlockCache current size " + StringUtils.byteDesc(currentSize) + + " has exceeded acceptable size " + StringUtils.byteDesc(currentAcceptableSize) + " too many." + + " the hard limit size is " + StringUtils.byteDesc(hardLimitSize) + ", failed to put cacheKey:" + + cacheKey + " into LruBlockCache."); + } + if (!evictionInProgress) { + runEviction(); + } + return; + } cb = new LruCachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory); long newSize = updateSizeMetrics(cb, false); map.put(cacheKey, cb); @@ -367,7 +397,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { long size = map.size(); assertCounterSanity(size, val); } - if (newSize > acceptableSize() && !evictionInProgress) { + if (newSize > currentAcceptableSize && !evictionInProgress) { runEviction(); } } @@ -904,7 +934,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public final static long CACHE_FIXED_OVERHEAD = ClassSize.align( (4 * Bytes.SIZEOF_LONG) + (9 * ClassSize.REFERENCE) + - (5 * Bytes.SIZEOF_FLOAT) + (2 * Bytes.SIZEOF_BOOLEAN) + (6 * Bytes.SIZEOF_FLOAT) + (2 * Bytes.SIZEOF_BOOLEAN) + ClassSize.OBJECT); @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/3bce24de/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index c89ef92..f6cae6d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -152,6 +152,8 @@ public class TestCacheOnWrite { // default blockcaches.add(new CacheConfig(conf).getBlockCache()); + //set LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME to 2.0f due to HBASE-16287 + TEST_UTIL.getConfiguration().setFloat(LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 2.0f); // memory BlockCache lru = new LruBlockCache(128 * 1024 * 1024, 64 * 1024, TEST_UTIL.getConfiguration()); blockcaches.add(lru); http://git-wip-us.apache.org/repos/asf/hbase/blob/3bce24de/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java index fd239ab..00e0e20 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -267,6 +267,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false, 16 * 1024 * 1024); @@ -388,6 +389,7 @@ public class TestLruBlockCache { 0.2f, // single 0.3f, // multi 0.5f, // memory + 1.2f, // limit true, 16 * 1024 * 1024); @@ -494,6 +496,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false, 16 * 1024 * 1024); @@ -557,6 +560,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false, 1024); CachedItem [] tooLong = generateFixedBlocks(10, 1024+5, "long"); @@ -596,6 +600,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false, 16 * 1024 * 1024);