Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 04A9F19A45 for ; Mon, 7 Mar 2016 16:13:38 +0000 (UTC) Received: (qmail 59892 invoked by uid 500); 7 Mar 2016 16:13:37 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 59709 invoked by uid 500); 7 Mar 2016 16:13:37 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 59688 invoked by uid 99); 7 Mar 2016 16:13:37 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Mar 2016 16:13:37 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 71AC4DF99E; Mon, 7 Mar 2016 16:13:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Mon, 07 Mar 2016 16:13:39 -0000 Message-Id: <9d4e6e5d9bcf452e886a4cbc754a15c5@git.apache.org> In-Reply-To: <0e0e397b78fe4c43933b2afc1cdc126f@git.apache.org> References: <0e0e397b78fe4c43933b2afc1cdc126f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [03/51] [partial] hbase-site git commit: Published site at 05161fcbfdd78f5684b9cb52c49a02be5ad14499. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/2211f347/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/CacheConfig.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/CacheConfig.html b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/CacheConfig.html index 6d8219b..429c2fd 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/CacheConfig.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/CacheConfig.html @@ -53,649 +53,657 @@ 045 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName()); 046 047 /** -048 * Configuration key to cache data blocks on write. There are separate -049 * switches for bloom blocks and non-root index blocks. +048 * Configuration key to cache data blocks on read. Bloom blocks and index blocks are always be +049 * cached if the block cache is enabled. 050 */ -051 public static final String CACHE_BLOCKS_ON_WRITE_KEY = -052 "hbase.rs.cacheblocksonwrite"; -053 -054 /** -055 * Configuration key to cache leaf and intermediate-level index blocks on -056 * write. -057 */ -058 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY = -059 "hfile.block.index.cacheonwrite"; -060 -061 /** -062 * Configuration key to cache compound bloom filter blocks on write. +051 public static final String CACHE_DATA_ON_READ_KEY = "hbase.block.data.cacheonread"; +052 +053 /** +054 * Configuration key to cache data blocks on write. There are separate +055 * switches for bloom blocks and non-root index blocks. +056 */ +057 public static final String CACHE_BLOCKS_ON_WRITE_KEY = +058 "hbase.rs.cacheblocksonwrite"; +059 +060 /** +061 * Configuration key to cache leaf and intermediate-level index blocks on +062 * write. 063 */ -064 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY = -065 "hfile.block.bloom.cacheonwrite"; +064 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY = +065 "hfile.block.index.cacheonwrite"; 066 067 /** -068 * Configuration key to cache data blocks in compressed and/or encrypted format. +068 * Configuration key to cache compound bloom filter blocks on write. 069 */ -070 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY = -071 "hbase.block.data.cachecompressed"; +070 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY = +071 "hfile.block.bloom.cacheonwrite"; 072 073 /** -074 * Configuration key to evict all blocks of a given file from the block cache -075 * when the file is closed. -076 */ -077 public static final String EVICT_BLOCKS_ON_CLOSE_KEY = -078 "hbase.rs.evictblocksonclose"; -079 -080 /** -081 * Configuration keys for Bucket cache +074 * Configuration key to cache data blocks in compressed and/or encrypted format. +075 */ +076 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY = +077 "hbase.block.data.cachecompressed"; +078 +079 /** +080 * Configuration key to evict all blocks of a given file from the block cache +081 * when the file is closed. 082 */ -083 -084 /** -085 * If the chosen ioengine can persist its state across restarts, the path to the file to persist -086 * to. This file is NOT the data file. It is a file into which we will serialize the map of -087 * what is in the data file. For example, if you pass the following argument as -088 * BUCKET_CACHE_IOENGINE_KEY ("hbase.bucketcache.ioengine"), -089 * <code>file:/tmp/bucketcache.data </code>, then we will write the bucketcache data to the file -090 * <code>/tmp/bucketcache.data</code> but the metadata on where the data is in the supplied file -091 * is an in-memory map that needs to be persisted across restarts. Where to store this -092 * in-memory state is what you supply here: e.g. <code>/tmp/bucketcache.map</code>. -093 */ -094 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = -095 "hbase.bucketcache.persistent.path"; -096 -097 /** -098 * If the bucket cache is used in league with the lru on-heap block cache (meta blocks such -099 * as indices and blooms are kept in the lru blockcache and the data blocks in the -100 * bucket cache). -101 */ -102 public static final String BUCKET_CACHE_COMBINED_KEY = -103 "hbase.bucketcache.combinedcache.enabled"; -104 -105 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads"; -106 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = -107 "hbase.bucketcache.writer.queuelength"; -108 -109 /** -110 * A comma-delimited array of values for use as bucket sizes. -111 */ -112 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes"; -113 -114 /** -115 * Defaults for Bucket cache -116 */ -117 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true; -118 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3; -119 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64; -120 -121 /** -122 * Configuration key to prefetch all blocks of a given file into the block cache -123 * when the file is opened. -124 */ -125 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY = -126 "hbase.rs.prefetchblocksonopen"; -127 -128 /** -129 * The target block size used by blockcache instances. Defaults to -130 * {@link HConstants#DEFAULT_BLOCKSIZE}. -131 * TODO: this config point is completely wrong, as it's used to determine the -132 * target block size of BlockCache instances. Rename. -133 */ -134 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize"; -135 -136 private static final String EXTERNAL_BLOCKCACHE_KEY = "hbase.blockcache.use.external"; -137 private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false; -138 -139 private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY="hbase.blockcache.external.class"; -140 private static final String DROP_BEHIND_CACHE_COMPACTION_KEY="hbase.hfile.drop.behind.compaction"; -141 private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true; -142 -143 /** -144 * Enum of all built in external block caches. -145 * This is used for config. -146 */ -147 private static enum ExternalBlockCaches { -148 memcached("org.apache.hadoop.hbase.io.hfile.MemcachedBlockCache"); -149 // TODO(eclark): Consider more. Redis, etc. -150 Class<? extends BlockCache> clazz; -151 ExternalBlockCaches(String clazzName) { -152 try { -153 clazz = (Class<? extends BlockCache>) Class.forName(clazzName); -154 } catch (ClassNotFoundException cnef) { -155 clazz = null; -156 } -157 } -158 ExternalBlockCaches(Class<? extends BlockCache> clazz) { -159 this.clazz = clazz; -160 } -161 } -162 -163 // Defaults -164 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true; -165 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false; -166 public static final boolean DEFAULT_IN_MEMORY = false; -167 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false; -168 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false; -169 public static final boolean DEFAULT_EVICT_ON_CLOSE = false; -170 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false; -171 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false; -172 -173 /** Local reference to the block cache, null if completely disabled */ -174 private final BlockCache blockCache; -175 -176 /** -177 * Whether blocks should be cached on read (default is on if there is a -178 * cache but this can be turned off on a per-family or per-request basis). -179 * If off we will STILL cache meta blocks; i.e. INDEX and BLOOM types. -180 * This cannot be disabled. -181 */ -182 private boolean cacheDataOnRead; -183 -184 /** Whether blocks should be flagged as in-memory when being cached */ -185 private final boolean inMemory; -186 -187 /** Whether data blocks should be cached when new files are written */ -188 private boolean cacheDataOnWrite; -189 -190 /** Whether index blocks should be cached when new files are written */ -191 private final boolean cacheIndexesOnWrite; -192 -193 /** Whether compound bloom filter blocks should be cached on write */ -194 private final boolean cacheBloomsOnWrite; -195 -196 /** Whether blocks of a file should be evicted when the file is closed */ -197 private boolean evictOnClose; -198 -199 /** Whether data blocks should be stored in compressed and/or encrypted form in the cache */ -200 private final boolean cacheDataCompressed; -201 -202 /** Whether data blocks should be prefetched into the cache */ -203 private final boolean prefetchOnOpen; -204 -205 /** -206 * If true and if more than one tier in this cache deploy -- e.g. CombinedBlockCache has an L1 -207 * and an L2 tier -- then cache data blocks up in the L1 tier (The meta blocks are likely being -208 * cached up in L1 already. At least this is the case if CombinedBlockCache). -209 */ -210 private boolean cacheDataInL1; +083 public static final String EVICT_BLOCKS_ON_CLOSE_KEY = +084 "hbase.rs.evictblocksonclose"; +085 +086 /** +087 * Configuration keys for Bucket cache +088 */ +089 +090 /** +091 * If the chosen ioengine can persist its state across restarts, the path to the file to persist +092 * to. This file is NOT the data file. It is a file into which we will serialize the map of +093 * what is in the data file. For example, if you pass the following argument as +094 * BUCKET_CACHE_IOENGINE_KEY ("hbase.bucketcache.ioengine"), +095 * <code>file:/tmp/bucketcache.data </code>, then we will write the bucketcache data to the file +096 * <code>/tmp/bucketcache.data</code> but the metadata on where the data is in the supplied file +097 * is an in-memory map that needs to be persisted across restarts. Where to store this +098 * in-memory state is what you supply here: e.g. <code>/tmp/bucketcache.map</code>. +099 */ +100 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = +101 "hbase.bucketcache.persistent.path"; +102 +103 /** +104 * If the bucket cache is used in league with the lru on-heap block cache (meta blocks such +105 * as indices and blooms are kept in the lru blockcache and the data blocks in the +106 * bucket cache). +107 */ +108 public static final String BUCKET_CACHE_COMBINED_KEY = +109 "hbase.bucketcache.combinedcache.enabled"; +110 +111 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads"; +112 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = +113 "hbase.bucketcache.writer.queuelength"; +114 +115 /** +116 * A comma-delimited array of values for use as bucket sizes. +117 */ +118 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes"; +119 +120 /** +121 * Defaults for Bucket cache +122 */ +123 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true; +124 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3; +125 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64; +126 +127 /** +128 * Configuration key to prefetch all blocks of a given file into the block cache +129 * when the file is opened. +130 */ +131 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY = +132 "hbase.rs.prefetchblocksonopen"; +133 +134 /** +135 * The target block size used by blockcache instances. Defaults to +136 * {@link HConstants#DEFAULT_BLOCKSIZE}. +137 * TODO: this config point is completely wrong, as it's used to determine the +138 * target block size of BlockCache instances. Rename. +139 */ +140 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize"; +141 +142 private static final String EXTERNAL_BLOCKCACHE_KEY = "hbase.blockcache.use.external"; +143 private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false; +144 +145 private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY = "hbase.blockcache.external.class"; +146 private static final String DROP_BEHIND_CACHE_COMPACTION_KEY = +147 "hbase.hfile.drop.behind.compaction"; +148 private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true; +149 +150 /** +151 * Enum of all built in external block caches. +152 * This is used for config. +153 */ +154 private static enum ExternalBlockCaches { +155 memcached("org.apache.hadoop.hbase.io.hfile.MemcachedBlockCache"); +156 // TODO(eclark): Consider more. Redis, etc. +157 Class<? extends BlockCache> clazz; +158 ExternalBlockCaches(String clazzName) { +159 try { +160 clazz = (Class<? extends BlockCache>) Class.forName(clazzName); +161 } catch (ClassNotFoundException cnef) { +162 clazz = null; +163 } +164 } +165 ExternalBlockCaches(Class<? extends BlockCache> clazz) { +166 this.clazz = clazz; +167 } +168 } +169 +170 // Defaults +171 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true; +172 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false; +173 public static final boolean DEFAULT_IN_MEMORY = false; +174 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false; +175 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false; +176 public static final boolean DEFAULT_EVICT_ON_CLOSE = false; +177 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false; +178 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false; +179 +180 /** Local reference to the block cache, null if completely disabled */ +181 private final BlockCache blockCache; +182 +183 /** +184 * Whether blocks should be cached on read (default is on if there is a +185 * cache but this can be turned off on a per-family or per-request basis). +186 * If off we will STILL cache meta blocks; i.e. INDEX and BLOOM types. +187 * This cannot be disabled. +188 */ +189 private boolean cacheDataOnRead; +190 +191 /** Whether blocks should be flagged as in-memory when being cached */ +192 private final boolean inMemory; +193 +194 /** Whether data blocks should be cached when new files are written */ +195 private boolean cacheDataOnWrite; +196 +197 /** Whether index blocks should be cached when new files are written */ +198 private final boolean cacheIndexesOnWrite; +199 +200 /** Whether compound bloom filter blocks should be cached on write */ +201 private final boolean cacheBloomsOnWrite; +202 +203 /** Whether blocks of a file should be evicted when the file is closed */ +204 private boolean evictOnClose; +205 +206 /** Whether data blocks should be stored in compressed and/or encrypted form in the cache */ +207 private final boolean cacheDataCompressed; +208 +209 /** Whether data blocks should be prefetched into the cache */ +210 private final boolean prefetchOnOpen; 211 -212 private final boolean dropBehindCompaction; -213 -214 /** -215 * Create a cache configuration using the specified configuration object and -216 * family descriptor. -217 * @param conf hbase configuration -218 * @param family column family configuration -219 */ -220 public CacheConfig(Configuration conf, HColumnDescriptor family) { -221 this(CacheConfig.instantiateBlockCache(conf), -222 family.isBlockCacheEnabled(), -223 family.isInMemory(), -224 // For the following flags we enable them regardless of per-schema settings -225 // if they are enabled in the global configuration. -226 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, -227 DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(), -228 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, -229 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(), -230 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, -231 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(), -232 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, -233 DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(), -234 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), -235 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, -236 DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(), -237 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, -238 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(), -239 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) -240 ); -241 } -242 -243 /** -244 * Create a cache configuration using the specified configuration object and -245 * defaults for family level settings. -246 * @param conf hbase configuration -247 */ -248 public CacheConfig(Configuration conf) { -249 this(CacheConfig.instantiateBlockCache(conf), -250 DEFAULT_CACHE_DATA_ON_READ, -251 DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set -252 // strictly from conf -253 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE), -254 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE), -255 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE), -256 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE), -257 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), -258 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN), -259 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, -260 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1), -261 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) -262 ); -263 } -264 -265 /** -266 * Create a block cache configuration with the specified cache and -267 * configuration parameters. -268 * @param blockCache reference to block cache, null if completely disabled -269 * @param cacheDataOnRead whether DATA blocks should be cached on read (we always cache INDEX -270 * blocks and BLOOM blocks; this cannot be disabled). -271 * @param inMemory whether blocks should be flagged as in-memory -272 * @param cacheDataOnWrite whether data blocks should be cached on write -273 * @param cacheIndexesOnWrite whether index blocks should be cached on write -274 * @param cacheBloomsOnWrite whether blooms should be cached on write -275 * @param evictOnClose whether blocks should be evicted when HFile is closed -276 * @param cacheDataCompressed whether to store blocks as compressed in the cache -277 * @param prefetchOnOpen whether to prefetch blocks upon open -278 * @param cacheDataInL1 If more than one cache tier deployed, if true, cache this column families -279 * data blocks up in the L1 tier. -280 */ -281 CacheConfig(final BlockCache blockCache, -282 final boolean cacheDataOnRead, final boolean inMemory, -283 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite, -284 final boolean cacheBloomsOnWrite, final boolean evictOnClose, -285 final boolean cacheDataCompressed, final boolean prefetchOnOpen, -286 final boolean cacheDataInL1, final boolean dropBehindCompaction) { -287 this.blockCache = blockCache; -288 this.cacheDataOnRead = cacheDataOnRead; -289 this.inMemory = inMemory; -290 this.cacheDataOnWrite = cacheDataOnWrite; -291 this.cacheIndexesOnWrite = cacheIndexesOnWrite; -292 this.cacheBloomsOnWrite = cacheBloomsOnWrite; -293 this.evictOnClose = evictOnClose; -294 this.cacheDataCompressed = cacheDataCompressed; -295 this.prefetchOnOpen = prefetchOnOpen; -296 this.cacheDataInL1 = cacheDataInL1; -297 this.dropBehindCompaction = dropBehindCompaction; -298 LOG.info(this); -299 } -300 -301 /** -302 * Constructs a cache configuration copied from the specified configuration. -303 * @param cacheConf -304 */ -305 public CacheConfig(CacheConfig cacheConf) { -306 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory, -307 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite, -308 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose, -309 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen, -310 cacheConf.cacheDataInL1, cacheConf.dropBehindCompaction); -311 } -312 -313 /** -314 * Checks whether the block cache is enabled. -315 */ -316 public boolean isBlockCacheEnabled() { -317 return this.blockCache != null; -318 } -319 -320 /** -321 * Returns the block cache. -322 * @return the block cache, or null if caching is completely disabled +212 /** +213 * If true and if more than one tier in this cache deploy -- e.g. CombinedBlockCache has an L1 +214 * and an L2 tier -- then cache data blocks up in the L1 tier (The meta blocks are likely being +215 * cached up in L1 already. At least this is the case if CombinedBlockCache). +216 */ +217 private boolean cacheDataInL1; +218 +219 private final boolean dropBehindCompaction; +220 +221 /** +222 * Create a cache configuration using the specified configuration object and +223 * family descriptor. +224 * @param conf hbase configuration +225 * @param family column family configuration +226 */ +227 public CacheConfig(Configuration conf, HColumnDescriptor family) { +228 this(CacheConfig.instantiateBlockCache(conf), +229 conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ) +230 && family.isBlockCacheEnabled(), +231 family.isInMemory(), +232 // For the following flags we enable them regardless of per-schema settings +233 // if they are enabled in the global configuration. +234 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, +235 DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(), +236 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, +237 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(), +238 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, +239 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(), +240 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, +241 DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(), +242 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), +243 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, +244 DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(), +245 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, +246 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(), +247 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT) +248 ); +249 } +250 +251 /** +252 * Create a cache configuration using the specified configuration object and +253 * defaults for family level settings. +254 * @param conf hbase configuration +255 */ +256 public CacheConfig(Configuration conf) { +257 this(CacheConfig.instantiateBlockCache(conf), +258 conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ), +259 DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set +260 // strictly from conf +261 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE), +262 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE), +263 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE), +264 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE), +265 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), +266 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN), +267 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, +268 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1), +269 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT) +270 ); +271 } +272 +273 /** +274 * Create a block cache configuration with the specified cache and +275 * configuration parameters. +276 * @param blockCache reference to block cache, null if completely disabled +277 * @param cacheDataOnRead whether DATA blocks should be cached on read (we always cache INDEX +278 * blocks and BLOOM blocks; this cannot be disabled). +279 * @param inMemory whether blocks should be flagged as in-memory +280 * @param cacheDataOnWrite whether data blocks should be cached on write +281 * @param cacheIndexesOnWrite whether index blocks should be cached on write +282 * @param cacheBloomsOnWrite whether blooms should be cached on write +283 * @param evictOnClose whether blocks should be evicted when HFile is closed +284 * @param cacheDataCompressed whether to store blocks as compressed in the cache +285 * @param prefetchOnOpen whether to prefetch blocks upon open +286 * @param cacheDataInL1 If more than one cache tier deployed, if true, cache this column families +287 * data blocks up in the L1 tier. +288 */ +289 CacheConfig(final BlockCache blockCache, +290 final boolean cacheDataOnRead, final boolean inMemory, +291 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite, +292 final boolean cacheBloomsOnWrite, final boolean evictOnClose, +293 final boolean cacheDataCompressed, final boolean prefetchOnOpen, +294 final boolean cacheDataInL1, final boolean dropBehindCompaction) { +295 this.blockCache = blockCache; +296 this.cacheDataOnRead = cacheDataOnRead; +297 this.inMemory = inMemory; +298 this.cacheDataOnWrite = cacheDataOnWrite; +299 this.cacheIndexesOnWrite = cacheIndexesOnWrite; +300 this.cacheBloomsOnWrite = cacheBloomsOnWrite; +301 this.evictOnClose = evictOnClose; +302 this.cacheDataCompressed = cacheDataCompressed; +303 this.prefetchOnOpen = prefetchOnOpen; +304 this.cacheDataInL1 = cacheDataInL1; +305 this.dropBehindCompaction = dropBehindCompaction; +306 LOG.info(this); +307 } +308 +309 /** +310 * Constructs a cache configuration copied from the specified configuration. +311 * @param cacheConf +312 */ +313 public CacheConfig(CacheConfig cacheConf) { +314 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory, +315 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite, +316 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose, +317 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen, +318 cacheConf.cacheDataInL1, cacheConf.dropBehindCompaction); +319 } +320 +321 /** +322 * Checks whether the block cache is enabled. 323 */ -324 public BlockCache getBlockCache() { -325 return this.blockCache; +324 public boolean isBlockCacheEnabled() { +325 return this.blockCache != null; 326 } 327 328 /** -329 * Returns whether the DATA blocks of this HFile should be cached on read or not (we always -330 * cache the meta blocks, the INDEX and BLOOM blocks). -331 * @return true if blocks should be cached on read, false if not -332 */ -333 public boolean shouldCacheDataOnRead() { -334 return isBlockCacheEnabled() && cacheDataOnRead; -335 } -336 -337 public boolean shouldDropBehindCompaction() { -338 return dropBehindCompaction; -339 } -340 -341 /** -342 * Should we cache a block of a particular category? We always cache -343 * important blocks such as index blocks, as long as the block cache is -344 * available. -345 */ -346 public boolean shouldCacheBlockOnRead(BlockCategory category) { -347 return isBlockCacheEnabled() -348 && (cacheDataOnRead || -349 category == BlockCategory.INDEX || -350 category == BlockCategory.BLOOM || -351 (prefetchOnOpen && -352 (category != BlockCategory.META && -353 category != BlockCategory.UNKNOWN))); -354 } -355 -356 /** -357 * @return true if blocks in this file should be flagged as in-memory -358 */ -359 public boolean isInMemory() { -360 return isBlockCacheEnabled() && this.inMemory; -361 } -362 -363 /** -364 * @return True if cache data blocks in L1 tier (if more than one tier in block cache deploy). -365 */ -366 public boolean isCacheDataInL1() { -367 return isBlockCacheEnabled() && this.cacheDataInL1; -368 } -369 -370 /** -371 * @return true if data blocks should be written to the cache when an HFile is -372 * written, false if not +329 * Returns the block cache. +330 * @return the block cache, or null if caching is completely disabled +331 */ +332 public BlockCache getBlockCache() { +333 return this.blockCache; +334 } +335 +336 /** +337 * Returns whether the DATA blocks of this HFile should be cached on read or not (we always +338 * cache the meta blocks, the INDEX and BLOOM blocks). +339 * @return true if blocks should be cached on read, false if not +340 */ +341 public boolean shouldCacheDataOnRead() { +342 return isBlockCacheEnabled() && cacheDataOnRead; +343 } +344 +345 public boolean shouldDropBehindCompaction() { +346 return dropBehindCompaction; +347 } +348 +349 /** +350 * Should we cache a block of a particular category? We always cache +351 * important blocks such as index blocks, as long as the block cache is +352 * available. +353 */ +354 public boolean shouldCacheBlockOnRead(BlockCategory category) { +355 return isBlockCacheEnabled() +356 && (cacheDataOnRead || +357 category == BlockCategory.INDEX || +358 category == BlockCategory.BLOOM || +359 (prefetchOnOpen && +360 (category != BlockCategory.META && +361 category != BlockCategory.UNKNOWN))); +362 } +363 +364 /** +365 * @return true if blocks in this file should be flagged as in-memory +366 */ +367 public boolean isInMemory() { +368 return isBlockCacheEnabled() && this.inMemory; +369 } +370 +371 /** +372 * @return True if cache data blocks in L1 tier (if more than one tier in block cache deploy). 373 */ -374 public boolean shouldCacheDataOnWrite() { -375 return isBlockCacheEnabled() && this.cacheDataOnWrite; +374 public boolean isCacheDataInL1() { +375 return isBlockCacheEnabled() && this.cacheDataInL1; 376 } 377 378 /** -379 * Only used for testing. -380 * @param cacheDataOnWrite whether data blocks should be written to the cache -381 * when an HFile is written -382 */ -383 @VisibleForTesting -384 public void setCacheDataOnWrite(boolean cacheDataOnWrite) { -385 this.cacheDataOnWrite = cacheDataOnWrite; -386 } -387 -388 /** -389 * Only used for testing. -390 * @param cacheDataInL1 Whether to cache data blocks up in l1 (if a multi-tier cache -391 * implementation). -392 */ -393 @VisibleForTesting -394 public void setCacheDataInL1(boolean cacheDataInL1) { -395 this.cacheDataInL1 = cacheDataInL1; -396 } -397 -398 /** -399 * @return true if index blocks should be written to the cache when an HFile -400 * is written, false if not -401 */ -402 public boolean shouldCacheIndexesOnWrite() { -403 return isBlockCacheEnabled() && this.cacheIndexesOnWrite; +379 * @return true if data blocks should be written to the cache when an HFile is +380 * written, false if not +381 */ +382 public boolean shouldCacheDataOnWrite() { +383 return isBlockCacheEnabled() && this.cacheDataOnWrite; +384 } +385 +386 /** +387 * Only used for testing. +388 * @param cacheDataOnWrite whether data blocks should be written to the cache +389 * when an HFile is written +390 */ +391 @VisibleForTesting +392 public void setCacheDataOnWrite(boolean cacheDataOnWrite) { +393 this.cacheDataOnWrite = cacheDataOnWrite; +394 } +395 +396 /** +397 * Only used for testing. +398 * @param cacheDataInL1 Whether to cache data blocks up in l1 (if a multi-tier cache +399 * implementation). +400 */ +401 @VisibleForTesting +402 public void setCacheDataInL1(boolean cacheDataInL1) { +403 this.cacheDataInL1 = cacheDataInL1; 404 } 405 406 /** -407 * @return true if bloom blocks should be written to the cache when an HFile +407 * @return true if index blocks should be written to the cache when an HFile 408 * is written, false if not 409 */ -410 public boolean shouldCacheBloomsOnWrite() { -411 return isBlockCacheEnabled() && this.cacheBloomsOnWrite; +410 public boolean shouldCacheIndexesOnWrite() { +411 return isBlockCacheEnabled() && this.cacheIndexesOnWrite; 412 } 413 414 /** -415 * @return true if blocks should be evicted from the cache when an HFile -416 * reader is closed, false if not +415 * @return true if bloom blocks should be written to the cache when an HFile +416 * is written, false if not 417 */ -418 public boolean shouldEvictOnClose() { -419 return isBlockCacheEnabled() && this.evictOnClose; +418 public boolean shouldCacheBloomsOnWrite() { +419 return isBlockCacheEnabled() && this.cacheBloomsOnWrite; 420 } 421 422 /** -423 * Only used for testing. -424 * @param evictOnClose whether blocks should be evicted from the cache when an -425 * HFile reader is closed -426 */ -427 public void setEvictOnClose(boolean evictOnClose) { -428 this.evictOnClose = evictOnClose; -429 } -430 -431 /** -432 * @return true if data blocks should be compressed in the cache, false if not -433 */ -434 public boolean shouldCacheDataCompressed() { -435 return isBlockCacheEnabled() && this.cacheDataCompressed; -436 } -437 -438 /** -439 * @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise -440 */ -441 public boolean shouldCacheCompressed(BlockCategory category) { -442 if (!isBlockCacheEnabled()) return false; -443 switch (category) { -444 case DATA: -445 return this.cacheDataCompressed; -446 default: -447 return false; -448 } -449 } -450 -451 /** -452 * @return true if blocks should be prefetched into the cache on open, false if not -453 */ -454 public boolean shouldPrefetchOnOpen() { -455 return isBlockCacheEnabled() && this.prefetchOnOpen; -456 } -457 -458 /** -459 * Return true if we may find this type of block in block cache. -460 * <p> -461 * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we -462 * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in -463 * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled} -464 * configuration. -465 */ -466 public boolean shouldReadBlockFromCache(BlockType blockType) { -467 if (!isBlockCacheEnabled()) { -468 return false; -469 } -470 if (cacheDataOnRead) { -471 return true; -472 } -473 if (prefetchOnOpen) { -474 return true; -475 } -476 if (cacheDataOnWrite) { -477 return true; -478 } -479 if (blockType == null) { -480 return true; -481 } -482 if (blockType.getCategory() == BlockCategory.BLOOM || -483 blockType.getCategory() == BlockCategory.INDEX) { -484 return true; -485 } -486 return false; -487 } -488 -489 /** -490 * If we make sure the block could not be cached, we will not acquire the lock -491 * otherwise we will acquire lock -492 */ -493 public boolean shouldLockOnCacheMiss(BlockType blockType) { -494 if (blockType == null) { -495 return true; -496 } -497 return shouldCacheBlockOnRead(blockType.getCategory()); -498 } -499 -500 @Override -501 public String toString() { -502 if (!isBlockCacheEnabled()) { -503 return "CacheConfig:disabled"; +423 * @return true if blocks should be evicted from the cache when an HFile +424 * reader is closed, false if not +425 */ +426 public boolean shouldEvictOnClose() { +427 return isBlockCacheEnabled() && this.evictOnClose; +428 } +429 +430 /** +431 * Only used for testing. +432 * @param evictOnClose whether blocks should be evicted from the cache when an +433 * HFile reader is closed +434 */ +435 public void setEvictOnClose(boolean evictOnClose) { +436 this.evictOnClose = evictOnClose; +437 } +438 +439 /** +440 * @return true if data blocks should be compressed in the cache, false if not +441 */ +442 public boolean shouldCacheDataCompressed() { +443 return isBlockCacheEnabled() && this.cacheDataOnRead && this.cacheDataCompressed; +444 } +445 +446 /** +447 * @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise +448 */ +449 public boolean shouldCacheCompressed(BlockCategory category) { +450 if (!isBlockCacheEnabled()) return false; +451 switch (category) { +452 case DATA: +453 return this.cacheDataOnRead && this.cacheDataCompressed; +454 default: +455 return false; +456 } +457 } +458 +459 /** +460 * @return true if blocks should be prefetched into the cache on open, false if not +461 */ +462 public boolean shouldPrefetchOnOpen() { +463 return isBlockCacheEnabled() && this.prefetchOnOpen; +464 } +465 +466 /** +467 * Return true if we may find this type of block in block cache. +468 * <p> +469 * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we +470 * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in +471 * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled} +472 * configuration. +473 */ +474 public boolean shouldReadBlockFromCache(BlockType blockType) { +475 if (!isBlockCacheEnabled()) { +476 return false; +477 } +478 if (cacheDataOnRead) { +479 return true; +480 } +481 if (prefetchOnOpen) { +482 return true; +483 } +484 if (cacheDataOnWrite) { +485 return true; +486 } +487 if (blockType == null) { +488 return true; +489 } +490 if (blockType.getCategory() == BlockCategory.BLOOM || +491 blockType.getCategory() == BlockCategory.INDEX) { +492 return true; +493 } +494 return false; +495 } +496 +497 /** +498 * If we make sure the block could not be cached, we will not acquire the lock +499 * otherwise we will acquire lock +500 */ +501 public boolean shouldLockOnCacheMiss(BlockType blockType) { +502 if (blockType == null) { +503 return true; 504