Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A23AF18AD2 for ; Mon, 1 Feb 2016 16:56:59 +0000 (UTC) Received: (qmail 27751 invoked by uid 500); 1 Feb 2016 16:56:58 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 27662 invoked by uid 500); 1 Feb 2016 16:56:58 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 27247 invoked by uid 99); 1 Feb 2016 16:56:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Feb 2016 16:56:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D42F5E0942; Mon, 1 Feb 2016 16:56:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Mon, 01 Feb 2016 16:57:02 -0000 Message-Id: In-Reply-To: <14392ad1c5b44bc388d1159e43dc3416@git.apache.org> References: <14392ad1c5b44bc388d1159e43dc3416@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [06/51] [partial] hbase-site git commit: Published site at 2cc48e039d1f800832ac8880bbc820982e0ac8a5. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7e5a8ce/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.NotSeekedException.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.NotSeekedException.html b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.NotSeekedException.html index e8248d4..c36648a 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.NotSeekedException.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.NotSeekedException.html @@ -59,1847 +59,1848 @@ 051import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext; 052import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; 053import org.apache.hadoop.hbase.nio.ByteBuff; -054import org.apache.hadoop.hbase.security.EncryptionUtil; -055import org.apache.hadoop.hbase.util.ByteBufferUtils; -056import org.apache.hadoop.hbase.util.Bytes; -057import org.apache.hadoop.hbase.util.IdLock; -058import org.apache.hadoop.hbase.util.ObjectIntPair; -059import org.apache.hadoop.io.WritableUtils; -060import org.apache.htrace.Trace; -061import org.apache.htrace.TraceScope; -062 -063import com.google.common.annotations.VisibleForTesting; -064 -065/** -066 * Implementation that can handle all hfile versions of {@link HFile.Reader}. -067 */ -068@InterfaceAudience.Private -069@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD") -070public class HFileReaderImpl implements HFile.Reader, Configurable { -071 // This class is HFileReaderV3 + HFileReaderV2 + AbstractHFileReader all squashed together into -072 // one file. Ditto for all the HFileReader.ScannerV? implementations. I was running up against -073 // the MaxInlineLevel limit because too many tiers involved reading from an hfile. Was also hard -074 // to navigate the source code when so many classes participating in read. -075 private static final Log LOG = LogFactory.getLog(HFileReaderImpl.class); -076 -077 /** Data block index reader keeping the root data index in memory */ -078 private HFileBlockIndex.CellBasedKeyBlockIndexReader dataBlockIndexReader; -079 -080 /** Meta block index reader -- always single level */ -081 private HFileBlockIndex.ByteArrayKeyBlockIndexReader metaBlockIndexReader; -082 -083 private final FixedFileTrailer trailer; -084 -085 /** Filled when we read in the trailer. */ -086 private final Compression.Algorithm compressAlgo; -087 -088 private boolean isPrimaryReplicaReader; -089 -090 /** -091 * What kind of data block encoding should be used while reading, writing, -092 * and handling cache. -093 */ -094 private HFileDataBlockEncoder dataBlockEncoder = NoOpDataBlockEncoder.INSTANCE; -095 -096 /** Last key in the file. Filled in when we read in the file info */ -097 private Cell lastKeyCell = null; -098 -099 /** Average key length read from file info */ -100 private int avgKeyLen = -1; -101 -102 /** Average value length read from file info */ -103 private int avgValueLen = -1; -104 -105 /** Key comparator */ -106 private CellComparator comparator = CellComparator.COMPARATOR; -107 -108 /** Size of this file. */ -109 private final long fileSize; -110 -111 /** Block cache configuration. */ -112 private final CacheConfig cacheConf; -113 -114 /** Path of file */ -115 private final Path path; -116 -117 /** File name to be used for block names */ -118 private final String name; -119 -120 private FileInfo fileInfo; -121 -122 private Configuration conf; -123 -124 private HFileContext hfileContext; -125 -126 /** Filesystem-level block reader. */ -127 private HFileBlock.FSReader fsBlockReader; -128 -129 /** -130 * A "sparse lock" implementation allowing to lock on a particular block -131 * identified by offset. The purpose of this is to avoid two clients loading -132 * the same block, and have all but one client wait to get the block from the -133 * cache. -134 */ -135 private IdLock offsetLock = new IdLock(); -136 -137 /** -138 * Blocks read from the load-on-open section, excluding data root index, meta -139 * index, and file info. -140 */ -141 private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>(); -142 -143 /** Minimum minor version supported by this HFile format */ -144 static final int MIN_MINOR_VERSION = 0; -145 -146 /** Maximum minor version supported by this HFile format */ -147 // We went to version 2 when we moved to pb'ing fileinfo and the trailer on -148 // the file. This version can read Writables version 1. -149 static final int MAX_MINOR_VERSION = 3; -150 -151 /** -152 * We can read files whose major version is v2 IFF their minor version is at least 3. -153 */ -154 private static final int MIN_V2_MINOR_VERSION_WITH_PB = 3; -155 -156 /** Minor versions starting with this number have faked index key */ -157 static final int MINOR_VERSION_WITH_FAKED_KEY = 3; -158 -159 /** -160 * Opens a HFile. You must load the index before you can use it by calling -161 * {@link #loadFileInfo()}. -162 * @param path -163 * Path to HFile. -164 * @param trailer -165 * File trailer. -166 * @param fsdis -167 * input stream. -168 * @param fileSize -169 * Length of the stream. -170 * @param cacheConf -171 * Cache configuration. -172 * @param hfs -173 * The file system. -174 * @param conf -175 * Configuration -176 */ -177 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD") -178 public HFileReaderImpl(final Path path, FixedFileTrailer trailer, -179 final FSDataInputStreamWrapper fsdis, -180 final long fileSize, final CacheConfig cacheConf, final HFileSystem hfs, -181 final Configuration conf) -182 throws IOException { -183 this.trailer = trailer; -184 this.compressAlgo = trailer.getCompressionCodec(); -185 this.cacheConf = cacheConf; -186 this.fileSize = fileSize; -187 this.path = path; -188 this.name = path.getName(); -189 this.conf = conf; -190 checkFileVersion(); -191 this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer); -192 this.fsBlockReader = new HFileBlock.FSReaderImpl(fsdis, fileSize, hfs, path, hfileContext); -193 -194 // Comparator class name is stored in the trailer in version 2. -195 comparator = trailer.createComparator(); -196 dataBlockIndexReader = new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator, -197 trailer.getNumDataIndexLevels(), this); -198 metaBlockIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1); -199 -200 // Parse load-on-open data. -201 -202 HFileBlock.BlockIterator blockIter = fsBlockReader.blockRange( -203 trailer.getLoadOnOpenDataOffset(), -204 fileSize - trailer.getTrailerSize()); -205 -206 // Data index. We also read statistics about the block index written after -207 // the root level. -208 dataBlockIndexReader.readMultiLevelIndexRoot( -209 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), -210 trailer.getDataIndexCount()); -211 -212 // Meta index. -213 metaBlockIndexReader.readRootIndex( -214 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), -215 trailer.getMetaIndexCount()); -216 -217 // File info -218 fileInfo = new FileInfo(); -219 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream()); -220 byte[] creationTimeBytes = fileInfo.get(FileInfo.CREATE_TIME_TS); -221 this.hfileContext.setFileCreateTime(creationTimeBytes == null? 0: -222 Bytes.toLong(creationTimeBytes)); -223 if (fileInfo.get(FileInfo.LASTKEY) != null) { -224 lastKeyCell = new KeyValue.KeyOnlyKeyValue(fileInfo.get(FileInfo.LASTKEY)); -225 } -226 avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN)); -227 avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN)); -228 byte [] keyValueFormatVersion = fileInfo.get(HFileWriterImpl.KEY_VALUE_VERSION); -229 includesMemstoreTS = keyValueFormatVersion != null && -230 Bytes.toInt(keyValueFormatVersion) == HFileWriterImpl.KEY_VALUE_VER_WITH_MEMSTORE; -231 fsBlockReader.setIncludesMemstoreTS(includesMemstoreTS); -232 if (includesMemstoreTS) { -233 decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterImpl.MAX_MEMSTORE_TS_KEY)) > 0; -234 } -235 -236 // Read data block encoding algorithm name from file info. -237 dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo); -238 fsBlockReader.setDataBlockEncoder(dataBlockEncoder); -239 -240 // Store all other load-on-open blocks for further consumption. -241 HFileBlock b; -242 while ((b = blockIter.nextBlock()) != null) { -243 loadOnOpenBlocks.add(b); -244 } -245 -246 // Prefetch file blocks upon open if requested -247 if (cacheConf.shouldPrefetchOnOpen()) { -248 PrefetchExecutor.request(path, new Runnable() { -249 public void run() { -250 try { -251 long offset = 0; -252 long end = fileSize - getTrailer().getTrailerSize(); -253 HFileBlock prevBlock = null; -254 while (offset < end) { -255 if (Thread.interrupted()) { -256 break; -257 } -258 long onDiskSize = -1; -259 if (prevBlock != null) { -260 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader(); -261 } -262 HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false, -263 null, null); -264 // Need not update the current block. Ideally here the readBlock won't find the -265 // block in cache. We call this readBlock so that block data is read from FS and -266 // cached in BC. So there is no reference count increment that happens here. -267 // The return will ideally be a noop because the block is not of MemoryType SHARED. -268 returnBlock(block); -269 prevBlock = block; -270 offset += block.getOnDiskSizeWithHeader(); -271 } -272 } catch (IOException e) { -273 // IOExceptions are probably due to region closes (relocation, etc.) -274 if (LOG.isTraceEnabled()) { -275 LOG.trace("Exception encountered while prefetching " + path + ":", e); -276 } -277 } catch (Exception e) { -278 // Other exceptions are interesting -279 LOG.warn("Exception encountered while prefetching " + path + ":", e); -280 } finally { -281 PrefetchExecutor.complete(path); -282 } -283 } -284 }); -285 } -286 -287 byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN); -288 // max tag length is not present in the HFile means tags were not at all written to file. -289 if (tmp != null) { -290 hfileContext.setIncludesTags(true); -291 tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED); -292 if (tmp != null && Bytes.toBoolean(tmp)) { -293 hfileContext.setCompressTags(true); -294 } -295 } -296 } -297 -298 /** -299 * File version check is a little sloppy. We read v3 files but can also read v2 files if their -300 * content has been pb'd; files written with 0.98. -301 */ -302 private void checkFileVersion() { -303 int majorVersion = trailer.getMajorVersion(); -304 if (majorVersion == getMajorVersion()) return; -305 int minorVersion = trailer.getMinorVersion(); -306 if (majorVersion == 2 && minorVersion >= MIN_V2_MINOR_VERSION_WITH_PB) return; -307 // We can read v3 or v2 versions of hfile. -308 throw new IllegalArgumentException("Invalid HFile version: major=" + -309 trailer.getMajorVersion() + ", minor=" + trailer.getMinorVersion() + ": expected at least " + -310 "major=2 and minor=" + MAX_MINOR_VERSION); -311 } -312 -313 @SuppressWarnings("serial") -314 public static class BlockIndexNotLoadedException extends IllegalStateException { -315 public BlockIndexNotLoadedException() { -316 // Add a message in case anyone relies on it as opposed to class name. -317 super("Block index not loaded"); -318 } -319 } -320 -321 private String toStringFirstKey() { -322 if(getFirstKey() == null) -323 return null; -324 return CellUtil.getCellKeyAsString(getFirstKey()); -325 } -326 -327 private String toStringLastKey() { -328 return CellUtil.toString(getLastKey(), false); -329 } -330 -331 @Override -332 public String toString() { -333 return "reader=" + path.toString() + -334 (!isFileInfoLoaded()? "": -335 ", compression=" + compressAlgo.getName() + -336 ", cacheConf=" + cacheConf + -337 ", firstKey=" + toStringFirstKey() + -338 ", lastKey=" + toStringLastKey()) + -339 ", avgKeyLen=" + avgKeyLen + -340 ", avgValueLen=" + avgValueLen + -341 ", entries=" + trailer.getEntryCount() + -342 ", length=" + fileSize; -343 } -344 -345 @Override -346 public long length() { -347 return fileSize; -348 } -349 -350 @Override -351 public void returnBlock(HFileBlock block) { -352 BlockCache blockCache = this.cacheConf.getBlockCache(); -353 if (blockCache != null && block != null) { -354 BlockCacheKey cacheKey = new BlockCacheKey(this.getFileContext().getHFileName(), -355 block.getOffset(), this.isPrimaryReplicaReader()); -356 blockCache.returnBlock(cacheKey, block); -357 } -358 } -359 /** -360 * @return the first key in the file. May be null if file has no entries. Note -361 * that this is not the first row key, but rather the byte form of the -362 * first KeyValue. -363 */ -364 @Override -365 public Cell getFirstKey() { -366 if (dataBlockIndexReader == null) { -367 throw new BlockIndexNotLoadedException(); -368 } -369 return dataBlockIndexReader.isEmpty() ? null -370 : dataBlockIndexReader.getRootBlockKey(0); -371 } -372 -373 /** -374 * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's -375 * patch goes in to eliminate {@link KeyValue} here. -376 * -377 * @return the first row key, or null if the file is empty. -378 */ -379 @Override -380 public byte[] getFirstRowKey() { -381 Cell firstKey = getFirstKey(); -382 // We have to copy the row part to form the row key alone -383 return firstKey == null? null: CellUtil.cloneRow(firstKey); -384 } -385 -386 /** -387 * TODO left from {@link HFile} version 1: move this to StoreFile after -388 * Ryan's patch goes in to eliminate {@link KeyValue} here. -389 * -390 * @return the last row key, or null if the file is empty. -391 */ -392 @Override -393 public byte[] getLastRowKey() { -394 Cell lastKey = getLastKey(); -395 return lastKey == null? null: CellUtil.cloneRow(lastKey); -396 } -397 -398 /** @return number of KV entries in this HFile */ -399 @Override -400 public long getEntries() { -401 return trailer.getEntryCount(); -402 } -403 -404 /** @return comparator */ -405 @Override -406 public CellComparator getComparator() { -407 return comparator; -408 } -409 -410 /** @return compression algorithm */ -411 @Override -412 public Compression.Algorithm getCompressionAlgorithm() { -413 return compressAlgo; -414 } -415 -416 /** -417 * @return the total heap size of data and meta block indexes in bytes. Does -418 * not take into account non-root blocks of a multilevel data index. -419 */ -420 public long indexSize() { -421 return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0) -422 + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize() -423 : 0); -424 } -425 -426 @Override -427 public String getName() { -428 return name; -429 } -430 -431 @Override -432 public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() { -433 return dataBlockIndexReader; -434 } -435 -436 @Override -437 public FixedFileTrailer getTrailer() { -438 return trailer; -439 } -440 -441 @Override -442 public boolean isPrimaryReplicaReader() { -443 return isPrimaryReplicaReader; -444 } -445 -446 @Override -447 public void setPrimaryReplicaReader(boolean isPrimaryReplicaReader) { -448 this.isPrimaryReplicaReader = isPrimaryReplicaReader; -449 } -450 -451 @Override -452 public FileInfo loadFileInfo() throws IOException { -453 return fileInfo; -454 } -455 -456 /** -457 * An exception thrown when an operation requiring a scanner to be seeked -458 * is invoked on a scanner that is not seeked. -459 */ -460 @SuppressWarnings("serial") -461 public static class NotSeekedException extends IllegalStateException { -462 public NotSeekedException() { -463 super("Not seeked to a key/value"); -464 } -465 } -466 -467 protected static class HFileScannerImpl implements HFileScanner { -468 private ByteBuff blockBuffer; -469 protected final boolean cacheBlocks; -470 protected final boolean pread; -471 protected final boolean isCompaction; -472 private int currKeyLen; -473 private int currValueLen; -474 private int currMemstoreTSLen; -475 private long currMemstoreTS; -476 // Updated but never read? -477 protected AtomicInteger blockFetches = new AtomicInteger(0); -478 protected final HFile.Reader reader; -479 private int currTagsLen; -480 // buffer backed keyonlyKV -481 private ByteBufferedKeyOnlyKeyValue bufBackedKeyOnlyKv = new ByteBufferedKeyOnlyKeyValue(); -482 // A pair for reusing in blockSeek() so that we don't garbage lot of objects -483 final ObjectIntPair<ByteBuffer> pair = new ObjectIntPair<ByteBuffer>(); -484 -485 /** -486 * The next indexed key is to keep track of the indexed key of the next data block. -487 * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the -488 * current data block is the last data block. -489 * -490 * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet. -491 */ -492 protected Cell nextIndexedKey; -493 // Current block being used -494 protected HFileBlock curBlock; -495 // Previous blocks that were used in the course of the read -496 protected final ArrayList<HFileBlock> prevBlocks = new ArrayList<HFileBlock>(); -497 -498 public HFileScannerImpl(final HFile.Reader reader, final boolean cacheBlocks, -499 final boolean pread, final boolean isCompaction) { -500 this.reader = reader; -501 this.cacheBlocks = cacheBlocks; -502 this.pread = pread; -503 this.isCompaction = isCompaction; -504 } -505 -506 void updateCurrBlockRef(HFileBlock block) { -507 if (block != null && this.curBlock != null && -508 block.getOffset() == this.curBlock.getOffset()) { -509 return; -510 } -511 // We don't have to keep ref to EXCLUSIVE type of block -512 if (this.curBlock != null && this.curBlock.usesSharedMemory()) { -513 prevBlocks.add(this.curBlock); -514 } -515 this.curBlock = block; -516 } -517 -518 void reset() { -519 // We don't have to keep ref to EXCLUSIVE type of block -520 if (this.curBlock != null && this.curBlock.usesSharedMemory()) { -521 this.prevBlocks.add(this.curBlock); -522 } -523 this.curBlock = null; -524 } -525 -526 private void returnBlockToCache(HFileBlock block) { -527 if (LOG.isTraceEnabled()) { -528 LOG.trace("Returning the block : " + block); -529 } -530 this.reader.returnBlock(block); -531 } -532 -533 private void returnBlocks(boolean returnAll) { -534 for (int i = 0; i < this.prevBlocks.size(); i++) { -535 returnBlockToCache(this.prevBlocks.get(i)); -536 } -537 this.prevBlocks.clear(); -538 if (returnAll && this.curBlock != null) { -539 returnBlockToCache(this.curBlock); -540 this.curBlock = null; -541 } -542 } -543 @Override -544 public boolean isSeeked(){ -545 return blockBuffer != null; -546 } -547 -548 @Override -549 public String toString() { -550 return "HFileScanner for reader " + String.valueOf(getReader()); -551 } -552 -553 protected void assertSeeked() { -554 if (!isSeeked()) -555 throw new NotSeekedException(); -556 } -557 -558 @Override -559 public HFile.Reader getReader() { -560 return reader; -561 } -562 -563 protected int getCellBufSize() { -564 int kvBufSize = KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen; -565 if (this.reader.getFileContext().isIncludesTags()) { -566 kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen; -567 } -568 return kvBufSize; -569 } -570 -571 @Override -572 public void close() { -573 this.returnBlocks(true); -574 } -575 -576 protected int getCurCellSize() { -577 int curCellSize = KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen -578 + currMemstoreTSLen; -579 if (this.reader.getFileContext().isIncludesTags()) { -580 curCellSize += Bytes.SIZEOF_SHORT + currTagsLen; -581 } -582 return curCellSize; -583 } -584 -585 protected void readKeyValueLen() { -586 // This is a hot method. We go out of our way to make this method short so it can be -587 // inlined and is not too big to compile. We also manage position in ByteBuffer ourselves -588 // because it is faster than going via range-checked ByteBuffer methods or going through a -589 // byte buffer array a byte at a time. -590 // Get a long at a time rather than read two individual ints. In micro-benchmarking, even -591 // with the extra bit-fiddling, this is order-of-magnitude faster than getting two ints. -592 // Trying to imitate what was done - need to profile if this is better or -593 // earlier way is better by doing mark and reset? -594 // But ensure that you read long instead of two ints -595 long ll = blockBuffer.getLongAfterPosition(0); -596 // Read top half as an int of key length and bottom int as value length -597 this.currKeyLen = (int)(ll >> Integer.SIZE); -598 this.currValueLen = (int)(Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll); -599 checkKeyValueLen(); -600 // Move position past the key and value lengths and then beyond the key and value -601 int p = (Bytes.SIZEOF_LONG + currKeyLen + currValueLen); -602 if (reader.getFileContext().isIncludesTags()) { -603 // Tags length is a short. -604 this.currTagsLen = blockBuffer.getShortAfterPosition(p); -605 checkTagsLen(); -606 p += (Bytes.SIZEOF_SHORT + currTagsLen); -607 } -608 readMvccVersion(p); -609 } -610 -611 private final void checkTagsLen() { -612 if (checkLen(this.currTagsLen)) { -613 throw new IllegalStateException("Invalid currTagsLen " + this.currTagsLen + -614 ". Block offset: " + curBlock.getOffset() + ", block length: " + -615 this.blockBuffer.limit() + -616 ", position: " + this.blockBuffer.position() + " (without header)."); -617 } -618 } -619 -620 /** -621 * Read mvcc. Does checks to see if we even need to read the mvcc at all. -622 * @param offsetFromPos -623 */ -624 protected void readMvccVersion(final int offsetFromPos) { -625 // See if we even need to decode mvcc. -626 if (!this.reader.shouldIncludeMemstoreTS()) return; -627 if (!this.reader.isDecodeMemstoreTS()) { -628 currMemstoreTS = 0; -629 currMemstoreTSLen = 1; -630 return; -631 } -632 _readMvccVersion(offsetFromPos); -633 } -634 -635 /** -636 * Actually do the mvcc read. Does no checks. -637 * @param offsetFromPos -638 */ -639 private void _readMvccVersion(int offsetFromPos) { -640 // This is Bytes#bytesToVint inlined so can save a few instructions in this hot method; i.e. -641 // previous if one-byte vint, we'd redo the vint call to find int size. -642 // Also the method is kept small so can be inlined. -643 byte firstByte = blockBuffer.getByteAfterPosition(offsetFromPos); -644 int len = WritableUtils.decodeVIntSize(firstByte); -645 if (len == 1) { -646 this.currMemstoreTS = firstByte; -647 } else { -648 int remaining = len -1; -649 long i = 0; -650 offsetFromPos++; -651 if (remaining >= Bytes.SIZEOF_INT) { -652 i = blockBuffer.getIntAfterPosition(offsetFromPos); -653 remaining -= Bytes.SIZEOF_INT; -654 offsetFromPos += Bytes.SIZEOF_INT; -655 } -656 if (remaining >= Bytes.SIZEOF_SHORT) { -657 short s = blockBuffer.getShortAfterPosition(offsetFromPos); -658 i = i << 16; -659 i = i | (s & 0xFFFF); -660 remaining -= Bytes.SIZEOF_SHORT; -661 offsetFromPos += Bytes.SIZEOF_SHORT; -662 } -663 for (int idx = 0; idx < remaining; idx++) { -664 byte b = blockBuffer.getByteAfterPosition(offsetFromPos + idx); -665 i = i << 8; -666 i = i | (b & 0xFF); -667 } -668 currMemstoreTS = (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); -669 } -670 this.currMemstoreTSLen = len; -671 } -672 -673 /** -674 * Within a loaded block, seek looking for the last key that is smaller than -675 * (or equal to?) the key we are interested in. -676 * A note on the seekBefore: if you have seekBefore = true, AND the first -677 * key in the block = key, then you'll get thrown exceptions. The caller has -678 * to check for that case and load the previous block as appropriate. -679 * @param key -680 * the key to find -681 * @param seekBefore -682 * find the key before the given key in case of exact match. -683 * @return 0 in case of an exact key match, 1 in case of an inexact match, -684 * -2 in case of an inexact match and furthermore, the input key -685 * less than the first key of current block(e.g. using a faked index -686 * key) -687 */ -688 protected int blockSeek(Cell key, boolean seekBefore) { -689 int klen, vlen, tlen = 0; -690 int lastKeyValueSize = -1; -691 int offsetFromPos; -692 do { -693 offsetFromPos = 0; -694 // Better to ensure that we use the BB Utils here -695 long ll = blockBuffer.getLongAfterPosition(offsetFromPos); -696 klen = (int)(ll >> Integer.SIZE); -697 vlen = (int)(Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll); -698 if (klen < 0 || vlen < 0 || klen > blockBuffer.limit() -699 || vlen > blockBuffer.limit()) { -700 throw new IllegalStateException("Invalid klen " + klen + " or vlen " -701 + vlen + ". Block offset: " -702 + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: " -703 + blockBuffer.position() + " (without header)."); -704 } -705 offsetFromPos += Bytes.SIZEOF_LONG; -706 blockBuffer.asSubByteBuffer(blockBuffer.position() + offsetFromPos, klen, pair); -707 bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), klen); -708 int comp = reader.getComparator().compareKeyIgnoresMvcc(key, bufBackedKeyOnlyKv); -709 offsetFromPos += klen + vlen; -710 if (this.reader.getFileContext().isIncludesTags()) { -711 // Read short as unsigned, high byte first -712 tlen = ((blockBuffer.getByteAfterPosition(offsetFromPos) & 0xff) << 8) -713 ^ (blockBuffer.getByteAfterPosition(offsetFromPos + 1) & 0xff); -714 if (tlen < 0 || tlen > blockBuffer.limit()) { -715 throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: " -716 + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: " -717 + blockBuffer.position() + " (without header)."); -718 } -719 // add the two bytes read for the tags. -720 offsetFromPos += tlen + (Bytes.SIZEOF_SHORT); -721 } -722 if (this.reader.shouldIncludeMemstoreTS()) { -723 // Directly read the mvcc based on current position -724 readMvccVersion(offsetFromPos); -725 } -726 if (comp == 0) { -727 if (seekBefore) { -728 if (lastKeyValueSize < 0) { -729 throw new IllegalStateException("blockSeek with seekBefore " -730 + "at the first key of the block: key=" + CellUtil.getCellKeyAsString(key) -731 + ", blockOffset=" + curBlock.getOffset() + ", onDiskSize=" -732 + curBlock.getOnDiskSizeWithHeader()); -733 } -734 blockBuffer.moveBack(lastKeyValueSize); -735 readKeyValueLen(); -736 return 1; // non exact match. -737 } -738 currKeyLen = klen; -739 currValueLen = vlen; -740 currTagsLen = tlen; -741 return 0; // indicate exact match -742 } else if (comp < 0) { -743 if (lastKeyValueSize > 0) { -744 blockBuffer.moveBack(lastKeyValueSize); -745 } -746 readKeyValueLen(); -747 if (lastKeyValueSize == -1 && blockBuffer.position() == 0) { -748 return HConstants.INDEX_KEY_MAGIC; -749 } -750 return 1; -751 } -752 // The size of this key/value tuple, including key/value length fields. -753 lastKeyValueSize = klen + vlen + currMemstoreTSLen + KEY_VALUE_LEN_SIZE; -754 // include tag length also if tags included with KV -755 if (reader.getFileContext().isIncludesTags()) { -756 lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT; -757 } -758 blockBuffer.skip(lastKeyValueSize); -759 } while (blockBuffer.hasRemaining()); -760 -761 // Seek to the last key we successfully read. This will happen if this is -762 // the last key/value pair in the file, in which case the following call -763 // to next() has to return false. -764 blockBuffer.moveBack(lastKeyValueSize); -765 readKeyValueLen(); -766 return 1; // didn't exactly find it. -767 } -768 -769 @Override -770 public Cell getNextIndexedKey() { -771 return nextIndexedKey; -772 } -773 -774 @Override -775 public int seekTo(Cell key) throws IOException { -776 return seekTo(key, true); -777 } -778 -779 @Override -780 public int reseekTo(Cell key) throws IOException { -781 int compared; -782 if (isSeeked()) { -783 compared = compareKey(reader.getComparator(), key); -784 if (compared < 1) { -785 // If the required key is less than or equal to current key, then -786 // don't do anything. -787 return compared; -788 } else { -789 // The comparison with no_next_index_key has to be checked -790 if (this.nextIndexedKey != null && -791 (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader -792 .getComparator().compareKeyIgnoresMvcc(key, nextIndexedKey) < 0)) { -793 // The reader shall continue to scan the current data block instead -794 // of querying the -795 // block index as long as it knows the target key is strictly -796 // smaller than -797 // the next indexed key or the current data block is the last data -798 // block. -799 return loadBlockAndSeekToKey(this.curBlock, nextIndexedKey, false, key, -800 false); -801 } -802 -803 } -804 } -805 // Don't rewind on a reseek operation, because reseek implies that we are -806 // always going forward in the file. -807 return seekTo(key, false); -808 } -809 -810 /** -811 * An internal API function. Seek to the given key, optionally rewinding to -812 * the first key of the block before doing the seek. -813 * -814 * @param key - a cell representing the key that we need to fetch -815 * @param rewind whether to rewind to the first key of the block before -816 * doing the seek. If this is false, we are assuming we never go -817 * back, otherwise the result is undefined. -818 * @return -1 if the key is earlier than the first key of the file, -819 * 0 if we are at the given key, 1 if we are past the given key -820 * -2 if the key is earlier than the first key of the file while -821 * using a faked index key -822 * @throws IOException -823 */ -824 public int seekTo(Cell key, boolean rewind) throws IOException { -825 HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader(); -826 BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, curBlock, -827 cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding()); -828 if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) { -829 // This happens if the key e.g. falls before the beginning of the -830 // file. -831 return -1; -832 } -833 return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(), -834 blockWithScanInfo.getNextIndexedKey(), rewind, key, false); -835 } -836 -837 @Override -838 public boolean seekBefore(Cell key) throws IOException { -839 HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, curBlock, -840 cacheBlocks, pread, isCompaction, reader.getEffectiveEncodingInCache(isCompaction)); -841 if (seekToBlock == null) { -842 return false; -843 } -844 Cell firstKey = getFirstKeyCellInBlock(seekToBlock); -845 if (reader.getComparator() -846 .compareKeyIgnoresMvcc(firstKey, key) >= 0) { -847 long previousBlockOffset = seekToBlock.getPrevBlockOffset(); -848 // The key we are interested in -849 if (previousBlockOffset == -1) { -850 // we have a 'problem', the key we want is the first of the file. -851 return false; -852 } -853 -854 // The first key in the current block 'seekToBlock' is greater than the given -855 // seekBefore key. We will go ahead by reading the next block that satisfies the -856 // given key. Return the current block before reading the next one. -857 reader.returnBlock(seekToBlock); -858 // It is important that we compute and pass onDiskSize to the block -859 // reader so that it does not have to read the header separately to -860 // figure out the size. Currently, we do not have a way to do this -861 // correctly in the general case however. -862 // TODO: See https://issues.apache.org/jira/browse/HBASE-14576 -863 int prevBlockSize = -1; -864 seekToBlock = reader.readBlock(previousBlockOffset, -865 prevBlockSize, cacheBlocks, -866 pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding()); -867 // TODO shortcut: seek forward in this block to the last key of the -868 // block. -869 } -870 loadBlockAndSeekToKey(seekToBlock, firstKey, true, key, true); -871 return true; -872 } -873 -874 /** -875 * Scans blocks in the "scanned" section of the {@link HFile} until the next -876 * data block is found. -877 * -878 * @return the next block, or null if there are no more data blocks -879 * @throws IOException -880 */ -881 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH", -882 justification="Yeah, unnecessary null check; could do w/ clean up") -883 protected HFileBlock readNextDataBlock() throws IOException { -884 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); -885 if (curBlock == null) -886 return null; -887 -888 HFileBlock block = this.curBlock; -889 -890 do { -891 if (block.getOffset() >= lastDataBlockOffset) { -892 return null; -893 } -894 -895 if (block.getOffset() < 0) { -896 throw new IOException("Invalid block file offset: " + block); -897 } -898 -899 // We are reading the next block without block type validation, because -900 // it might turn out to be a non-data block. -901 block = reader.readBlock(block.getOffset() -902 + block.getOnDiskSizeWithHeader(), -903 block.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread, -904 isCompaction, true, null, getEffectiveDataBlockEncoding()); -905 if (block != null && !block.getBlockType().isData()) { // Findbugs: NP_NULL_ON_SOME_PATH -906 // Whatever block we read we will be returning it unless -907 // it is a datablock. Just in case the blocks are non data blocks -908 reader.returnBlock(block); -909 } -910 } while (!block.getBlockType().isData()); -911 -912 return block; -913 } -914 -915 public DataBlockEncoding getEffectiveDataBlockEncoding() { -916 return this.reader.getEffectiveEncodingInCache(isCompaction); -917 } -918 -919 @Override -920 public Cell getCell() { -921 if (!isSeeked()) -922 return null; -923 -924 Cell ret; -925 int cellBufSize = getCellBufSize(); -926 long seqId = 0l; -927 if (this.reader.shouldIncludeMemstoreTS()) { -928 seqId = currMemstoreTS; -929 } -930 if (blockBuffer.hasArray()) { -931 // TODO : reduce the varieties of KV here. Check if based on a boolean -932 // we can handle the 'no tags' case. -933 if (currTagsLen > 0) { -934 if (this.curBlock.usesSharedMemory()) { -935 ret = new ShareableMemoryKeyValue(blockBuffer.array(), blockBuffer.arrayOffset() -936 + blockBuffer.position(), getCellBufSize(), seqId); -937 } else { -938 ret = new SizeCachedKeyValue(blockBuffer.array(), blockBuffer.arrayOffset() -939 + blockBuffer.position(), cellBufSize, seqId); -940 } -941 } else { -942 if (this.curBlock.usesSharedMemory()) { -943 ret = ne