Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 92107200D38 for ; Sun, 12 Nov 2017 16:17:00 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 90BB0160BF0; Sun, 12 Nov 2017 15:17:00 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 284A2160C05 for ; Sun, 12 Nov 2017 16:16:58 +0100 (CET) Received: (qmail 27249 invoked by uid 500); 12 Nov 2017 15:16:55 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 26302 invoked by uid 99); 12 Nov 2017 15:16:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 12 Nov 2017 15:16:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8C13BF5E0A; Sun, 12 Nov 2017 15:16:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sun, 12 Nov 2017 15:17:06 -0000 Message-Id: In-Reply-To: <6b29f3aa91c045f78e858f6d91ba1f26@git.apache.org> References: <6b29f3aa91c045f78e858f6d91ba1f26@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [14/51] [partial] hbase-site git commit: Published site at . archived-at: Sun, 12 Nov 2017 15:17:00 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/abb69192/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.EncodedScanner.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.EncodedScanner.html b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.EncodedScanner.html index a89df18..ea0bc8c 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.EncodedScanner.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.EncodedScanner.html @@ -49,26 +49,26 @@ 041import org.apache.hadoop.hbase.ByteBufferKeyValue; 042import org.apache.hadoop.hbase.SizeCachedKeyValue; 043import org.apache.hadoop.hbase.SizeCachedNoTagsKeyValue; -044import org.apache.yetus.audience.InterfaceAudience; -045import org.apache.hadoop.hbase.fs.HFileSystem; -046import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; -047import org.apache.hadoop.hbase.io.compress.Compression; -048import org.apache.hadoop.hbase.io.crypto.Cipher; -049import org.apache.hadoop.hbase.io.crypto.Encryption; -050import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; -051import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; -052import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext; -053import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; -054import org.apache.hadoop.hbase.nio.ByteBuff; -055import org.apache.hadoop.hbase.regionserver.KeyValueScanner; -056import org.apache.hadoop.hbase.security.EncryptionUtil; -057import org.apache.hadoop.hbase.util.ByteBufferUtils; -058import org.apache.hadoop.hbase.util.Bytes; -059import org.apache.hadoop.hbase.util.IdLock; -060import org.apache.hadoop.hbase.util.ObjectIntPair; -061import org.apache.hadoop.io.WritableUtils; -062import org.apache.htrace.Trace; -063import org.apache.htrace.TraceScope; +044import org.apache.hadoop.hbase.trace.TraceUtil; +045import org.apache.yetus.audience.InterfaceAudience; +046import org.apache.hadoop.hbase.fs.HFileSystem; +047import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; +048import org.apache.hadoop.hbase.io.compress.Compression; +049import org.apache.hadoop.hbase.io.crypto.Cipher; +050import org.apache.hadoop.hbase.io.crypto.Encryption; +051import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; +052import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +053import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext; +054import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; +055import org.apache.hadoop.hbase.nio.ByteBuff; +056import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +057import org.apache.hadoop.hbase.security.EncryptionUtil; +058import org.apache.hadoop.hbase.util.ByteBufferUtils; +059import org.apache.hadoop.hbase.util.Bytes; +060import org.apache.hadoop.hbase.util.IdLock; +061import org.apache.hadoop.hbase.util.ObjectIntPair; +062import org.apache.hadoop.io.WritableUtils; +063import org.apache.htrace.core.TraceScope; 064 065import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; 066 @@ -263,1235 +263,1235 @@ 255 // Prefetch file blocks upon open if requested 256 if (cacheConf.shouldPrefetchOnOpen()) { 257 PrefetchExecutor.request(path, new Runnable() { -258 public void run() { -259 long offset = 0; -260 long end = 0; -261 try { -262 end = getTrailer().getLoadOnOpenDataOffset(); -263 if (LOG.isTraceEnabled()) { -264 LOG.trace("Prefetch start " + getPathOffsetEndStr(path, offset, end)); -265 } -266 // TODO: Could we use block iterator in here? Would that get stuff into the cache? -267 HFileBlock prevBlock = null; -268 while (offset < end) { -269 if (Thread.interrupted()) { -270 break; -271 } -272 // Perhaps we got our block from cache? Unlikely as this may be, if it happens, then -273 // the internal-to-hfileblock thread local which holds the overread that gets the -274 // next header, will not have happened...so, pass in the onDiskSize gotten from the -275 // cached block. This 'optimization' triggers extremely rarely I'd say. -276 long onDiskSize = prevBlock != null? prevBlock.getNextBlockOnDiskSize(): -1; -277 HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false, -278 null, null); -279 // Need not update the current block. Ideally here the readBlock won't find the -280 // block in cache. We call this readBlock so that block data is read from FS and -281 // cached in BC. So there is no reference count increment that happens here. -282 // The return will ideally be a noop because the block is not of MemoryType SHARED. -283 returnBlock(block); -284 prevBlock = block; -285 offset += block.getOnDiskSizeWithHeader(); -286 } -287 } catch (IOException e) { -288 // IOExceptions are probably due to region closes (relocation, etc.) -289 if (LOG.isTraceEnabled()) { -290 LOG.trace("Prefetch " + getPathOffsetEndStr(path, offset, end), e); -291 } -292 } catch (NullPointerException e) { -293 LOG.warn("Stream moved/closed or prefetch cancelled?" + -294 getPathOffsetEndStr(path, offset, end), e); -295 } catch (Exception e) { -296 // Other exceptions are interesting -297 LOG.warn("Prefetch " + getPathOffsetEndStr(path, offset, end), e); -298 } finally { -299 PrefetchExecutor.complete(path); -300 } -301 } -302 }); -303 } -304 -305 byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN); -306 // max tag length is not present in the HFile means tags were not at all written to file. -307 if (tmp != null) { -308 hfileContext.setIncludesTags(true); -309 tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED); -310 if (tmp != null && Bytes.toBoolean(tmp)) { -311 hfileContext.setCompressTags(true); -312 } -313 } -314 } -315 -316 private static String getPathOffsetEndStr(final Path path, final long offset, final long end) { -317 return "path=" + path.toString() + ", offset=" + offset + ", end=" + end; -318 } -319 -320 /** -321 * File version check is a little sloppy. We read v3 files but can also read v2 files if their -322 * content has been pb'd; files written with 0.98. -323 */ -324 private void checkFileVersion() { -325 int majorVersion = trailer.getMajorVersion(); -326 if (majorVersion == getMajorVersion()) return; -327 int minorVersion = trailer.getMinorVersion(); -328 if (majorVersion == 2 && minorVersion >= MIN_V2_MINOR_VERSION_WITH_PB) return; -329 // We can read v3 or v2 versions of hfile. -330 throw new IllegalArgumentException("Invalid HFile version: major=" + -331 trailer.getMajorVersion() + ", minor=" + trailer.getMinorVersion() + ": expected at least " + -332 "major=2 and minor=" + MAX_MINOR_VERSION); -333 } -334 -335 @SuppressWarnings("serial") -336 public static class BlockIndexNotLoadedException extends IllegalStateException { -337 public BlockIndexNotLoadedException() { -338 // Add a message in case anyone relies on it as opposed to class name. -339 super("Block index not loaded"); -340 } -341 } -342 -343 private Optional<String> toStringFirstKey() { -344 return getFirstKey().map(CellUtil::getCellKeyAsString); -345 } -346 -347 private Optional<String> toStringLastKey() { -348 return getLastKey().map(CellUtil::getCellKeyAsString); -349 } -350 -351 @Override -352 public String toString() { -353 return "reader=" + path.toString() + -354 (!isFileInfoLoaded()? "": -355 ", compression=" + compressAlgo.getName() + -356 ", cacheConf=" + cacheConf + -357 ", firstKey=" + toStringFirstKey() + -358 ", lastKey=" + toStringLastKey()) + -359 ", avgKeyLen=" + avgKeyLen + -360 ", avgValueLen=" + avgValueLen + -361 ", entries=" + trailer.getEntryCount() + -362 ", length=" + fileSize; -363 } -364 -365 @Override -366 public long length() { -367 return fileSize; -368 } -369 -370 @Override -371 public void returnBlock(HFileBlock block) { -372 BlockCache blockCache = this.cacheConf.getBlockCache(); -373 if (blockCache != null && block != null) { -374 BlockCacheKey cacheKey = new BlockCacheKey(this.getFileContext().getHFileName(), -375 block.getOffset(), this.isPrimaryReplicaReader(), block.getBlockType()); -376 blockCache.returnBlock(cacheKey, block); -377 } -378 } -379 /** -380 * @return the first key in the file. May be null if file has no entries. Note -381 * that this is not the first row key, but rather the byte form of the -382 * first KeyValue. -383 */ -384 @Override -385 public Optional<Cell> getFirstKey() { -386 if (dataBlockIndexReader == null) { -387 throw new BlockIndexNotLoadedException(); -388 } -389 return dataBlockIndexReader.isEmpty() ? Optional.empty() -390 : Optional.of(dataBlockIndexReader.getRootBlockKey(0)); -391 } -392 -393 /** -394 * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's -395 * patch goes in to eliminate {@link KeyValue} here. -396 * -397 * @return the first row key, or null if the file is empty. -398 */ -399 @Override -400 public Optional<byte[]> getFirstRowKey() { -401 // We have to copy the row part to form the row key alone -402 return getFirstKey().map(CellUtil::cloneRow); -403 } -404 -405 /** -406 * TODO left from {@link HFile} version 1: move this to StoreFile after -407 * Ryan's patch goes in to eliminate {@link KeyValue} here. -408 * -409 * @return the last row key, or null if the file is empty. -410 */ -411 @Override -412 public Optional<byte[]> getLastRowKey() { -413 // We have to copy the row part to form the row key alone -414 return getLastKey().map(CellUtil::cloneRow); -415 } -416 -417 /** @return number of KV entries in this HFile */ -418 @Override -419 public long getEntries() { -420 return trailer.getEntryCount(); -421 } -422 -423 /** @return comparator */ -424 @Override -425 public CellComparator getComparator() { -426 return comparator; -427 } -428 -429 /** @return compression algorithm */ -430 @Override -431 public Compression.Algorithm getCompressionAlgorithm() { -432 return compressAlgo; -433 } -434 -435 /** -436 * @return the total heap size of data and meta block indexes in bytes. Does -437 * not take into account non-root blocks of a multilevel data index. -438 */ -439 public long indexSize() { -440 return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0) -441 + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize() -442 : 0); -443 } -444 -445 @Override -446 public String getName() { -447 return name; -448 } -449 -450 @Override -451 public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() { -452 return dataBlockIndexReader; -453 } -454 -455 @Override -456 public FixedFileTrailer getTrailer() { -457 return trailer; -458 } -459 -460 @Override -461 public boolean isPrimaryReplicaReader() { -462 return primaryReplicaReader; -463 } -464 -465 @Override -466 public FileInfo loadFileInfo() throws IOException { -467 return fileInfo; -468 } -469 -470 /** -471 * An exception thrown when an operation requiring a scanner to be seeked -472 * is invoked on a scanner that is not seeked. -473 */ -474 @SuppressWarnings("serial") -475 public static class NotSeekedException extends IllegalStateException { -476 public NotSeekedException() { -477 super("Not seeked to a key/value"); -478 } -479 } -480 -481 protected static class HFileScannerImpl implements HFileScanner { -482 private ByteBuff blockBuffer; -483 protected final boolean cacheBlocks; -484 protected final boolean pread; -485 protected final boolean isCompaction; -486 private int currKeyLen; -487 private int currValueLen; -488 private int currMemstoreTSLen; -489 private long currMemstoreTS; -490 // Updated but never read? -491 protected AtomicInteger blockFetches = new AtomicInteger(0); -492 protected final HFile.Reader reader; -493 private int currTagsLen; -494 // buffer backed keyonlyKV -495 private ByteBufferKeyOnlyKeyValue bufBackedKeyOnlyKv = new ByteBufferKeyOnlyKeyValue(); -496 // A pair for reusing in blockSeek() so that we don't garbage lot of objects -497 final ObjectIntPair<ByteBuffer> pair = new ObjectIntPair<>(); -498 -499 /** -500 * The next indexed key is to keep track of the indexed key of the next data block. -501 * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the -502 * current data block is the last data block. -503 * -504 * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet. -505 */ -506 protected Cell nextIndexedKey; -507 // Current block being used -508 protected HFileBlock curBlock; -509 // Previous blocks that were used in the course of the read -510 protected final ArrayList<HFileBlock> prevBlocks = new ArrayList<>(); -511 -512 public HFileScannerImpl(final HFile.Reader reader, final boolean cacheBlocks, -513 final boolean pread, final boolean isCompaction) { -514 this.reader = reader; -515 this.cacheBlocks = cacheBlocks; -516 this.pread = pread; -517 this.isCompaction = isCompaction; -518 } -519 -520 void updateCurrBlockRef(HFileBlock block) { -521 if (block != null && this.curBlock != null && -522 block.getOffset() == this.curBlock.getOffset()) { -523 return; -524 } -525 // We don't have to keep ref to EXCLUSIVE type of block -526 if (this.curBlock != null && this.curBlock.usesSharedMemory()) { -527 prevBlocks.add(this.curBlock); -528 } -529 this.curBlock = block; -530 } -531 -532 void reset() { -533 // We don't have to keep ref to EXCLUSIVE type of block -534 if (this.curBlock != null && this.curBlock.usesSharedMemory()) { -535 this.prevBlocks.add(this.curBlock); -536 } -537 this.curBlock = null; -538 } -539 -540 private void returnBlockToCache(HFileBlock block) { -541 if (LOG.isTraceEnabled()) { -542 LOG.trace("Returning the block : " + block); -543 } -544 this.reader.returnBlock(block); -545 } -546 -547 private void returnBlocks(boolean returnAll) { -548 for (int i = 0; i < this.prevBlocks.size(); i++) { -549 returnBlockToCache(this.prevBlocks.get(i)); -550 } -551 this.prevBlocks.clear(); -552 if (returnAll && this.curBlock != null) { -553 returnBlockToCache(this.curBlock); -554 this.curBlock = null; -555 } -556 } -557 @Override -558 public boolean isSeeked(){ -559 return blockBuffer != null; -560 } -561 -562 @Override -563 public String toString() { -564 return "HFileScanner for reader " + String.valueOf(getReader()); -565 } -566 -567 protected void assertSeeked() { -568 if (!isSeeked()) -569 throw new NotSeekedException(); -570 } -571 -572 @Override -573 public HFile.Reader getReader() { -574 return reader; -575 } -576 -577 // From non encoded HFiles, we always read back KeyValue or its descendant.(Note: When HFile -578 // block is in DBB, it will be OffheapKV). So all parts of the Cell is in a contiguous -579 // array/buffer. How many bytes we should wrap to make the KV is what this method returns. -580 private int getKVBufSize() { -581 int kvBufSize = KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen; -582 if (currTagsLen > 0) { -583 kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen; -584 } -585 return kvBufSize; -586 } -587 -588 @Override -589 public void close() { -590 if (!pread) { -591 // For seek + pread stream socket should be closed when the scanner is closed. HBASE-9393 -592 reader.unbufferStream(); -593 } -594 this.returnBlocks(true); -595 } -596 -597 // Returns the #bytes in HFile for the current cell. Used to skip these many bytes in current -598 // HFile block's buffer so as to position to the next cell. -599 private int getCurCellSerializedSize() { -600 int curCellSize = KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen -601 + currMemstoreTSLen; -602 if (this.reader.getFileContext().isIncludesTags()) { -603 curCellSize += Bytes.SIZEOF_SHORT + currTagsLen; -604 } -605 return curCellSize; -606 } -607 -608 protected void readKeyValueLen() { -609 // This is a hot method. We go out of our way to make this method short so it can be -610 // inlined and is not too big to compile. We also manage position in ByteBuffer ourselves -611 // because it is faster than going via range-checked ByteBuffer methods or going through a -612 // byte buffer array a byte at a time. -613 // Get a long at a time rather than read two individual ints. In micro-benchmarking, even -614 // with the extra bit-fiddling, this is order-of-magnitude faster than getting two ints. -615 // Trying to imitate what was done - need to profile if this is better or -616 // earlier way is better by doing mark and reset? -617 // But ensure that you read long instead of two ints -618 long ll = blockBuffer.getLongAfterPosition(0); -619 // Read top half as an int of key length and bottom int as value length -620 this.currKeyLen = (int)(ll >> Integer.SIZE); -621 this.currValueLen = (int)(Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll); -622 checkKeyValueLen(); -623 // Move position past the key and value lengths and then beyond the key and value -624 int p = (Bytes.SIZEOF_LONG + currKeyLen + currValueLen); -625 if (reader.getFileContext().isIncludesTags()) { -626 // Tags length is a short. -627 this.currTagsLen = blockBuffer.getShortAfterPosition(p); -628 checkTagsLen(); -629 p += (Bytes.SIZEOF_SHORT + currTagsLen); -630 } -631 readMvccVersion(p); -632 } -633 -634 private final void checkTagsLen() { -635 if (checkLen(this.currTagsLen)) { -636 throw new IllegalStateException("Invalid currTagsLen " + this.currTagsLen + -637 ". Block offset: " + curBlock.getOffset() + ", block length: " + -638 this.blockBuffer.limit() + -639 ", position: " + this.blockBuffer.position() + " (without header)."); -640 } -641 } -642 -643 /** -644 * Read mvcc. Does checks to see if we even need to read the mvcc at all. -645 * @param offsetFromPos -646 */ -647 protected void readMvccVersion(final int offsetFromPos) { -648 // See if we even need to decode mvcc. -649 if (!this.reader.shouldIncludeMemStoreTS()) return; -650 if (!this.reader.isDecodeMemStoreTS()) { -651 currMemstoreTS = 0; -652 currMemstoreTSLen = 1; -653 return; -654 } -655 _readMvccVersion(offsetFromPos); -656 } -657 -658 /** -659 * Actually do the mvcc read. Does no checks. -660 * @param offsetFromPos -661 */ -662 private void _readMvccVersion(int offsetFromPos) { -663 // This is Bytes#bytesToVint inlined so can save a few instructions in this hot method; i.e. -664 // previous if one-byte vint, we'd redo the vint call to find int size. -665 // Also the method is kept small so can be inlined. -666 byte firstByte = blockBuffer.getByteAfterPosition(offsetFromPos); -667 int len = WritableUtils.decodeVIntSize(firstByte); -668 if (len == 1) { -669 this.currMemstoreTS = firstByte; -670 } else { -671 int remaining = len -1; -672 long i = 0; -673 offsetFromPos++; -674 if (remaining >= Bytes.SIZEOF_INT) { -675 // The int read has to be converted to unsigned long so the & op -676 i = (blockBuffer.getIntAfterPosition(offsetFromPos) & 0x00000000ffffffffL); -677 remaining -= Bytes.SIZEOF_INT; -678 offsetFromPos += Bytes.SIZEOF_INT; -679 } -680 if (remaining >= Bytes.SIZEOF_SHORT) { -681 short s = blockBuffer.getShortAfterPosition(offsetFromPos); -682 i = i << 16; -683 i = i | (s & 0xFFFF); -684 remaining -= Bytes.SIZEOF_SHORT; -685 offsetFromPos += Bytes.SIZEOF_SHORT; -686 } -687 for (int idx = 0; idx < remaining; idx++) { -688 byte b = blockBuffer.getByteAfterPosition(offsetFromPos + idx); -689 i = i << 8; -690 i = i | (b & 0xFF); -691 } -692 currMemstoreTS = (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); -693 } -694 this.currMemstoreTSLen = len; -695 } -696 -697 /** -698 * Within a loaded block, seek looking for the last key that is smaller than -699 * (or equal to?) the key we are interested in. -700 * A note on the seekBefore: if you have seekBefore = true, AND the first -701 * key in the block = key, then you'll get thrown exceptions. The caller has -702 * to check for that case and load the previous block as appropriate. -703 * @param key -704 * the key to find -705 * @param seekBefore -706 * find the key before the given key in case of exact match. -707 * @return 0 in case of an exact key match, 1 in case of an inexact match, -708 * -2 in case of an inexact match and furthermore, the input key -709 * less than the first key of current block(e.g. using a faked index -710 * key) -711 */ -712 protected int blockSeek(Cell key, boolean seekBefore) { -713 int klen, vlen, tlen = 0; -714 int lastKeyValueSize = -1; -715 int offsetFromPos; -716 do { -717 offsetFromPos = 0; -718 // Better to ensure that we use the BB Utils here -719 long ll = blockBuffer.getLongAfterPosition(offsetFromPos); -720 klen = (int)(ll >> Integer.SIZE); -721 vlen = (int)(Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll); -722 if (checkKeyLen(klen) || checkLen(vlen)) { -723 throw new IllegalStateException("Invalid klen " + klen + " or vlen " -724 + vlen + ". Block offset: " -725 + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: " -726 + blockBuffer.position() + " (without header)."); -727 } -728 offsetFromPos += Bytes.SIZEOF_LONG; -729 blockBuffer.asSubByteBuffer(blockBuffer.position() + offsetFromPos, klen, pair); -730 bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), klen); -731 int comp = -732 PrivateCellUtil.compareKeyIgnoresMvcc(reader.getComparator(), key, bufBackedKeyOnlyKv); -733 offsetFromPos += klen + vlen; -734 if (this.reader.getFileContext().isIncludesTags()) { -735 // Read short as unsigned, high byte first -736 tlen = ((blockBuffer.getByteAfterPosition(offsetFromPos) & 0xff) << 8) -737 ^ (blockBuffer.getByteAfterPosition(offsetFromPos + 1) & 0xff); -738 if (checkLen(tlen)) { -739 throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: " -740 + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: " -741 + blockBuffer.position() + " (without header)."); -742 } -743 // add the two bytes read for the tags. -744 offsetFromPos += tlen + (Bytes.SIZEOF_SHORT); -745 } -746 if (this.reader.shouldIncludeMemStoreTS()) { -747 // Directly read the mvcc based on current position -748 readMvccVersion(offsetFromPos); -749 } -750 if (comp == 0) { -751 if (seekBefore) { -752 if (lastKeyValueSize < 0) { -753 throw new IllegalStateException("blockSeek with seekBefore " -754 + "at the first key of the block: key=" + CellUtil.getCellKeyAsString(key) -755 + ", blockOffset=" + curBlock.getOffset() + ", onDiskSize=" -756 + curBlock.getOnDiskSizeWithHeader()); -757 } -758 blockBuffer.moveBack(lastKeyValueSize); -759 readKeyValueLen(); -760 return 1; // non exact match. -761 } -762 currKeyLen = klen; -763 currValueLen = vlen; -764 currTagsLen = tlen; -765 return 0; // indicate exact match -766 } else if (comp < 0) { -767 if (lastKeyValueSize > 0) { -768 blockBuffer.moveBack(lastKeyValueSize); -769 } -770 readKeyValueLen(); -771 if (lastKeyValueSize == -1 && blockBuffer.position() == 0) { -772 return HConstants.INDEX_KEY_MAGIC; -773 } -774 return 1; -775 } -776 // The size of this key/value tuple, including key/value length fields. -777 lastKeyValueSize = klen + vlen + currMemstoreTSLen + KEY_VALUE_LEN_SIZE; -778 // include tag length also if tags included with KV -779 if (reader.getFileContext().isIncludesTags()) { -780 lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT; -781 } -782 blockBuffer.skip(lastKeyValueSize); -783 } while (blockBuffer.hasRemaining()); -784 -785 // Seek to the last key we successfully read. This will happen if this is -786 // the last key/value pair in the file, in which case the following call -787 // to next() has to return false. -788 blockBuffer.moveBack(lastKeyValueSize); -789 readKeyValueLen(); -790 return 1; // didn't exactly find it. -791 } -792 -793 @Override -794 public Cell getNextIndexedKey() { -795 return nextIndexedKey; -796 } -797 -798 @Override -799 public int seekTo(Cell key) throws IOException { -800 return seekTo(key, true); -801 } -802 -803 @Override -804 public int reseekTo(Cell key) throws IOException { -805 int compared; -806 if (isSeeked()) { -807 compared = compareKey(reader.getComparator(), key); -808 if (compared < 1) { -809 // If the required key is less than or equal to current key, then -810 // don't do anything. -811 return compared; -812 } else { -813 // The comparison with no_next_index_key has to be checked -814 if (this.nextIndexedKey != null && -815 (this.nextIndexedKey == KeyValueScanner.NO_NEXT_INDEXED_KEY || PrivateCellUtil -816 .compareKeyIgnoresMvcc(reader.getComparator(), key, nextIndexedKey) < 0)) { -817 // The reader shall continue to scan the current data block instead -818 // of querying the -819 // block index as long as it knows the target key is strictly -820 // smaller than -821 // the next indexed key or the current data block is the last data -822 // block. -823 return loadBlockAndSeekToKey(this.curBlock, nextIndexedKey, false, key, -824 false); -825 } -826 -827 } -828 } -829 // Don't rewind on a reseek operation, because reseek implies that we are -830 // always going forward in the file. -831 return seekTo(key, false); -832 } -833 -834 /** -835 * An internal API function. Seek to the given key, optionally rewinding to -836 * the first key of the block before doing the seek. -837 * -838 * @param key - a cell representing the key that we need to fetch -839 * @param rewind whether to rewind to the first key of the block before -840 * doing the seek. If this is false, we are assuming we never go -841 * back, otherwise the result is undefined. -842 * @return -1 if the key is earlier than the first key of the file, -843 * 0 if we are at the given key, 1 if we are past the given key -844 * -2 if the key is earlier than the first key of the file while -845 * using a faked index key -846 * @throws IOException -847 */ -848 public int seekTo(Cell key, boolean rewind) throws IOException { -849 HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader(); -850 BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, curBlock, -851 cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding()); -852 if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) { -853 // This happens if the key e.g. falls before the beginning of the -854 // file. -855 return -1; -856 } -857 return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(), -858 blockWithScanInfo.getNextIndexedKey(), rewind, key, false); -859 } -860 -861 @Override -862 public boolean seekBefore(Cell key) throws IOException { -863 HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, curBlock, -864 cacheBlocks, pread, isCompaction, reader.getEffectiveEncodingInCache(isCompaction)); -865 if (seekToBlock == null) { -866 return false; -867 } -868 Cell firstKey = getFirstKeyCellInBlock(seekToBlock); -869 if (PrivateCellUtil.compareKeyIgnoresMvcc(reader.getComparator(), firstKey, key) >= 0) { -870 long previousBlockOffset = seekToBlock.getPrevBlockOffset(); -871 // The key we are interested in -872 if (previousBlockOffset == -1) { -873 // we have a 'problem', the key we want is the first of the file. -874 return false; -875 } -876 -877 // The first key in the current block 'seekToBlock' is greater than the given -878 // seekBefore key. We will go ahead by reading the next block that satisfies the -879 // given key. Return the current block before reading the next one. -880 reader.returnBlock(seekToBlock); -881 // It is important that we compute and pass onDiskSize to the block -882 // reader so that it does not have to read the header separately to -883 // figure out the size. Currently, we do not have a way to do this -884 // correctly in the general case however. -885 // TODO: See https://issues.apache.org/jira/browse/HBASE-14576 -886 int prevBlockSize = -1; -887 seekToBlock = reader.readBlock(previousBlockOffset, -888 prevBlockSize, cacheBlocks, -889 pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding()); -890 // TODO shortcut: seek forward in this block to the last key of the -891 // block. -892 } -893 loadBlockAndSeekToKey(seekToBlock, firstKey, true, key, true); -894 return true; -895 } -896 -897 /** -898 * Scans blocks in the "scanned" section of the {@link HFile} until the next -899 * data block is found. -900 * -901 * @return the next block, or null if there are no more data blocks -902 * @throws IOException -903 */ -904 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH", -905 justification="Yeah, unnecessary null check; could do w/ clean up") -906 protected HFileBlock readNextDataBlock() throws IOException { -907 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); -908 if (curBlock == null) -909 return null; -910 -911 HFileBlock block = this.curBlock; +258 @Override +259 public void run() { +260 long offset = 0; +261 long end = 0; +262 try { +263 end = getTrailer().getLoadOnOpenDataOffset(); +264 if (LOG.isTraceEnabled()) { +265 LOG.trace("Prefetch start " + getPathOffsetEndStr(path, offset, end)); +266 } +267 // TODO: Could we use block iterator in here? Would that get stuff into the cache? +268 HFileBlock prevBlock = null; +269 while (offset < end) { +270 if (Thread.interrupted()) { +271 break; +272 } +273 // Perhaps we got our block from cache? Unlikely as this may be, if it happens, then +274 // the internal-to-hfileblock thread local which holds the overread that gets the +275 // next header, will not have happened...so, pass in the onDiskSize gotten from the +276 // cached block. This 'optimization' triggers extremely rarely I'd say. +277 long onDiskSize = prevBlock != null? prevBlock.getNextBlockOnDiskSize(): -1; +278 HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false, +279 null, null); +280 // Need not update the current block. Ideally here the readBlock won't find the +281 // block in cache. We call this readBlock so that block data is read from FS and +282 // cached in BC. So there is no reference count increment that happens here. +283 // The return will ideally be a noop because the block is not of MemoryType SHARED. +284 returnBlock(block); +285 prevBlock = block; +286 offset += block.getOnDiskSizeWithHeader(); +287 } +288 } catch (IOException e) { +289 // IOExceptions are probably due to region closes (relocation, etc.) +290 if (LOG.isTraceEnabled()) { +291 LOG.trace("Prefetch " + getPathOffsetEndStr(path, offset, end), e); +292 } +293 } catch (NullPointerException e) { +294 LOG.warn("Stream moved/closed or prefetch cancelled?" + +295 getPathOffsetEndStr(path, offset, end), e); +296 } catch (Exception e) { +297 // Other exceptions are interesting +298 LOG.warn("Prefetch " + getPathOffsetEndStr(path, offset, end), e); +299 } finally { +300 PrefetchExecutor.complete(path); +301 } +302 } +303 }); +304 } +305 +306 byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN); +307 // max tag length is not present in the HFile means tags were not at all written to file. +308 if (tmp != null) { +309 hfileContext.setIncludesTags(true); +310 tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED); +311 if (tmp != null && Bytes.toBoolean(tmp)) { +312 hfileContext.setCompressTags(true); +313 } +314 } +315 } +316 +317 private static String getPathOffsetEndStr(final Path path, final long offset, final long end) { +318 return "path=" + path.toString() + ", offset=" + offset + ", end=" + end; +319 } +320 +321 /** +322 * File version check is a little sloppy. We read v3 files but can also read v2 files if their +323 * content has been pb'd; files written with 0.98. +324 */ +325 private void checkFileVersion() { +326 int majorVersion = trailer.getMajorVersion(); +327 if (majorVersion == getMajorVersion()) return; +328 int minorVersion = trailer.getMinorVersion(); +329 if (majorVersion == 2 && minorVersion >= MIN_V2_MINOR_VERSION_WITH_PB) return; +330 // We can read v3 or v2 versions of hfile. +331 throw new IllegalArgumentException("Invalid HFile version: major=" + +332 trailer.getMajorVersion() + ", minor=" + trailer.getMinorVersion() + ": expected at least " + +333 "major=2 and minor=" + MAX_MINOR_VERSION); +334 } +335 +336 @SuppressWarnings("serial") +337 public static class BlockIndexNotLoadedException extends IllegalStateException { +338 public BlockIndexNotLoadedException() { +339 // Add a message in case anyone relies on it as opposed to class name. +340 super("Block index not loaded"); +341 } +342 } +343 +344 private Optional<String> toStringFirstKey() { +345 return getFirstKey().map(CellUtil::getCellKeyAsString); +346 } +347 +348 private Optional<String> toStringLastKey() { +349 return getLastKey().map(CellUtil::getCellKeyAsString); +350 } +351 +352 @Override +353 public String toString() { +354 return "reader=" + path.toString() + +355 (!isFileInfoLoaded()? "": +356 ", compression=" + compressAlgo.getName() + +357 ", cacheConf=" + cacheConf + +358 ", firstKey=" + toStringFirstKey() + +359 ", lastKey=" + toStringLastKey()) + +360 ", avgKeyLen=" + avgKeyLen + +361 ", avgValueLen=" + avgValueLen + +362 ", entries=" + trailer.getEntryCount() + +363 ", length=" + fileSize; +364 } +365 +366 @Override +367 public long length() { +368 return fileSize; +369 } +370 +371 @Override +372 public void returnBlock(HFileBlock block) { +373 BlockCache blockCache = this.cacheConf.getBlockCache(); +374 if (blockCache != null && block != null) { +375 BlockCacheKey cacheKey = new BlockCacheKey(this.getFileContext().getHFileName(), +376 block.getOffset(), this.isPrimaryReplicaReader(), block.getBlockType()); +377 blockCache.returnBlock(cacheKey, block); +378 } +379 } +380 /** +381 * @return the first key in the file. May be null if file has no entries. Note +382 * that this is not the first row key, but rather the byte form of the +383 * first KeyValue. +384 */ +385 @Override +386 public Optional<Cell> getFirstKey() { +387 if (dataBlockIndexReader == null) { +388 throw new BlockIndexNotLoadedException(); +389 } +390 return dataBlockIndexReader.isEmpty() ? Optional.empty() +391 : Optional.of(dataBlockIndexReader.getRootBlockKey(0)); +392 } +393 +394 /** +395 * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's +396 * patch goes in to eliminate {@link KeyValue} here. +397 * +398 * @return the first row key, or null if the file is empty. +399 */ +400 @Override +401 public Optional<byte[]> getFirstRowKey() { +402 // We have to copy the row part to form the row key alone +403 return getFirstKey().map(CellUtil::cloneRow); +404 } +405 +406 /** +407 * TODO left from {@link HFile} version 1: move this to StoreFile after +408 * Ryan's patch goes in to eliminate {@link KeyValue} here. +409 * +410 * @return the last row key, or null if the file is empty. +411 */ +412 @Override +413 public Optional<byte[]> getLastRowKey() { +414 // We have to copy the row part to form the row key alone +415 return getLastKey().map(CellUtil::cloneRow); +416 } +417 +418 /** @return number of KV entries in this HFile */ +419 @Override +420 public long getEntries() { +421 return trailer.getEntryCount(); +422 } +423 +424 /** @return comparator */ +425 @Override +426 public CellComparator getComparator() { +427 return comparator; +428 } +429 +430 /** @return compression algorithm */ +431 @Override +432 public Compression