Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7269718B29 for ; Wed, 30 Dec 2015 22:15:32 +0000 (UTC) Received: (qmail 41937 invoked by uid 500); 30 Dec 2015 22:15:24 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 41851 invoked by uid 500); 30 Dec 2015 22:15:24 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 40069 invoked by uid 99); 30 Dec 2015 22:15:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Dec 2015 22:15:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 95FD1E0537; Wed, 30 Dec 2015 22:15:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Wed, 30 Dec 2015 22:15:55 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [34/51] [partial] hbase-site git commit: Published site at 07b623670647686084f8f5fd2038e2bafcfdac54. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e5c3dcd1/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.HFileScannerImpl.ShareableMemoryKeyValue.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.HFileScannerImpl.ShareableMemoryKeyValue.html b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.HFileScannerImpl.ShareableMemoryKeyValue.html index 5be9775..128a211 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.HFileScannerImpl.ShareableMemoryKeyValue.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.HFileScannerImpl.ShareableMemoryKeyValue.html @@ -994,902 +994,908 @@ 986 return new KeyValue.KeyOnlyKeyValue(keyBuf.array(), keyBuf.arrayOffset() 987 + keyPair.getSecond(), currKeyLen); 988 } else { -989 return new ByteBufferedKeyOnlyKeyValue(keyBuf, keyPair.getSecond(), currKeyLen); -990 } -991 } -992 -993 private static class ShareableMemoryKeyValue extends SizeCachedKeyValue implements -994 ShareableMemory { -995 public ShareableMemoryKeyValue(byte[] bytes, int offset, int length, long seqId) { -996 super(bytes, offset, length, seqId); -997 } +989 // Better to do a copy here instead of holding on to this BB so that +990 // we could release the blocks referring to this key. This key is specifically used +991 // in HalfStoreFileReader to get the firstkey and lastkey by creating a new scanner +992 // every time. So holding onto the BB (incase of DBB) is not advised here. +993 byte[] key = new byte[currKeyLen]; +994 ByteBufferUtils.copyFromBufferToArray(key, keyBuf, keyPair.getSecond(), 0, currKeyLen); +995 return new KeyValue.KeyOnlyKeyValue(key, 0, currKeyLen); +996 } +997 } 998 -999 @Override -1000 public Cell cloneToCell() { -1001 byte[] copy = Bytes.copy(this.bytes, this.offset, this.length); -1002 return new SizeCachedKeyValue(copy, 0, copy.length, getSequenceId()); +999 private static class ShareableMemoryKeyValue extends SizeCachedKeyValue implements +1000 ShareableMemory { +1001 public ShareableMemoryKeyValue(byte[] bytes, int offset, int length, long seqId) { +1002 super(bytes, offset, length, seqId); 1003 } -1004 } -1005 -1006 private static class ShareableMemoryNoTagsKeyValue extends SizeCachedNoTagsKeyValue implements -1007 ShareableMemory { -1008 public ShareableMemoryNoTagsKeyValue(byte[] bytes, int offset, int length, long seqId) { -1009 super(bytes, offset, length, seqId); -1010 } +1004 +1005 @Override +1006 public Cell cloneToCell() { +1007 byte[] copy = Bytes.copy(this.bytes, this.offset, this.length); +1008 return new SizeCachedKeyValue(copy, 0, copy.length, getSequenceId()); +1009 } +1010 } 1011 -1012 @Override -1013 public Cell cloneToCell() { -1014 byte[] copy = Bytes.copy(this.bytes, this.offset, this.length); -1015 return new SizeCachedNoTagsKeyValue(copy, 0, copy.length, getSequenceId()); +1012 private static class ShareableMemoryNoTagsKeyValue extends SizeCachedNoTagsKeyValue implements +1013 ShareableMemory { +1014 public ShareableMemoryNoTagsKeyValue(byte[] bytes, int offset, int length, long seqId) { +1015 super(bytes, offset, length, seqId); 1016 } -1017 } -1018 -1019 private static class ShareableMemoryOffheapKeyValue extends OffheapKeyValue implements -1020 ShareableMemory { -1021 public ShareableMemoryOffheapKeyValue(ByteBuffer buf, int offset, int length, -1022 boolean hasTags, long seqId) { -1023 super(buf, offset, length, hasTags, seqId); -1024 } -1025 -1026 @Override -1027 public Cell cloneToCell() { -1028 byte[] copy = new byte[this.length]; -1029 ByteBufferUtils.copyFromBufferToArray(copy, this.buf, this.offset, 0, this.length); -1030 return new SizeCachedKeyValue(copy, 0, copy.length, getSequenceId()); -1031 } -1032 } -1033 -1034 @Override -1035 public ByteBuffer getValue() { -1036 assertSeeked(); -1037 // Okie to create new Pair. Not used in hot path -1038 ObjectIntPair<ByteBuffer> valuePair = new ObjectIntPair<ByteBuffer>(); -1039 this.blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen, -1040 currValueLen, valuePair); -1041 ByteBuffer valBuf = valuePair.getFirst().duplicate(); -1042 valBuf.position(valuePair.getSecond()); -1043 valBuf.limit(currValueLen + valuePair.getSecond()); -1044 return valBuf.slice(); -1045 } -1046 -1047 protected void setNonSeekedState() { -1048 reset(); -1049 blockBuffer = null; -1050 currKeyLen = 0; -1051 currValueLen = 0; -1052 currMemstoreTS = 0; -1053 currMemstoreTSLen = 0; -1054 currTagsLen = 0; -1055 } -1056 -1057 /** -1058 * Set the position on current backing blockBuffer. -1059 */ -1060 private void positionThisBlockBuffer() { -1061 try { -1062 blockBuffer.skip(getCurCellSize()); -1063 } catch (IllegalArgumentException e) { -1064 LOG.error("Current pos = " + blockBuffer.position() -1065 + "; currKeyLen = " + currKeyLen + "; currValLen = " -1066 + currValueLen + "; block limit = " + blockBuffer.limit() -1067 + "; HFile name = " + reader.getName() -1068 + "; currBlock currBlockOffset = " + this.curBlock.getOffset()); -1069 throw e; -1070 } -1071 } -1072 -1073 /** -1074 * Set our selves up for the next 'next' invocation, set up next block. -1075 * @return True is more to read else false if at the end. -1076 * @throws IOException -1077 */ -1078 private boolean positionForNextBlock() throws IOException { -1079 // Methods are small so they get inlined because they are 'hot'. -1080 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); -1081 if (this.curBlock.getOffset() >= lastDataBlockOffset) { -1082 setNonSeekedState(); -1083 return false; -1084 } -1085 return isNextBlock(); -1086 } -1087 -1088 -1089 private boolean isNextBlock() throws IOException { -1090 // Methods are small so they get inlined because they are 'hot'. -1091 HFileBlock nextBlock = readNextDataBlock(); -1092 if (nextBlock == null) { -1093 setNonSeekedState(); -1094 return false; -1095 } -1096 updateCurrentBlock(nextBlock); -1097 return true; -1098 } -1099 -1100 private final boolean _next() throws IOException { -1101 // Small method so can be inlined. It is a hot one. -1102 if (blockBuffer.remaining() <= 0) { -1103 return positionForNextBlock(); -1104 } +1017 +1018 @Override +1019 public Cell cloneToCell() { +1020 byte[] copy = Bytes.copy(this.bytes, this.offset, this.length); +1021 return new SizeCachedNoTagsKeyValue(copy, 0, copy.length, getSequenceId()); +1022 } +1023 } +1024 +1025 private static class ShareableMemoryOffheapKeyValue extends OffheapKeyValue implements +1026 ShareableMemory { +1027 public ShareableMemoryOffheapKeyValue(ByteBuffer buf, int offset, int length, +1028 boolean hasTags, long seqId) { +1029 super(buf, offset, length, hasTags, seqId); +1030 } +1031 +1032 @Override +1033 public Cell cloneToCell() { +1034 byte[] copy = new byte[this.length]; +1035 ByteBufferUtils.copyFromBufferToArray(copy, this.buf, this.offset, 0, this.length); +1036 return new SizeCachedKeyValue(copy, 0, copy.length, getSequenceId()); +1037 } +1038 } +1039 +1040 @Override +1041 public ByteBuffer getValue() { +1042 assertSeeked(); +1043 // Okie to create new Pair. Not used in hot path +1044 ObjectIntPair<ByteBuffer> valuePair = new ObjectIntPair<ByteBuffer>(); +1045 this.blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen, +1046 currValueLen, valuePair); +1047 ByteBuffer valBuf = valuePair.getFirst().duplicate(); +1048 valBuf.position(valuePair.getSecond()); +1049 valBuf.limit(currValueLen + valuePair.getSecond()); +1050 return valBuf.slice(); +1051 } +1052 +1053 protected void setNonSeekedState() { +1054 reset(); +1055 blockBuffer = null; +1056 currKeyLen = 0; +1057 currValueLen = 0; +1058 currMemstoreTS = 0; +1059 currMemstoreTSLen = 0; +1060 currTagsLen = 0; +1061 } +1062 +1063 /** +1064 * Set the position on current backing blockBuffer. +1065 */ +1066 private void positionThisBlockBuffer() { +1067 try { +1068 blockBuffer.skip(getCurCellSize()); +1069 } catch (IllegalArgumentException e) { +1070 LOG.error("Current pos = " + blockBuffer.position() +1071 + "; currKeyLen = " + currKeyLen + "; currValLen = " +1072 + currValueLen + "; block limit = " + blockBuffer.limit() +1073 + "; HFile name = " + reader.getName() +1074 + "; currBlock currBlockOffset = " + this.curBlock.getOffset()); +1075 throw e; +1076 } +1077 } +1078 +1079 /** +1080 * Set our selves up for the next 'next' invocation, set up next block. +1081 * @return True is more to read else false if at the end. +1082 * @throws IOException +1083 */ +1084 private boolean positionForNextBlock() throws IOException { +1085 // Methods are small so they get inlined because they are 'hot'. +1086 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); +1087 if (this.curBlock.getOffset() >= lastDataBlockOffset) { +1088 setNonSeekedState(); +1089 return false; +1090 } +1091 return isNextBlock(); +1092 } +1093 +1094 +1095 private boolean isNextBlock() throws IOException { +1096 // Methods are small so they get inlined because they are 'hot'. +1097 HFileBlock nextBlock = readNextDataBlock(); +1098 if (nextBlock == null) { +1099 setNonSeekedState(); +1100 return false; +1101 } +1102 updateCurrentBlock(nextBlock); +1103 return true; +1104 } 1105 -1106 // We are still in the same block. -1107 readKeyValueLen(); -1108 return true; -1109 } -1110 -1111 /** -1112 * Go to the next key/value in the block section. Loads the next block if -1113 * necessary. If successful, {@link #getKey()} and {@link #getValue()} can -1114 * be called. -1115 * -1116 * @return true if successfully navigated to the next key/value -1117 */ -1118 @Override -1119 public boolean next() throws IOException { -1120 // This is a hot method so extreme measures taken to ensure it is small and inlineable. -1121 // Checked by setting: -XX:+UnlockDiagnosticVMOptions -XX:+PrintInlining -XX:+PrintCompilation -1122 assertSeeked(); -1123 positionThisBlockBuffer(); -1124 return _next(); -1125 } -1126 -1127 /** -1128 * Positions this scanner at the start of the file. -1129 * -1130 * @return false if empty file; i.e. a call to next would return false and -1131 * the current key and value are undefined. -1132 * @throws IOException -1133 */ -1134 @Override -1135 public boolean seekTo() throws IOException { -1136 if (reader == null) { -1137 return false; -1138 } -1139 -1140 if (reader.getTrailer().getEntryCount() == 0) { -1141 // No data blocks. -1142 return false; -1143 } -1144 -1145 long firstDataBlockOffset = reader.getTrailer().getFirstDataBlockOffset(); -1146 if (curBlock != null -1147 && curBlock.getOffset() == firstDataBlockOffset) { -1148 return processFirstDataBlock(); +1106 private final boolean _next() throws IOException { +1107 // Small method so can be inlined. It is a hot one. +1108 if (blockBuffer.remaining() <= 0) { +1109 return positionForNextBlock(); +1110 } +1111 +1112 // We are still in the same block. +1113 readKeyValueLen(); +1114 return true; +1115 } +1116 +1117 /** +1118 * Go to the next key/value in the block section. Loads the next block if +1119 * necessary. If successful, {@link #getKey()} and {@link #getValue()} can +1120 * be called. +1121 * +1122 * @return true if successfully navigated to the next key/value +1123 */ +1124 @Override +1125 public boolean next() throws IOException { +1126 // This is a hot method so extreme measures taken to ensure it is small and inlineable. +1127 // Checked by setting: -XX:+UnlockDiagnosticVMOptions -XX:+PrintInlining -XX:+PrintCompilation +1128 assertSeeked(); +1129 positionThisBlockBuffer(); +1130 return _next(); +1131 } +1132 +1133 /** +1134 * Positions this scanner at the start of the file. +1135 * +1136 * @return false if empty file; i.e. a call to next would return false and +1137 * the current key and value are undefined. +1138 * @throws IOException +1139 */ +1140 @Override +1141 public boolean seekTo() throws IOException { +1142 if (reader == null) { +1143 return false; +1144 } +1145 +1146 if (reader.getTrailer().getEntryCount() == 0) { +1147 // No data blocks. +1148 return false; 1149 } 1150 -1151 readAndUpdateNewBlock(firstDataBlockOffset); -1152 return true; -1153 } -1154 -1155 protected boolean processFirstDataBlock() throws IOException{ -1156 blockBuffer.rewind(); -1157 readKeyValueLen(); +1151 long firstDataBlockOffset = reader.getTrailer().getFirstDataBlockOffset(); +1152 if (curBlock != null +1153 && curBlock.getOffset() == firstDataBlockOffset) { +1154 return processFirstDataBlock(); +1155 } +1156 +1157 readAndUpdateNewBlock(firstDataBlockOffset); 1158 return true; 1159 } 1160 -1161 protected void readAndUpdateNewBlock(long firstDataBlockOffset) throws IOException, -1162 CorruptHFileException { -1163 HFileBlock newBlock = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread, -1164 isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding()); -1165 if (newBlock.getOffset() < 0) { -1166 throw new IOException("Invalid block offset: " + newBlock.getOffset()); -1167 } -1168 updateCurrentBlock(newBlock); -1169 } -1170 -1171 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey, -1172 boolean rewind, Cell key, boolean seekBefore) throws IOException { -1173 if (this.curBlock == null -1174 || this.curBlock.getOffset() != seekToBlock.getOffset()) { -1175 updateCurrentBlock(seekToBlock); -1176 } else if (rewind) { -1177 blockBuffer.rewind(); -1178 } -1179 -1180 // Update the nextIndexedKey -1181 this.nextIndexedKey = nextIndexedKey; -1182 return blockSeek(key, seekBefore); -1183 } -1184 -1185 /** -1186 * @param v -1187 * @return True if v &lt; 0 or v &gt; current block buffer limit. -1188 */ -1189 protected final boolean checkLen(final int v) { -1190 return v < 0 || v > this.blockBuffer.limit(); -1191 } -1192 -1193 /** -1194 * Check key and value lengths are wholesome. -1195 */ -1196 protected final void checkKeyValueLen() { -1197 if (checkLen(this.currKeyLen) || checkLen(this.currValueLen)) { -1198 throw new IllegalStateException("Invalid currKeyLen " + this.currKeyLen -1199 + " or currValueLen " + this.currValueLen + ". Block offset: " -1200 + this.curBlock.getOffset() + ", block length: " -1201 + this.blockBuffer.limit() + ", position: " + this.blockBuffer.position() -1202 + " (without header)."); -1203 } -1204 } -1205 -1206 /** -1207 * Updates the current block to be the given {@link HFileBlock}. Seeks to -1208 * the the first key/value pair. -1209 * -1210 * @param newBlock the block to make current -1211 */ -1212 protected void updateCurrentBlock(HFileBlock newBlock) throws IOException { -1213 // Set the active block on the reader -1214 // sanity check -1215 if (newBlock.getBlockType() != BlockType.DATA) { -1216 throw new IllegalStateException("ScannerV2 works only on data " + "blocks, got " -1217 + newBlock.getBlockType() + "; " + "fileName=" + reader.getName() -1218 + ", " + "dataBlockEncoder=" + reader.getDataBlockEncoding() + ", " + "isCompaction=" -1219 + isCompaction); -1220 } -1221 -1222 updateCurrBlockRef(newBlock); -1223 blockBuffer = newBlock.getBufferWithoutHeader(); -1224 readKeyValueLen(); -1225 blockFetches++; -1226 -1227 // Reset the next indexed key -1228 this.nextIndexedKey = null; -1229 } -1230 -1231 protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) { -1232 ByteBuff buffer = curBlock.getBufferWithoutHeader(); -1233 // It is safe to manipulate this buffer because we own the buffer object. -1234 buffer.rewind(); -1235 int klen = buffer.getInt(); -1236 buffer.skip(Bytes.SIZEOF_INT);// Skip value len part -1237 ByteBuffer keyBuff = buffer.asSubByteBuffer(klen); -1238 if (keyBuff.hasArray()) { -1239 return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset() -1240 + keyBuff.position(), klen); -1241 } else { -1242 return new ByteBufferedKeyOnlyKeyValue(keyBuff, keyBuff.position(), klen); -1243 } -1244 } -1245 -1246 @Override -1247 public String getKeyString() { -1248 return CellUtil.toString(getKey(), false); -1249 } -1250 -1251 @Override -1252 public String getValueString() { -1253 return ByteBufferUtils.toStringBinary(getValue()); -1254 } -1255 -1256 public int compareKey(CellComparator comparator, Cell key) { -1257 blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen, pair); -1258 this.bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), currKeyLen); -1259 return comparator.compareKeyIgnoresMvcc(key, this.bufBackedKeyOnlyKv); +1161 protected boolean processFirstDataBlock() throws IOException{ +1162 blockBuffer.rewind(); +1163 readKeyValueLen(); +1164 return true; +1165 } +1166 +1167 protected void readAndUpdateNewBlock(long firstDataBlockOffset) throws IOException, +1168 CorruptHFileException { +1169 HFileBlock newBlock = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread, +1170 isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding()); +1171 if (newBlock.getOffset() < 0) { +1172 throw new IOException("Invalid block offset: " + newBlock.getOffset()); +1173 } +1174 updateCurrentBlock(newBlock); +1175 } +1176 +1177 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey, +1178 boolean rewind, Cell key, boolean seekBefore) throws IOException { +1179 if (this.curBlock == null +1180 || this.curBlock.getOffset() != seekToBlock.getOffset()) { +1181 updateCurrentBlock(seekToBlock); +1182 } else if (rewind) { +1183 blockBuffer.rewind(); +1184 } +1185 +1186 // Update the nextIndexedKey +1187 this.nextIndexedKey = nextIndexedKey; +1188 return blockSeek(key, seekBefore); +1189 } +1190 +1191 /** +1192 * @param v +1193 * @return True if v &lt; 0 or v &gt; current block buffer limit. +1194 */ +1195 protected final boolean checkLen(final int v) { +1196 return v < 0 || v > this.blockBuffer.limit(); +1197 } +1198 +1199 /** +1200 * Check key and value lengths are wholesome. +1201 */ +1202 protected final void checkKeyValueLen() { +1203 if (checkLen(this.currKeyLen) || checkLen(this.currValueLen)) { +1204 throw new IllegalStateException("Invalid currKeyLen " + this.currKeyLen +1205 + " or currValueLen " + this.currValueLen + ". Block offset: " +1206 + this.curBlock.getOffset() + ", block length: " +1207 + this.blockBuffer.limit() + ", position: " + this.blockBuffer.position() +1208 + " (without header)."); +1209 } +1210 } +1211 +1212 /** +1213 * Updates the current block to be the given {@link HFileBlock}. Seeks to +1214 * the the first key/value pair. +1215 * +1216 * @param newBlock the block to make current +1217 */ +1218 protected void updateCurrentBlock(HFileBlock newBlock) throws IOException { +1219 // Set the active block on the reader +1220 // sanity check +1221 if (newBlock.getBlockType() != BlockType.DATA) { +1222 throw new IllegalStateException("ScannerV2 works only on data " + "blocks, got " +1223 + newBlock.getBlockType() + "; " + "fileName=" + reader.getName() +1224 + ", " + "dataBlockEncoder=" + reader.getDataBlockEncoding() + ", " + "isCompaction=" +1225 + isCompaction); +1226 } +1227 +1228 updateCurrBlockRef(newBlock); +1229 blockBuffer = newBlock.getBufferWithoutHeader(); +1230 readKeyValueLen(); +1231 blockFetches++; +1232 +1233 // Reset the next indexed key +1234 this.nextIndexedKey = null; +1235 } +1236 +1237 protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) { +1238 ByteBuff buffer = curBlock.getBufferWithoutHeader(); +1239 // It is safe to manipulate this buffer because we own the buffer object. +1240 buffer.rewind(); +1241 int klen = buffer.getInt(); +1242 buffer.skip(Bytes.SIZEOF_INT);// Skip value len part +1243 ByteBuffer keyBuff = buffer.asSubByteBuffer(klen); +1244 if (keyBuff.hasArray()) { +1245 return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset() +1246 + keyBuff.position(), klen); +1247 } else { +1248 return new ByteBufferedKeyOnlyKeyValue(keyBuff, keyBuff.position(), klen); +1249 } +1250 } +1251 +1252 @Override +1253 public String getKeyString() { +1254 return CellUtil.toString(getKey(), false); +1255 } +1256 +1257 @Override +1258 public String getValueString() { +1259 return ByteBufferUtils.toStringBinary(getValue()); 1260 } 1261 -1262 @Override -1263 public void shipped() throws IOException { -1264 this.returnBlocks(false); -1265 } -1266 } +1262 public int compareKey(CellComparator comparator, Cell key) { +1263 blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen, pair); +1264 this.bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), currKeyLen); +1265 return comparator.compareKeyIgnoresMvcc(key, this.bufBackedKeyOnlyKv); +1266 } 1267 -1268 public Path getPath() { -1269 return path; -1270 } -1271 -1272 @Override -1273 public DataBlockEncoding getDataBlockEncoding() { -1274 return dataBlockEncoder.getDataBlockEncoding(); -1275 } -1276 -1277 @Override -1278 public Configuration getConf() { -1279 return conf; -1280 } -1281 -1282 @Override -1283 public void setConf(Configuration conf) { -1284 this.conf = conf; -1285 } -1286 -1287 /** Minor versions in HFile starting with this number have hbase checksums */ -1288 public static final int MINOR_VERSION_WITH_CHECKSUM = 1; -1289 /** In HFile minor version that does not support checksums */ -1290 public static final int MINOR_VERSION_NO_CHECKSUM = 0; -1291 -1292 /** HFile minor version that introduced pbuf filetrailer */ -1293 public static final int PBUF_TRAILER_MINOR_VERSION = 2; -1294 -1295 /** -1296 * The size of a (key length, value length) tuple that prefixes each entry in -1297 * a data block. -1298 */ -1299 public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT; +1268 @Override +1269 public void shipped() throws IOException { +1270 this.returnBlocks(false); +1271 } +1272 } +1273 +1274 public Path getPath() { +1275 return path; +1276 } +1277 +1278 @Override +1279 public DataBlockEncoding getDataBlockEncoding() { +1280 return dataBlockEncoder.getDataBlockEncoding(); +1281 } +1282 +1283 @Override +1284 public Configuration getConf() { +1285 return conf; +1286 } +1287 +1288 @Override +1289 public void setConf(Configuration conf) { +1290 this.conf = conf; +1291 } +1292 +1293 /** Minor versions in HFile starting with this number have hbase checksums */ +1294 public static final int MINOR_VERSION_WITH_CHECKSUM = 1; +1295 /** In HFile minor version that does not support checksums */ +1296 public static final int MINOR_VERSION_NO_CHECKSUM = 0; +1297 +1298 /** HFile minor version that introduced pbuf filetrailer */ +1299 public static final int PBUF_TRAILER_MINOR_VERSION = 2; 1300 -1301 private boolean includesMemstoreTS = false; -1302 protected boolean decodeMemstoreTS = false; -1303 -1304 -1305 public boolean isDecodeMemstoreTS() { -1306 return this.decodeMemstoreTS; -1307 } -1308 -1309 public boolean shouldIncludeMemstoreTS() { -1310 return includesMemstoreTS; -1311 } -1312 -1313 /** -1314 * Retrieve block from cache. Validates the retrieved block's type vs {@code expectedBlockType} -1315 * and its encoding vs. {@code expectedDataBlockEncoding}. Unpacks the block as necessary. -1316 */ -1317 private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock, -1318 boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType, -1319 DataBlockEncoding expectedDataBlockEncoding) throws IOException { -1320 // Check cache for block. If found return. -1321 if (cacheConf.isBlockCacheEnabled()) { -1322 BlockCache cache = cacheConf.getBlockCache(); -1323 HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, -1324 updateCacheMetrics); -1325 if (cachedBlock != null) { -1326 if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { -1327 HFileBlock compressedBlock = cachedBlock; -1328 cachedBlock = compressedBlock.unpack(hfileContext, fsBlockReader); -1329 // In case of compressed block after unpacking we can return the compressed block -1330 if (compressedBlock != cachedBlock) { -1331 cache.returnBlock(cacheKey, compressedBlock); -1332 } -1333 } -1334 validateBlockType(cachedBlock, expectedBlockType); -1335 -1336 if (expectedDataBlockEncoding == null) { -1337 return cachedBlock; -1338 } -1339 DataBlockEncoding actualDataBlockEncoding = -1340 cachedBlock.getDataBlockEncoding(); -1341 // Block types other than data blocks always have -1342 // DataBlockEncoding.NONE. To avoid false negative cache misses, only -1343 // perform this check if cached block is a data block. -1344 if (cachedBlock.getBlockType().isData() && -1345 !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) { -1346 // This mismatch may happen if a Scanner, which is used for say a -1347 // compaction, tries to read an encoded block from the block cache. -1348 // The reverse might happen when an EncodedScanner tries to read -1349 // un-encoded blocks which were cached earlier. -1350 // -1351 // Because returning a data block with an implicit BlockType mismatch -1352 // will cause the requesting scanner to throw a disk read should be -1353 // forced here. This will potentially cause a significant number of -1354 // cache misses, so update so we should keep track of this as it might -1355 // justify the work on a CompoundScanner. -1356 if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) && -1357 !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) { -1358 // If the block is encoded but the encoding does not match the -1359 // expected encoding it is likely the encoding was changed but the -1360 // block was not yet evicted. Evictions on file close happen async -1361 // so blocks with the old encoding still linger in cache for some -1362 // period of time. This event should be rare as it only happens on -1363 // schema definition change. -1364 LOG.info("Evicting cached block with key " + cacheKey + -1365 " because of a data block encoding mismatch" + -1366 "; expected: " + expectedDataBlockEncoding + -1367 ", actual: " + actualDataBlockEncoding); -1368 // This is an error scenario. so here we need to decrement the -1369 // count. -1370 cache.returnBlock(cacheKey, cachedBlock); -1371 cache.evictBlock(cacheKey); -1372 } -1373 return null; -1374 } -1375 return cachedBlock; -1376 } -1377 } -1378 return null; -1379 } -1380 -1381 /** -1382 * @param metaBlockName -1383 * @param cacheBlock Add block to cache, if found -1384 * @return block wrapped in a ByteBuffer, with header skipped -1385 * @throws IOException -1386 */ -1387 @Override -1388 public HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock) -1389 throws IOException { -1390 if (trailer.getMetaIndexCount() == 0) { -1391 return null; // there are no meta blocks -1392 } -1393 if (metaBlockIndexReader == null) { -1394 throw new IOException("Meta index not loaded"); -1395 } -1396 -1397 byte[] mbname = Bytes.toBytes(metaBlockName); -1398 int block = metaBlockIndexReader.rootBlockContainingKey(mbname, -1399 0, mbname.length); -1400 if (block == -1) -1401 return null; -1402 long blockSize = metaBlockIndexReader.getRootBlockDataSize(block); -1403 -1404 // Per meta key from any given file, synchronize reads for said block. This -1405 // is OK to do for meta blocks because the meta block index is always -1406 // single-level. -1407 synchronized (metaBlockIndexReader -1408 .getRootBlockKey(block)) { -1409 // Check cache for block. If found return. -1410 long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block); -1411 BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset, -1412 this.isPrimaryReplicaReader()); -1413 -1414 cacheBlock &= cacheConf.shouldCacheDataOnRead(); -1415 if (cacheConf.isBlockCacheEnabled()) { -1416 HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true, -1417 BlockType.META, null); -1418 if (cachedBlock != null) { -1419 assert cachedBlock.isUnpacked() : "Packed block leak."; -1420 // Return a distinct 'shallow copy' of the block, -1421 // so pos does not get messed by the scanner -1422 return cachedBlock; -1423 } -1424 // Cache Miss, please load. -1425 } -1426 -1427 HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset, -1428 blockSize, -1, true).unpack(hfileContext, fsBlockReader); -1429 -1430 // Cache the block -1431 if (cacheBlock) { -1432 cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock, -1433 cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1()); -1434 } +1301 /** +1302 * The size of a (key length, value length) tuple that prefixes each entry in +1303 * a data block. +1304 */ +1305 public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT; +1306 +1307 private boolean includesMemstoreTS = false; +1308 protected boolean decodeMemstoreTS = false; +1309 +1310 +1311 public boolean isDecodeMemstoreTS() { +1312 return this.decodeMemstoreTS; +1313 } +1314 +1315 public boolean shouldIncludeMemstoreTS() { +1316 return includesMemstoreTS; +1317 } +1318 +1319 /** +1320 * Retrieve block from cache. Validates the retrieved block's type vs {@code expectedBlockType} +1321 * and its encoding vs. {@code expectedDataBlockEncoding}. Unpacks the block as necessary. +1322 */ +1323 private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock, +1324 boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType, +1325 DataBlockEncoding expectedDataBlockEncoding) throws IOException { +1326 // Check cache for block. If found return. +1327 if (cacheConf.isBlockCacheEnabled()) { +1328 BlockCache cache = cacheConf.getBlockCache(); +1329 HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, +1330 updateCacheMetrics); +1331 if (cachedBlock != null) { +1332 if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { +1333 HFileBlock compressedBlock = cachedBlock; +1334 cachedBlock = compressedBlock.unpack(hfileContext, fsBlockReader); +1335 // In case of compressed block after unpacking we can return the compressed block +1336 if (compressedBlock != cachedBlock) { +1337 cache.returnBlock(cacheKey, compressedBlock); +1338 } +1339 } +1340 validateBlockType(cachedBlock, expectedBlockType); +1341 +1342 if (expectedDataBlockEncoding == null) { +1343 return cachedBlock; +1344 } +1345 DataBlockEncoding actualDataBlockEncoding = +1346 cachedBlock.getDataBlockEncoding(); +1347 // Block types other than data blocks always have +1348 // DataBlockEncoding.NONE. To avoid false negative cache misses, only +1349 // perform this check if cached block is a data block. +1350 if (cachedBlock.getBlockType().isData() && +1351 !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) { +1352 // This mismatch may happen if a Scanner, which is used for say a +1353 // compaction, tries to read an encoded block from the block cache. +1354 // The reverse might happen when an EncodedScanner tries to read +1355 // un-encoded blocks which were cached earlier. +1356 // +1357 // Because returning a data block with an implicit BlockType mismatch +1358 // will cause the requesting scanner to throw a disk read should be +1359 // forced here. This will potentially cause a significant number of +1360 // cache misses, so update so we should keep track of this as it might +1361 // justify the work on a CompoundScanner. +1362 if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) && +1363 !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) { +1364 // If the block is encoded but the encoding does not match the +1365 // expected encoding it is likely the encoding was changed but the +1366 // block was not yet evicted. Evictions on file close happen async +1367 // so blocks with the old encoding still linger in cache for some +1368 // period of time. This event should be rare as it only happens on +1369 // schema definition change. +1370 LOG.info("Evicting cached block with key " + cacheKey + +1371 " because of a data block encoding mismatch" + +1372 "; expected: " + expectedDataBlockEncoding + +1373 ", actual: " + actualDataBlockEncoding); +1374 // This is an error scenario. so here we need to decrement the +1375 // count. +1376 cache.returnBlock(cacheKey, cachedBlock); +1377 cache.evictBlock(cacheKey); +1378 } +1379 return null; +1380 } +1381 return cachedBlock; +1382 } +1383 } +1384 return null; +1385 } +1386 +1387 /** +1388 * @param metaBlockName +1389 * @param cacheBlock Add block to cache, if found +1390 * @return block wrapped in a ByteBuffer, with header skipped +1391 * @throws IOException +1392 */ +1393 @Override +1394 public HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock) +1395 throws IOException { +1396 if (trailer.getMetaIndexCount() == 0) { +1397 return null; // there are no meta blocks +1398 } +1399 if (metaBlockIndexReader == null) { +1400 throw new IOException("Meta index not loaded"); +1401 } +1402 +1403 byte[] mbname = Bytes.toBytes(metaBlockName); +1404 int block = metaBlockIndexReader.rootBlockContainingKey(mbname, +1405 0, mbname.length); +1406 if (block == -1) +1407 return null; +1408 long blockSize = metaBlockIndexReader.getRootBlockDataSize(block); +1409 +1410 // Per meta key from any given file, synchronize reads for said block. This +1411 // is OK to do for meta blocks because the meta block index is always +1412 // single-level. +1413 synchronized (metaBlockIndexReader +1414 .getRootBlockKey(block)) { +1415 // Check cache for block. If found return. +1416 long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block); +1417 BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset, +1418 this.isPrimaryReplicaReader()); +1419 +1420 cacheBlock &= cacheConf.shouldCacheDataOnRead(); +1421 if (cacheConf.isBlockCacheEnabled()) { +1422 HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true, +1423 BlockType.META, null); +1424 if (cachedBlock != null) { +1425 assert cachedBlock.isUnpacked() : "Packed block leak."; +1426 // Return a distinct 'shallow copy' of the block, +1427 // so pos does not get messed by the scanner +1428 return cachedBlock; +1429 } +1430 // Cache Miss, please load. +1431 } +1432 +1433 HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,