hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From git-site-r...@apache.org
Subject [22/29] hbase-site git commit: Published site at .
Date Mon, 09 Oct 2017 15:13:06 GMT
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e6ae7c3e/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html
index df1cfa8..df8ff34 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html
@@ -1042,132 +1042,132 @@
 <span class="sourceLineNo">1034</span>   * @param includesTag - includesTag or not<a name="line.1034"></a>
 <span class="sourceLineNo">1035</span>   * @return Writer for a new StoreFile in the tmp dir.<a name="line.1035"></a>
 <span class="sourceLineNo">1036</span>   */<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span>  public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span>      boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag,<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span>      boolean shouldDropBehind) throws IOException {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span>    return createWriterInTmp(maxKeyCount, compression, isCompaction, includeMVCCReadpoint,<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span>      includesTag, shouldDropBehind, null);<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span>  }<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span><a name="line.1043"></a>
-<span class="sourceLineNo">1044</span>  /**<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span>   * @param maxKeyCount<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span>   * @param compression Compression algorithm to use<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span>   * @param isCompaction whether we are creating a new file in a compaction<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span>   * @param includeMVCCReadpoint - whether to include MVCC or not<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span>   * @param includesTag - includesTag or not<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span>   * @return Writer for a new StoreFile in the tmp dir.<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span>   */<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span>  // TODO : allow the Writer factory to create Writers of ShipperListener type only in case of<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span>  // compaction<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span>  public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span>      boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag,<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span>      boolean shouldDropBehind, TimeRangeTracker trt) throws IOException {<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span>    final CacheConfig writerCacheConf;<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span>    if (isCompaction) {<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span>      // Don't cache data on write on compactions.<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span>      writerCacheConf = new CacheConfig(cacheConf);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span>      writerCacheConf.setCacheDataOnWrite(false);<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span>    } else {<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span>      writerCacheConf = cacheConf;<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span>    }<a name="line.1064"></a>
-<span class="sourceLineNo">1065</span>    InetSocketAddress[] favoredNodes = null;<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span>    if (region.getRegionServerServices() != null) {<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span>      favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span>          region.getRegionInfo().getEncodedName());<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span>    }<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span>    HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span>      cryptoContext);<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span>    Path familyTempDir = new Path(fs.getTempDir(), family.getNameAsString());<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span>    StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf,<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span>        this.getFileSystem())<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span>            .withOutputDir(familyTempDir)<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span>            .withComparator(comparator)<a name="line.1076"></a>
-<span class="sourceLineNo">1077</span>            .withBloomType(family.getBloomFilterType())<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span>            .withMaxKeyCount(maxKeyCount)<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span>            .withFavoredNodes(favoredNodes)<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span>            .withFileContext(hFileContext)<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span>            .withShouldDropCacheBehind(shouldDropBehind);<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span>    if (trt != null) {<a name="line.1082"></a>
-<span class="sourceLineNo">1083</span>      builder.withTimeRangeTracker(trt);<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span>    }<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span>    return builder.build();<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span>  }<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span><a name="line.1087"></a>
-<span class="sourceLineNo">1088</span>  private HFileContext createFileContext(Compression.Algorithm compression,<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span>      boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span>    if (compression == null) {<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span>      compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span>    }<a name="line.1092"></a>
-<span class="sourceLineNo">1093</span>    HFileContext hFileContext = new HFileContextBuilder()<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span>                                .withIncludesMvcc(includeMVCCReadpoint)<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span>                                .withIncludesTags(includesTag)<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span>                                .withCompression(compression)<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span>                                .withCompressTags(family.isCompressTags())<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span>                                .withChecksumType(checksumType)<a name="line.1098"></a>
-<span class="sourceLineNo">1099</span>                                .withBytesPerCheckSum(bytesPerChecksum)<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span>                                .withBlockSize(blocksize)<a name="line.1100"></a>
-<span class="sourceLineNo">1101</span>                                .withHBaseCheckSum(true)<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span>                                .withDataBlockEncoding(family.getDataBlockEncoding())<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span>                                .withEncryptionContext(cryptoContext)<a name="line.1103"></a>
-<span class="sourceLineNo">1104</span>                                .withCreateTime(EnvironmentEdgeManager.currentTime())<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span>                                .build();<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span>    return hFileContext;<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span>  }<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span><a name="line.1108"></a>
-<span class="sourceLineNo">1109</span><a name="line.1109"></a>
-<span class="sourceLineNo">1110</span>  private long getTotalSize(Collection&lt;HStoreFile&gt; sfs) {<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span>    return sfs.stream().mapToLong(sf -&gt; sf.getReader().length()).sum();<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span>  }<a name="line.1112"></a>
-<span class="sourceLineNo">1113</span><a name="line.1113"></a>
-<span class="sourceLineNo">1114</span>  /**<a name="line.1114"></a>
-<span class="sourceLineNo">1115</span>   * Change storeFiles adding into place the Reader produced by this new flush.<a name="line.1115"></a>
-<span class="sourceLineNo">1116</span>   * @param sfs Store files<a name="line.1116"></a>
-<span class="sourceLineNo">1117</span>   * @param snapshotId<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span>   * @throws IOException<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span>   * @return Whether compaction is required.<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span>   */<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span>  private boolean updateStorefiles(List&lt;HStoreFile&gt; sfs, long snapshotId) throws IOException {<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span>    this.lock.writeLock().lock();<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span>    try {<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span>      this.storeEngine.getStoreFileManager().insertNewFiles(sfs);<a name="line.1124"></a>
-<span class="sourceLineNo">1125</span>      if (snapshotId &gt; 0) {<a name="line.1125"></a>
-<span class="sourceLineNo">1126</span>        this.memstore.clearSnapshot(snapshotId);<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span>      }<a name="line.1127"></a>
-<span class="sourceLineNo">1128</span>    } finally {<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span>      // We need the lock, as long as we are updating the storeFiles<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span>      // or changing the memstore. Let us release it before calling<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span>      // notifyChangeReadersObservers. See HBASE-4485 for a possible<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span>      // deadlock scenario that could have happened if continue to hold<a name="line.1132"></a>
-<span class="sourceLineNo">1133</span>      // the lock.<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span>      this.lock.writeLock().unlock();<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span>    }<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span>    // notify to be called here - only in case of flushes<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span>    notifyChangedReadersObservers(sfs);<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span>    if (LOG.isTraceEnabled()) {<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span>      long totalSize = getTotalSize(sfs);<a name="line.1139"></a>
-<span class="sourceLineNo">1140</span>      String traceMessage = "FLUSH time,count,size,store size,store files ["<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span>          + EnvironmentEdgeManager.currentTime() + "," + sfs.size() + "," + totalSize<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span>          + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span>      LOG.trace(traceMessage);<a name="line.1143"></a>
+<span class="sourceLineNo">1037</span>  // TODO : allow the Writer factory to create Writers of ShipperListener type only in case of<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span>  // compaction<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span>  public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span>      boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag,<a name="line.1040"></a>
+<span class="sourceLineNo">1041</span>      boolean shouldDropBehind) throws IOException {<a name="line.1041"></a>
+<span class="sourceLineNo">1042</span>    final CacheConfig writerCacheConf;<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span>    if (isCompaction) {<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span>      // Don't cache data on write on compactions.<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span>      writerCacheConf = new CacheConfig(cacheConf);<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span>      writerCacheConf.setCacheDataOnWrite(false);<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span>    } else {<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span>      writerCacheConf = cacheConf;<a name="line.1048"></a>
+<span class="sourceLineNo">1049</span>    }<a name="line.1049"></a>
+<span class="sourceLineNo">1050</span>    InetSocketAddress[] favoredNodes = null;<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span>    if (region.getRegionServerServices() != null) {<a name="line.1051"></a>
+<span class="sourceLineNo">1052</span>      favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(<a name="line.1052"></a>
+<span class="sourceLineNo">1053</span>          region.getRegionInfo().getEncodedName());<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span>    }<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span>    HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,<a name="line.1055"></a>
+<span class="sourceLineNo">1056</span>      cryptoContext);<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span>    Path familyTempDir = new Path(fs.getTempDir(), family.getNameAsString());<a name="line.1057"></a>
+<span class="sourceLineNo">1058</span>    StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf,<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span>        this.getFileSystem())<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span>            .withOutputDir(familyTempDir)<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span>            .withComparator(comparator)<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span>            .withBloomType(family.getBloomFilterType())<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span>            .withMaxKeyCount(maxKeyCount)<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span>            .withFavoredNodes(favoredNodes)<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span>            .withFileContext(hFileContext)<a name="line.1065"></a>
+<span class="sourceLineNo">1066</span>            .withShouldDropCacheBehind(shouldDropBehind);<a name="line.1066"></a>
+<span class="sourceLineNo">1067</span>    return builder.build();<a name="line.1067"></a>
+<span class="sourceLineNo">1068</span>  }<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span><a name="line.1069"></a>
+<span class="sourceLineNo">1070</span>  private HFileContext createFileContext(Compression.Algorithm compression,<a name="line.1070"></a>
+<span class="sourceLineNo">1071</span>      boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span>    if (compression == null) {<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span>      compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span>    }<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span>    HFileContext hFileContext = new HFileContextBuilder()<a name="line.1075"></a>
+<span class="sourceLineNo">1076</span>                                .withIncludesMvcc(includeMVCCReadpoint)<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span>                                .withIncludesTags(includesTag)<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span>                                .withCompression(compression)<a name="line.1078"></a>
+<span class="sourceLineNo">1079</span>                                .withCompressTags(family.isCompressTags())<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span>                                .withChecksumType(checksumType)<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span>                                .withBytesPerCheckSum(bytesPerChecksum)<a name="line.1081"></a>
+<span class="sourceLineNo">1082</span>                                .withBlockSize(blocksize)<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span>                                .withHBaseCheckSum(true)<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span>                                .withDataBlockEncoding(family.getDataBlockEncoding())<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span>                                .withEncryptionContext(cryptoContext)<a name="line.1085"></a>
+<span class="sourceLineNo">1086</span>                                .withCreateTime(EnvironmentEdgeManager.currentTime())<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span>                                .build();<a name="line.1087"></a>
+<span class="sourceLineNo">1088</span>    return hFileContext;<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span>  }<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span><a name="line.1090"></a>
+<span class="sourceLineNo">1091</span><a name="line.1091"></a>
+<span class="sourceLineNo">1092</span>  private long getTotalSize(Collection&lt;HStoreFile&gt; sfs) {<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span>    return sfs.stream().mapToLong(sf -&gt; sf.getReader().length()).sum();<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span>  }<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span><a name="line.1095"></a>
+<span class="sourceLineNo">1096</span>  /**<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span>   * Change storeFiles adding into place the Reader produced by this new flush.<a name="line.1097"></a>
+<span class="sourceLineNo">1098</span>   * @param sfs Store files<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span>   * @param snapshotId<a name="line.1099"></a>
+<span class="sourceLineNo">1100</span>   * @throws IOException<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span>   * @return Whether compaction is required.<a name="line.1101"></a>
+<span class="sourceLineNo">1102</span>   */<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span>  private boolean updateStorefiles(List&lt;HStoreFile&gt; sfs, long snapshotId) throws IOException {<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span>    this.lock.writeLock().lock();<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span>    try {<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span>      this.storeEngine.getStoreFileManager().insertNewFiles(sfs);<a name="line.1106"></a>
+<span class="sourceLineNo">1107</span>      if (snapshotId &gt; 0) {<a name="line.1107"></a>
+<span class="sourceLineNo">1108</span>        this.memstore.clearSnapshot(snapshotId);<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span>      }<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span>    } finally {<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span>      // We need the lock, as long as we are updating the storeFiles<a name="line.1111"></a>
+<span class="sourceLineNo">1112</span>      // or changing the memstore. Let us release it before calling<a name="line.1112"></a>
+<span class="sourceLineNo">1113</span>      // notifyChangeReadersObservers. See HBASE-4485 for a possible<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span>      // deadlock scenario that could have happened if continue to hold<a name="line.1114"></a>
+<span class="sourceLineNo">1115</span>      // the lock.<a name="line.1115"></a>
+<span class="sourceLineNo">1116</span>      this.lock.writeLock().unlock();<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span>    }<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span>    // notify to be called here - only in case of flushes<a name="line.1118"></a>
+<span class="sourceLineNo">1119</span>    notifyChangedReadersObservers(sfs);<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span>    if (LOG.isTraceEnabled()) {<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span>      long totalSize = getTotalSize(sfs);<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span>      String traceMessage = "FLUSH time,count,size,store size,store files ["<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span>          + EnvironmentEdgeManager.currentTime() + "," + sfs.size() + "," + totalSize<a name="line.1123"></a>
+<span class="sourceLineNo">1124</span>          + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span>      LOG.trace(traceMessage);<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span>    }<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span>    return needsCompaction();<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span>  }<a name="line.1128"></a>
+<span class="sourceLineNo">1129</span><a name="line.1129"></a>
+<span class="sourceLineNo">1130</span>  /**<a name="line.1130"></a>
+<span class="sourceLineNo">1131</span>   * Notify all observers that set of Readers has changed.<a name="line.1131"></a>
+<span class="sourceLineNo">1132</span>   * @throws IOException<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span>   */<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span>  private void notifyChangedReadersObservers(List&lt;HStoreFile&gt; sfs) throws IOException {<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span>    for (ChangedReadersObserver o : this.changedReaderObservers) {<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span>      List&lt;KeyValueScanner&gt; memStoreScanners;<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span>      this.lock.readLock().lock();<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span>      try {<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span>        memStoreScanners = this.memstore.getScanners(o.getReadPoint());<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span>      } finally {<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span>        this.lock.readLock().unlock();<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span>      }<a name="line.1142"></a>
+<span class="sourceLineNo">1143</span>      o.updateReaders(sfs, memStoreScanners);<a name="line.1143"></a>
 <span class="sourceLineNo">1144</span>    }<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span>    return needsCompaction();<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span>  }<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span><a name="line.1147"></a>
-<span class="sourceLineNo">1148</span>  /**<a name="line.1148"></a>
-<span class="sourceLineNo">1149</span>   * Notify all observers that set of Readers has changed.<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span>   * @throws IOException<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span>   */<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span>  private void notifyChangedReadersObservers(List&lt;HStoreFile&gt; sfs) throws IOException {<a name="line.1152"></a>
-<span class="sourceLineNo">1153</span>    for (ChangedReadersObserver o : this.changedReaderObservers) {<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span>      List&lt;KeyValueScanner&gt; memStoreScanners;<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span>      this.lock.readLock().lock();<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span>      try {<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span>        memStoreScanners = this.memstore.getScanners(o.getReadPoint());<a name="line.1157"></a>
-<span class="sourceLineNo">1158</span>      } finally {<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span>        this.lock.readLock().unlock();<a name="line.1159"></a>
-<span class="sourceLineNo">1160</span>      }<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span>      o.updateReaders(sfs, memStoreScanners);<a name="line.1161"></a>
-<span class="sourceLineNo">1162</span>    }<a name="line.1162"></a>
+<span class="sourceLineNo">1145</span>  }<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span><a name="line.1146"></a>
+<span class="sourceLineNo">1147</span>  /**<a name="line.1147"></a>
+<span class="sourceLineNo">1148</span>   * Get all scanners with no filtering based on TTL (that happens further down the line).<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span>   * @param cacheBlocks cache the blocks or not<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span>   * @param usePread true to use pread, false if not<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span>   * @param isCompaction true if the scanner is created for compaction<a name="line.1151"></a>
+<span class="sourceLineNo">1152</span>   * @param matcher the scan query matcher<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span>   * @param startRow the start row<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span>   * @param stopRow the stop row<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span>   * @param readPt the read point of the current scan<a name="line.1155"></a>
+<span class="sourceLineNo">1156</span>   * @return all scanners for this store<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span>   */<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span>  public List&lt;KeyValueScanner&gt; getScanners(boolean cacheBlocks, boolean isGet, boolean usePread,<a name="line.1158"></a>
+<span class="sourceLineNo">1159</span>      boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow, byte[] stopRow, long readPt)<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span>      throws IOException {<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span>    return getScanners(cacheBlocks, usePread, isCompaction, matcher, startRow, true, stopRow, false,<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span>      readPt);<a name="line.1162"></a>
 <span class="sourceLineNo">1163</span>  }<a name="line.1163"></a>
 <span class="sourceLineNo">1164</span><a name="line.1164"></a>
 <span class="sourceLineNo">1165</span>  /**<a name="line.1165"></a>
@@ -1177,1433 +1177,1415 @@
 <span class="sourceLineNo">1169</span>   * @param isCompaction true if the scanner is created for compaction<a name="line.1169"></a>
 <span class="sourceLineNo">1170</span>   * @param matcher the scan query matcher<a name="line.1170"></a>
 <span class="sourceLineNo">1171</span>   * @param startRow the start row<a name="line.1171"></a>
-<span class="sourceLineNo">1172</span>   * @param stopRow the stop row<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span>   * @param readPt the read point of the current scan<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span>   * @return all scanners for this store<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span>   */<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>  public List&lt;KeyValueScanner&gt; getScanners(boolean cacheBlocks, boolean isGet, boolean usePread,<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span>      boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow, byte[] stopRow, long readPt)<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>      throws IOException {<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>    return getScanners(cacheBlocks, usePread, isCompaction, matcher, startRow, true, stopRow, false,<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>      readPt);<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>  }<a name="line.1181"></a>
-<span class="sourceLineNo">1182</span><a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>  /**<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span>   * Get all scanners with no filtering based on TTL (that happens further down the line).<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>   * @param cacheBlocks cache the blocks or not<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>   * @param usePread true to use pread, false if not<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>   * @param isCompaction true if the scanner is created for compaction<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>   * @param matcher the scan query matcher<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>   * @param startRow the start row<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>   * @param includeStartRow true to include start row, false if not<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>   * @param stopRow the stop row<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span>   * @param includeStopRow true to include stop row, false if not<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>   * @param readPt the read point of the current scan<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>   * @return all scanners for this store<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>   */<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>  public List&lt;KeyValueScanner&gt; getScanners(boolean cacheBlocks, boolean usePread,<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span>      boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow, boolean includeStartRow,<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>      byte[] stopRow, boolean includeStopRow, long readPt) throws IOException {<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span>    Collection&lt;HStoreFile&gt; storeFilesToScan;<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>    List&lt;KeyValueScanner&gt; memStoreScanners;<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>    this.lock.readLock().lock();<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>    try {<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>      storeFilesToScan = this.storeEngine.getStoreFileManager().getFilesForScan(startRow,<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>        includeStartRow, stopRow, includeStopRow);<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>      memStoreScanners = this.memstore.getScanners(readPt);<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    } finally {<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span>      this.lock.readLock().unlock();<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span>    }<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span><a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>    // First the store file scanners<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span><a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>    // TODO this used to get the store files in descending order,<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span>    // but now we get them in ascending order, which I think is<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>    // actually more correct, since memstore get put at the end.<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>    List&lt;StoreFileScanner&gt; sfScanners = StoreFileScanner.getScannersForStoreFiles(storeFilesToScan,<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>      cacheBlocks, usePread, isCompaction, false, matcher, readPt);<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>    List&lt;KeyValueScanner&gt; scanners = new ArrayList&lt;&gt;(sfScanners.size() + 1);<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    scanners.addAll(sfScanners);<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span>    // Then the memstore scanners<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>    scanners.addAll(memStoreScanners);<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>    return scanners;<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>  }<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span><a name="line.1223"></a>
-<span class="sourceLineNo">1224</span>  /**<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>   * Create scanners on the given files and if needed on the memstore with no filtering based on TTL<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>   * (that happens further down the line).<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span>   * @param files the list of files on which the scanners has to be created<a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>   * @param cacheBlocks cache the blocks or not<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>   * @param usePread true to use pread, false if not<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>   * @param isCompaction true if the scanner is created for compaction<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>   * @param matcher the scan query matcher<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>   * @param startRow the start row<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span>   * @param stopRow the stop row<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>   * @param readPt the read point of the current scan<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>   * @param includeMemstoreScanner true if memstore has to be included<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>   * @return scanners on the given files and on the memstore if specified<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>   */<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span>  public List&lt;KeyValueScanner&gt; getScanners(List&lt;HStoreFile&gt; files, boolean cacheBlocks,<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span>      boolean isGet, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher,<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>      byte[] startRow, byte[] stopRow, long readPt, boolean includeMemstoreScanner)<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>      throws IOException {<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>    return getScanners(files, cacheBlocks, usePread, isCompaction, matcher, startRow, true, stopRow,<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>      false, readPt, includeMemstoreScanner);<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span>  }<a name="line.1244"></a>
-<span class="sourceLineNo">1245</span><a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>  /**<a name="line.1246"></a>
-<span class="sourceLineNo">1247</span>   * Create scanners on the given files and if needed on the memstore with no filtering based on TTL<a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>   * (that happens further down the line).<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>   * @param files the list of files on which the scanners has to be created<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span>   * @param cacheBlocks ache the blocks or not<a name="line.1250"></a>
-<span class="sourceLineNo">1251</span>   * @param usePread true to use pread, false if not<a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>   * @param isCompaction true if the scanner is created for compaction<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>   * @param matcher the scan query matcher<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span>   * @param startRow the start row<a name="line.1254"></a>
-<span class="sourceLineNo">1255</span>   * @param includeStartRow true to include start row, false if not<a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>   * @param stopRow the stop row<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>   * @param includeStopRow true to include stop row, false if not<a name="line.1257"></a>
-<span class="sourceLineNo">1258</span>   * @param readPt the read point of the current scan<a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>   * @param includeMemstoreScanner true if memstore has to be included<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>   * @return scanners on the given files and on the memstore if specified<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>   */<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>  public List&lt;KeyValueScanner&gt; getScanners(List&lt;HStoreFile&gt; files, boolean cacheBlocks,<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span>      boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow,<a name="line.1263"></a>
-<span class="sourceLineNo">1264</span>      boolean includeStartRow, byte[] stopRow, boolean includeStopRow, long readPt,<a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>      boolean includeMemstoreScanner) throws IOException {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    List&lt;KeyValueScanner&gt; memStoreScanners = null;<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>    if (includeMemstoreScanner) {<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span>      this.lock.readLock().lock();<a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>      try {<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>        memStoreScanners = this.memstore.getScanners(readPt);<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>      } finally {<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span>        this.lock.readLock().unlock();<a name="line.1272"></a>
-<span class="sourceLineNo">1273</span>      }<a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>    }<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>    List&lt;StoreFileScanner&gt; sfScanners = StoreFileScanner.getScannersForStoreFiles(files,<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>      cacheBlocks, usePread, isCompaction, false, matcher, readPt);<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>    List&lt;KeyValueScanner&gt; scanners = new ArrayList&lt;&gt;(sfScanners.size() + 1);<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span>    scanners.addAll(sfScanners);<a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>    // Then the memstore scanners<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>    if (memStoreScanners != null) {<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>      scanners.addAll(memStoreScanners);<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span>    }<a name="line.1282"></a>
-<span class="sourceLineNo">1283</span>    return scanners;<a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>  }<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span><a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>  /**<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>   * @param o Observer who wants to know about changes in set of Readers<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>   */<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>  public void addChangedReaderObserver(ChangedReadersObserver o) {<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>    this.changedReaderObservers.add(o);<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>  }<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span><a name="line.1292"></a>
-<span class="sourceLineNo">1293</span>  /**<a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>   * @param o Observer no longer interested in changes in set of Readers.<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   */<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>  public void deleteChangedReaderObserver(ChangedReadersObserver o) {<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>    // We don't check if observer present; it may not be (legitimately)<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>    this.changedReaderObservers.remove(o);<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>  }<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span><a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>  //////////////////////////////////////////////////////////////////////////////<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>  // Compaction<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span>  //////////////////////////////////////////////////////////////////////////////<a name="line.1303"></a>
-<span class="sourceLineNo">1304</span><a name="line.1304"></a>
-<span class="sourceLineNo">1305</span>  /**<a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>   * Compact the StoreFiles.  This method may take some time, so the calling<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   * thread must be able to block for long periods.<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   *<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   * &lt;p&gt;During this time, the Store can work as usual, getting values from<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   * StoreFiles and writing new StoreFiles from the memstore.<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span>   *<a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>   * Existing StoreFiles are not destroyed until the new compacted StoreFile is<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>   * completely written-out to disk.<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>   *<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>   * &lt;p&gt;The compactLock prevents multiple simultaneous compactions.<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>   * The structureLock prevents us from interfering with other write operations.<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>   *<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>   * &lt;p&gt;We don't want to hold the structureLock for the whole time, as a compact()<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>   * can be lengthy and we want to allow cache-flushes during this period.<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>   *<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span>   * &lt;p&gt; Compaction event should be idempotent, since there is no IO Fencing for<a name="line.1321"></a>
-<span class="sourceLineNo">1322</span>   * the region directory in hdfs. A region server might still try to complete the<a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>   * compaction after it lost the region. That is why the following events are carefully<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>   * ordered for a compaction:<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>   *  1. Compaction writes new files under region/.tmp directory (compaction output)<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>   *  2. Compaction atomically moves the temporary file under region directory<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>   *  3. Compaction appends a WAL edit containing the compaction input and output files.<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>   *  Forces sync on WAL.<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span>   *  4. Compaction deletes the input files from the region directory.<a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>   *<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>   * Failure conditions are handled like this:<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span>   *  - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes<a name="line.1332"></a>
-<span class="sourceLineNo">1333</span>   *  the compaction later, it will only write the new data file to the region directory.<a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>   *  Since we already have this data, this will be idempotent but we will have a redundant<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>   *  copy of the data.<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>   *  - If RS fails between 2 and 3, the region will have a redundant copy of the data. The<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>   *  RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>   *  - If RS fails after 3, the region region server who opens the region will pick up the<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span>   *  the compaction marker from the WAL and replay it by removing the compaction input files.<a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>   *  Failed RS can also attempt to delete those files, but the operation will be idempotent<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>   *<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>   * See HBASE-2231 for details.<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>   *<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>   * @param compaction compaction details obtained from requestCompaction()<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>   * @throws IOException<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>   * @return Storefile we compacted into or null if we failed or opted out early.<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>   */<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>  public List&lt;HStoreFile&gt; compact(CompactionContext compaction,<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span>    ThroughputController throughputController, User user) throws IOException {<a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>    assert compaction != null;<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>    List&lt;HStoreFile&gt; sfs = null;<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>    CompactionRequestImpl cr = compaction.getRequest();<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>    try {<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>      // Do all sanity checking in here if we have a valid CompactionRequestImpl<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>      // because we need to clean up after it on the way out in a finally<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span>      // block below<a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>      long compactionStartTime = EnvironmentEdgeManager.currentTime();<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span>      assert compaction.hasSelection();<a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>      Collection&lt;HStoreFile&gt; filesToCompact = cr.getFiles();<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>      assert !filesToCompact.isEmpty();<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>      synchronized (filesCompacting) {<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span>        // sanity check: we're compacting files that this store knows about<a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>        // TODO: change this to LOG.error() after more debugging<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>        Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>      }<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span><a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      // Ready to go. Have list of files to compact.<a name="line.1367"></a>
-<span class="sourceLineNo">1368</span>      LOG.info("Starting compaction of " + filesToCompact +<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>        " into tmpdir=" + fs.getTempDir() + ", totalSize=" +<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>          TraditionalBinaryPrefix.long2String(cr.getSize(), "", 1));<a name="line.1370"></a>
-<span class="sourceLineNo">1371</span><a name="line.1371"></a>
-<span class="sourceLineNo">1372</span>      // Commence the compaction.<a name="line.1372"></a>
-<span class="sourceLineNo">1373</span>      List&lt;Path&gt; newFiles = compaction.compact(throughputController, user);<a name="line.1373"></a>
-<span class="sourceLineNo">1374</span><a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>      // TODO: get rid of this!<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>      if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>        LOG.warn("hbase.hstore.compaction.complete is set to false");<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>        sfs = new ArrayList&lt;&gt;(newFiles.size());<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>        final boolean evictOnClose =<a name="line.1379"></a>
-<span class="sourceLineNo">1380</span>            cacheConf != null? cacheConf.shouldEvictOnClose(): true;<a name="line.1380"></a>
-<span class="sourceLineNo">1381</span>        for (Path newFile : newFiles) {<a name="line.1381"></a>
-<span class="sourceLineNo">1382</span>          // Create storefile around what we wrote with a reader on it.<a name="line.1382"></a>
-<span class="sourceLineNo">1383</span>          HStoreFile sf = createStoreFileAndReader(newFile);<a name="line.1383"></a>
-<span class="sourceLineNo">1384</span>          sf.closeStoreFile(evictOnClose);<a name="line.1384"></a>
-<span class="sourceLineNo">1385</span>          sfs.add(sf);<a name="line.1385"></a>
-<span class="sourceLineNo">1386</span>        }<a name="line.1386"></a>
-<span class="sourceLineNo">1387</span>        return sfs;<a name="line.1387"></a>
-<span class="sourceLineNo">1388</span>      }<a name="line.1388"></a>
-<span class="sourceLineNo">1389</span>      // Do the steps necessary to complete the compaction.<a name="line.1389"></a>
-<span class="sourceLineNo">1390</span>      sfs = moveCompactedFilesIntoPlace(cr, newFiles, user);<a name="line.1390"></a>
-<span class="sourceLineNo">1391</span>      writeCompactionWalRecord(filesToCompact, sfs);<a name="line.1391"></a>
-<span class="sourceLineNo">1392</span>      replaceStoreFiles(filesToCompact, sfs);<a name="line.1392"></a>
-<span class="sourceLineNo">1393</span>      if (cr.isMajor()) {<a name="line.1393"></a>
-<span class="sourceLineNo">1394</span>        majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;<a name="line.1394"></a>
-<span class="sourceLineNo">1395</span>        majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;<a name="line.1395"></a>
-<span class="sourceLineNo">1396</span>      } else {<a name="line.1396"></a>
-<span class="sourceLineNo">1397</span>        compactedCellsCount += getCompactionProgress().totalCompactingKVs;<a name="line.1397"></a>
-<span class="sourceLineNo">1398</span>        compactedCellsSize += getCompactionProgress().totalCompactedSize;<a name="line.1398"></a>
-<span class="sourceLineNo">1399</span>      }<a name="line.1399"></a>
-<span class="sourceLineNo">1400</span>      long outputBytes = getTotalSize(sfs);<a name="line.1400"></a>
+<span class="sourceLineNo">1172</span>   * @param includeStartRow true to include start row, false if not<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span>   * @param stopRow the stop row<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span>   * @param includeStopRow true to include stop row, false if not<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span>   * @param readPt the read point of the current scan<a name="line.1175"></a>
+<span class="sourceLineNo">1176</span>   * @return all scanners for this store<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>   */<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>  public List&lt;KeyValueScanner&gt; getScanners(boolean cacheBlocks, boolean usePread,<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>      boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow, boolean includeStartRow,<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span>      byte[] stopRow, boolean includeStopRow, long readPt) throws IOException {<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>    Collection&lt;HStoreFile&gt; storeFilesToScan;<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span>    List&lt;KeyValueScanner&gt; memStoreScanners;<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span>    this.lock.readLock().lock();<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span>    try {<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span>      storeFilesToScan = this.storeEngine.getStoreFileManager().getFilesForScan(startRow,<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>        includeStartRow, stopRow, includeStopRow);<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>      memStoreScanners = this.memstore.getScanners(readPt);<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>    } finally {<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>      this.lock.readLock().unlock();<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>    }<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span><a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>    // First the store file scanners<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span><a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>    // TODO this used to get the store files in descending order,<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span>    // but now we get them in ascending order, which I think is<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>    // actually more correct, since memstore get put at the end.<a name="line.1196"></a>
+<span class="sourceLineNo">1197</span>    List&lt;StoreFileScanner&gt; sfScanners = StoreFileScanner.getScannersForStoreFiles(storeFilesToScan,<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span>      cacheBlocks, usePread, isCompaction, false, matcher, readPt);<a name="line.1198"></a>
+<span class="sourceLineNo">1199</span>    List&lt;KeyValueScanner&gt; scanners = new ArrayList&lt;&gt;(sfScanners.size() + 1);<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span>    scanners.addAll(sfScanners);<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>    // Then the memstore scanners<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>    scanners.addAll(memStoreScanners);<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>    return scanners;<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>  }<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span><a name="line.1205"></a>
+<span class="sourceLineNo">1206</span>  /**<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>   * Create scanners on the given files and if needed on the memstore with no filtering based on TTL<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span>   * (that happens further down the line).<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>   * @param files the list of files on which the scanners has to be created<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>   * @param cacheBlocks cache the blocks or not<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span>   * @param usePread true to use pread, false if not<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>   * @param isCompaction true if the scanner is created for compaction<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>   * @param matcher the scan query matcher<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>   * @param startRow the start row<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>   * @param stopRow the stop row<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span>   * @param readPt the read point of the current scan<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>   * @param includeMemstoreScanner true if memstore has to be included<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span>   * @return scanners on the given files and on the memstore if specified<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>   */<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>  public List&lt;KeyValueScanner&gt; getScanners(List&lt;HStoreFile&gt; files, boolean cacheBlocks,<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>      boolean isGet, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher,<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>      byte[] startRow, byte[] stopRow, long readPt, boolean includeMemstoreScanner)<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span>      throws IOException {<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>    return getScanners(files, cacheBlocks, usePread, isCompaction, matcher, startRow, true, stopRow,<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>      false, readPt, includeMemstoreScanner);<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>  }<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span><a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>  /**<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>   * Create scanners on the given files and if needed on the memstore with no filtering based on TTL<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span>   * (that happens further down the line).<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>   * @param files the list of files on which the scanners has to be created<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>   * @param cacheBlocks ache the blocks or not<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>   * @param usePread true to use pread, false if not<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>   * @param isCompaction true if the scanner is created for compaction<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>   * @param matcher the scan query matcher<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span>   * @param startRow the start row<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span>   * @param includeStartRow true to include start row, false if not<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>   * @param stopRow the stop row<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>   * @param includeStopRow true to include stop row, false if not<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>   * @param readPt the read point of the current scan<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>   * @param includeMemstoreScanner true if memstore has to be included<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span>   * @return scanners on the given files and on the memstore if specified<a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>   */<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span>  public List&lt;KeyValueScanner&gt; getScanners(List&lt;HStoreFile&gt; files, boolean cacheBlocks,<a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>      boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow,<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>      boolean includeStartRow, byte[] stopRow, boolean includeStopRow, long readPt,<a name="line.1246"></a>
+<span class="sourceLineNo">1247</span>      boolean includeMemstoreScanner) throws IOException {<a name="line.1247"></a>
+<span class="sourceLineNo">1248</span>    List&lt;KeyValueScanner&gt; memStoreScanners = null;<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span>    if (includeMemstoreScanner) {<a name="line.1249"></a>
+<span class="sourceLineNo">1250</span>      this.lock.readLock().lock();<a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>      try {<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>        memStoreScanners = this.memstore.getScanners(readPt);<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span>      } finally {<a name="line.1253"></a>
+<span class="sourceLineNo">1254</span>        this.lock.readLock().unlock();<a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>      }<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>    }<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span>    List&lt;StoreFileScanner&gt; sfScanners = StoreFileScanner.getScannersForStoreFiles(files,<a name="line.1257"></a>
+<span class="sourceLineNo">1258</span>      cacheBlocks, usePread, isCompaction, false, matcher, readPt);<a name="line.1258"></a>
+<span class="sourceLineNo">1259</span>    List&lt;KeyValueScanner&gt; scanners = new ArrayList&lt;&gt;(sfScanners.size() + 1);<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>    scanners.addAll(sfScanners);<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span>    // Then the memstore scanners<a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>    if (memStoreScanners != null) {<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span>      scanners.addAll(memStoreScanners);<a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>    }<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>    return scanners;<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span>  }<a name="line.1266"></a>
+<span class="sourceLineNo">1267</span><a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>  /**<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>   * @param o Observer who wants to know about changes in set of Readers<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>   */<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span>  public void addChangedReaderObserver(ChangedReadersObserver o) {<a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>    this.changedReaderObservers.add(o);<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>  }<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span><a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>  /**<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span>   * @param o Observer no longer interested in changes in set of Readers.<a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>   */<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>  public void deleteChangedReaderObserver(ChangedReadersObserver o) {<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    // We don't check if observer present; it may not be (legitimately)<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>    this.changedReaderObservers.remove(o);<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span>  }<a name="line.1281"></a>
+<span class="sourceLineNo">1282</span><a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>  //////////////////////////////////////////////////////////////////////////////<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>  // Compaction<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>  //////////////////////////////////////////////////////////////////////////////<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span><a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>  /**<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>   * Compact the StoreFiles.  This method may take some time, so the calling<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>   * thread must be able to block for long periods.<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>   *<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>   * &lt;p&gt;During this time, the Store can work as usual, getting values from<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span>   * StoreFiles and writing new StoreFiles from the memstore.<a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>   *<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>   * Existing StoreFiles are not destroyed until the new compacted StoreFile is<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   * completely written-out to disk.<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   *<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>   * &lt;p&gt;The compactLock prevents multiple simultaneous compactions.<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>   * The structureLock prevents us from interfering with other write operations.<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>   *<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>   * &lt;p&gt;We don't want to hold the structureLock for the whole time, as a compact()<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>   * can be lengthy and we want to allow cache-flushes during this period.<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>   *<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>   * &lt;p&gt; Compaction event should be idempotent, since there is no IO Fencing for<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span>   * the region directory in hdfs. A region server might still try to complete the<a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>   * compaction after it lost the region. That is why the following events are carefully<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span>   * ordered for a compaction:<a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   *  1. Compaction writes new files under region/.tmp directory (compaction output)<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   *  2. Compaction atomically moves the temporary file under region directory<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   *  3. Compaction appends a WAL edit containing the compaction input and output files.<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>   *  Forces sync on WAL.<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>   *  4. Compaction deletes the input files from the region directory.<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>   *<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>   * Failure conditions are handled like this:<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span>   *  - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes<a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>   *  the compaction later, it will only write the new data file to the region directory.<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>   *  Since we already have this data, this will be idempotent but we will have a redundant<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>   *  copy of the data.<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>   *  - If RS fails between 2 and 3, the region will have a redundant copy of the data. The<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>   *  RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>   *  - If RS fails after 3, the region region server who opens the region will pick up the<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span>   *  the compaction marker from the WAL and replay it by removing the compaction input files.<a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>   *  Failed RS can also attempt to delete those files, but the operation will be idempotent<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>   *<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span>   * See HBASE-2231 for details.<a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>   *<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>   * @param compaction compaction details obtained from requestCompaction()<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>   * @throws IOException<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>   * @return Storefile we compacted into or null if we failed or opted out early.<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>   */<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>  public List&lt;HStoreFile&gt; compact(CompactionContext compaction,<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span>    ThroughputController throughputController, User user) throws IOException {<a name="line.1331"></a>
+<span class="sourceLineNo">1332</span>    assert compaction != null;<a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>    List&lt;HStoreFile&gt; sfs = null;<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>    CompactionRequestImpl cr = compaction.getRequest();<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>    try {<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span>      // Do all sanity checking in here if we have a valid CompactionRequestImpl<a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>      // because we need to clean up after it on the way out in a finally<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>      // block below<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>      long compactionStartTime = EnvironmentEdgeManager.currentTime();<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>      assert compaction.hasSelection();<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>      Collection&lt;HStoreFile&gt; filesToCompact = cr.getFiles();<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span>      assert !filesToCompact.isEmpty();<a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>      synchronized (filesCompacting) {<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>        // sanity check: we're compacting files that this store knows about<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>        // TODO: change this to LOG.error() after more debugging<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>        Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>      }<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span><a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>      // Ready to go. Have list of files to compact.<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>      LOG.info("Starting compaction of " + filesToCompact +<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>        " into tmpdir=" + fs.getTempDir() + ", totalSize=" +<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>          TraditionalBinaryPrefix.long2String(cr.getSize(), "", 1));<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span><a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>      // Commence the compaction.<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>      List&lt;Path&gt; newFiles = compaction.compact(throughputController, user);<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span><a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>      // TODO: get rid of this!<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>      if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span>        LOG.warn("hbase.hstore.compaction.complete is set to false");<a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>        sfs = new ArrayList&lt;&gt;(newFiles.size());<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span>        final boolean evictOnClose =<a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>            cacheConf != null? cacheConf.shouldEvictOnClose(): true;<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>        for (Path newFile : newFiles) {<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>          // Create storefile around what we wrote with a reader on it.<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span>          HStoreFile sf = createStoreFileAndReader(newFile);<a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>          sf.closeStoreFile(evictOnClose);<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>          sfs.add(sf);<a name="line.1367"></a>
+<span class="sourceLineNo">1368</span>        }<a name="line.1368"></a>
+<span class="sourceLineNo">1369</span>        return sfs;<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      }<a name="line.1370"></a>
+<span class="sourceLineNo">1371</span>      // Do the steps necessary to complete the compaction.<a name="line.1371"></a>
+<span class="sourceLineNo">1372</span>      sfs = moveCompactedFilesIntoPlace(cr, newFiles, user);<a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>      writeCompactionWalRecord(filesToCompact, sfs);<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>      replaceStoreFiles(filesToCompact, sfs);<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span>      if (cr.isMajor()) {<a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>        majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;<a name="line.1376"></a>
+<span class="sourceLineNo">1377</span>        majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;<a name="line.1377"></a>
+<span class="sourceLineNo">1378</span>      } else {<a name="line.1378"></a>
+<span class="sourceLineNo">1379</span>        compactedCellsCount += getCompactionProgress().totalCompactingKVs;<a name="line.1379"></a>
+<span class="sourceLineNo">1380</span>        compactedCellsSize += getCompactionProgress().totalCompactedSize;<a name="line.1380"></a>
+<span class="sourceLineNo">1381</span>      }<a name="line.1381"></a>
+<span class="sourceLineNo">1382</span>      long outputBytes = getTotalSize(sfs);<a name="line.1382"></a>
+<span class="sourceLineNo">1383</span><a name="line.1383"></a>
+<span class="sourceLineNo">1384</span>      // At this point the store will use new files for all new scanners.<a name="line.1384"></a>
+<span class="sourceLineNo">1385</span>      completeCompaction(filesToCompact); // update store size.<a name="line.1385"></a>
+<span class="sourceLineNo">1386</span><a name="line.1386"></a>
+<span class="sourceLineNo">1387</span>      long now = EnvironmentEdgeManager.currentTime();<a name="line.1387"></a>
+<span class="sourceLineNo">1388</span>      if (region.getRegionServerServices() != null<a name="line.1388"></a>
+<span class="sourceLineNo">1389</span>          &amp;&amp; region.getRegionServerServices().getMetrics() != null) {<a name="line.1389"></a>
+<span class="sourceLineNo">1390</span>        region.getRegionServerServices().getMetrics().updateCompaction(cr.isMajor(),<a name="line.1390"></a>
+<span class="sourceLineNo">1391</span>          now - compactionStartTime, cr.getFiles().size(), newFiles.size(), cr.getSize(),<a name="line.1391"></a>
+<span class="sourceLineNo">1392</span>          outputBytes);<a name="line.1392"></a>
+<span class="sourceLineNo">1393</span>      }<a name="line.1393"></a>
+<span class="sourceLineNo">1394</span><a name="line.1394"></a>
+<span class="sourceLineNo">1395</span>      logCompactionEndMessage(cr, sfs, now, compactionStartTime);<a name="line.1395"></a>
+<span class="sourceLineNo">1396</span>      return sfs;<a name="line.1396"></a>
+<span class="sourceLineNo">1397</span>    } finally {<a name="line.1397"></a>
+<span class="sourceLineNo">1398</span>      finishCompactionRequest(cr);<a name="line.1398"></a>
+<span class="sourceLineNo">1399</span>    }<a name="line.1399"></a>
+<span class="sourceLineNo">1400</span>  }<a name="line.1400"></a>
 <span class="sourceLineNo">1401</span><a name="line.1401"></a>
-<span class="sourceLineNo">1402</span>      // At this point the store will use new files for all new scanners.<a name="line.1402"></a>
-<span class="sourceLineNo">1403</span>      completeCompaction(filesToCompact); // update store size.<a name="line.1403"></a>
-<span class="sourceLineNo">1404</span><a name="line.1404"></a>
-<span class="sourceLineNo">1405</span>      long now = EnvironmentEdgeManager.currentTime();<a name="line.1405"></a>
-<span class="sourceLineNo">1406</span>      if (region.getRegionServerServices() != null<a name="line.1406"></a>
-<span class="sourceLineNo">1407</span>          &amp;&amp; region.getRegionServerServices().getMetrics() != null) {<a name="line.1407"></a>
-<span class="sourceLineNo">1408</span>        region.getRegionServerServices().getMetrics().updateCompaction(cr.isMajor(),<a name="line.1408"></a>
-<span class="sourceLineNo">1409</span>          now - compactionStartTime, cr.getFiles().size(), newFiles.size(), cr.getSize(),<a name="line.1409"></a>
-<span class="sourceLineNo">1410</span>          outputBytes);<a name="line.1410"></a>
-<span class="sourceLineNo">1411</span>      }<a name="line.1411"></a>
-<span class="sourceLineNo">1412</span><a name="line.1412"></a>
-<span class="sourceLineNo">1413</span>      logCompactionEndMessage(cr, sfs, now, compactionStartTime);<a name="line.1413"></a>
-<span class="sourceLineNo">1414</span>      return sfs;<a name="line.1414"></a>
-<span class="sourceLineNo">1415</span>    } finally {<a name="line.1415"></a>
-<span class="sourceLineNo">1416</span>      finishCompactionRequest(cr);<a name="line.1416"></a>
-<span class="sourceLineNo">1417</span>    }<a name="line.1417"></a>
-<span class="sourceLineNo">1418</span>  }<a name="line.1418"></a>
-<span class="sourceLineNo">1419</span><a name="line.1419"></a>
-<span class="sourceLineNo">1420</span>  private List&lt;HStoreFile&gt; moveCompactedFilesIntoPlace(CompactionRequestImpl cr, List&lt;Path&gt; newFiles,<a name="line.1420"></a>
-<span class="sourceLineNo">1421</span>      User user) throws IOException {<a name="line.1421"></a>
-<span class="sourceLineNo">1422</span>    List&lt;HStoreFile&gt; sfs = new ArrayList&lt;&gt;(newFiles.size());<a name="line.1422"></a>
-<span class="sourceLineNo">1423</span>    for (Path newFile : newFiles) {<a name="line.1423"></a>
-<span class="sourceLineNo">1424</span>      assert newFile != null;<a name="line.1424"></a>
-<span class="sourceLineNo">1425</span>      HStoreFile sf = moveFileIntoPlace(newFile);<a name="line.1425"></a>
-<span class="sourceLineNo">1426</span>      if (this.getCoprocessorHost() != null) {<a name="line.1426"></a>
-<span class="sourceLineNo">1427</span>        getCoprocessorHost().postCompact(this, sf, cr.getTracker(), cr, user);<a name="line.1427"></a>
-<span class="sourceLineNo">1428</span>      }<a name="line.1428"></a>
-<span class="sourceLineNo">1429</span>      assert sf != null;<a name="line.1429"></a>
-<span class="sourceLineNo">1430</span>      sfs.add(sf);<a name="line.1430"></a>
-<span class="sourceLineNo">1431</span>    }<a name="line.1431"></a>
-<span class="sourceLineNo">1432</span>    return sfs;<a name="line.1432"></a>
-<span class="sourceLineNo">1433</span>  }<a name="line.1433"></a>
-<span class="sourceLineNo">1434</span><a name="line.1434"></a>
-<span class="sourceLineNo">1435</span>  // Package-visible for tests<a name="line.1435"></a>
-<span class="sourceLineNo">1436</span>  HStoreFile moveFileIntoPlace(Path newFile) throws IOException {<a name="line.1436"></a>
-<span class="sourceLineNo">1437</span>    validateStoreFile(newFile);<a name="line.1437"></a>
-<span class="sourceLineNo">1438</span>    // Move the file into the right spot<a name="line.1438"></a>
-<span class="sourceLineNo">1439</span>    Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);<a name="line.1439"></a>
-<span class="sourceLineNo">1440</span>    return createStoreFileAndReader(destPath);<a name="line.1440"></a>
-<span class="sourceLineNo">1441</span>  }<a name="line.1441"></a>
-<span class="sourceLineNo">1442</span><a name="line.1442"></a>
-<span class="sourceLineNo">1443</span>  /**<a name="line.1443"></a>
-<span class="sourceLineNo">1444</span>   * Writes the compaction WAL record.<a name="line.1444"></a>
-<span class="sourceLineNo">1445</span>   * @param filesCompacted Files compacted (input).<a name="line.1445"></a>
-<span class="sourceLineNo">1446</span>   * @param newFiles Files from compaction.<a name="line.1446"></a>
-<span class="sourceLineNo">1447</span>   */<a name="line.1447"></a>
-<span class="sourceLineNo">1448</span>  private void writeCompactionWalRecord(Collection&lt;HStoreFile&gt; filesCompacted,<a name="line.1448"></a>
-<span class="sourceLineNo">1449</span>      Collection&lt;HStoreFile&gt; newFiles) throws IOException {<a name="line.1449"></a>
-<span class="sourceLineNo">1450</span>    if (region.getWAL() == null) {<a name="line.1450"></a>
-<span class="sourceLineNo">1451</span>      return;<a name="line.1451"></a>
-<span class="sourceLineNo">1452</span>    }<a name="line.1452"></a>
-<span class="sourceLineNo">1453</span>    List&lt;Path&gt; inputPaths =<a name="line.1453"></a>
-<span class="sourceLineNo">1454</span>        filesCompacted.stream().map(HStoreFile::getPath).collect(Collectors.toList());<a name="line.1454"></a>
-<span class="sourceLineNo">1455</span>    List&lt;Path&gt; outputPaths =<a name="line.1455"></a>
-<span class="sourceLineNo">1456</span>        newFiles.stream().map(HStoreFile::getPath).collect(Collectors.toList());<a name="line.1456"></a>
-<span class="sourceLineNo">1457</span>    RegionInfo info = this.region.getRegionInfo();<a name="line.1457"></a>
-<span class="sourceLineNo">1458</span>    CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,<a name="line.1458"></a>
-<span class="sourceLineNo">1459</span>        family.getName(), inputPaths, outputPaths, fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString()));<a name="line.1459"></a>
-<span class="sourceLineNo">1460</span>    // Fix reaching into Region to get the maxWaitForSeqId.<a name="line.1460"></a>
-<span class="sourceLineNo">1461</span>    // Does this method belong in Region altogether given it is making so many references up there?<a name="line.1461"></a>
-<span class="sourceLineNo">1462</span>    // Could be Region#writeCompactionMarker(compactionDescriptor);<a name="line.1462"></a>
-<span class="sourceLineNo">1463</span>    WALUtil.writeCompactionMarker(this.region.getWAL(), this.region.getReplicationScope(),<a name="line.1463"></a>
-<span class="sourceLineNo">1464</span>        this.region.getRegionInfo(), compactionDescriptor, this.region.getMVCC());<a name="line.1464"></a>
-<span class="sourceLineNo">1465</span>  }<a name="line.1465"></a>
-<span class="sourceLineNo">1466</span><a name="line.1466"></a>
-<span class="sourceLineNo">1467</span>  @VisibleForTesting<a name="line.1467"></a>
-<span class="sourceLineNo">1468</span>  void replaceStoreFiles(Collection&lt;HStoreFile&gt; compactedFiles, Collection&lt;HStoreFile&gt; result)<a name="line.1468"></a>
-<span class="sourceLineNo">1469</span>      throws IOException {<a name="line.1469"></a>
-<span class="sourceLineNo">1470</span>    this.lock.writeLock().lock();<a name="line.1470"></a>
-<span class="sourceLineNo">1471</span>    try {<a name="line.1471"></a>
-<span class="sourceLineNo">1472</span>      this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result);<a name="line.1472"></a>
-<span class="sourceLineNo">1473</span>      filesCompacting.removeAll(compactedFiles); // safe bc: lock.writeLock();<a name="line.1473"></a>
-<span class="sourceLineNo">1474</span>    } finally {<a name="line.1474"></a>
-<span class="sourceLineNo">1475</span>      this.lock.writeLock().unlock();<a name="line.1475"></a>
-<span class="sourceLineNo">1476</span>    }<a name="line.1476"></a>
-<span class="sourceLineNo">1477</span>  }<a name="line.1477"></a>
-<span class="sourceLineNo">1478</span><a name="line.1478"></a>
-<span class="sourceLineNo">1479</span>  /**<a name="line.1479"></a>
-<span class="sourceLineNo">1480</span>   * Log a very elaborate compaction completion message.<a name="line.1480"></a>
-<span class="sourceLineNo">1481</span>   * @param cr Request.<a name="line.1481"></a>
-<span class="sourceLineNo">1482</span>   * @param sfs Resulting files.<a name="line.1482"></a>
-<span class="sourceLineNo">1483</span>   * @param compactionStartTime Start time.<a name="line.1483"></a>
-<span class="sourceLineNo">1484</span>   */<a name="line.1484"></a>
-<span class="sourceLineNo">1485</span>  private void logCompactionEndMessage(<a name="line.1485"></a>
-<span class="sourceLineNo">1486</span>      CompactionRequestImpl cr, List&lt;HStoreFile&gt; sfs, long now, long compactionStartTime) {<a name="line.1486"></a>
-<span class="sourceLineNo">1487</span>    StringBuilder message = new StringBuilder(<a name="line.1487"></a>
-<span class="sourceLineNo">1488</span>      "Completed" + (cr.isMajor() ? " major" : "") + " compaction of "<a name="line.1488"></a>
-<span class="sourceLineNo">1489</span>      + cr.getFiles().size() + (cr.isAllFiles() ? " (all)" : "") + " file(s) in "<a name="line.1489"></a>
-<span class="sourceLineNo">1490</span>      + this + " of " + this.getRegionInfo().getShortNameToLog() + " into ");<a name="line.1490"></a>
-<span class="sourceLineNo">1491</span>    if (sfs.isEmpty()) {<a name="line.1491"></a>
-<span class="sourceLineNo">1492</span>      message.append("none, ");<a name="line.1492"></a>
-<span class="sourceLineNo">1493</span>    } else {<a name="line.1493"></a>
-<span class="sourceLineNo">1494</span>      for (HStoreFile sf: sfs) {<a name="line.1494"></a>
-<span class="sourceLineNo">1495</span>        message.append(sf.getPath().getName());<a name="line.1495"></a>
-<span class="sourceLineNo">1496</span>        message.append("(size=");<a name="line.1496"></a>
-<span class="sourceLineNo">1497</span>        message.append(TraditionalBinaryPrefix.long2String(sf.getReader().length(), "", 1));<a name="line.1497"></a>
-<span class="sourceLineNo">1498</span>        message.append(

<TRUNCATED>

Mime
View raw message