hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mi...@apache.org
Subject [08/45] hbase-site git commit: Published site at 04622254f7209c5cfeadcfa137a97fbed161075a.
Date Mon, 14 Dec 2015 16:38:19 GMT
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/736088b3/xref/org/apache/hadoop/hbase/regionserver/HStore.html
----------------------------------------------------------------------
diff --git a/xref/org/apache/hadoop/hbase/regionserver/HStore.html b/xref/org/apache/hadoop/hbase/regionserver/HStore.html
index 1a9620d..929329a 100644
--- a/xref/org/apache/hadoop/hbase/regionserver/HStore.html
+++ b/xref/org/apache/hadoop/hbase/regionserver/HStore.html
@@ -1052,7 +1052,7 @@
 <a class="jxr_linenumber" name="1042" href="#1042">1042</a>       <strong class="jxr_keyword">this</strong>.lock.writeLock().unlock();
 <a class="jxr_linenumber" name="1043" href="#1043">1043</a>     }
 <a class="jxr_linenumber" name="1044" href="#1044">1044</a>     <em class="jxr_comment">// notify to be called here - only in case of flushes</em>
-<a class="jxr_linenumber" name="1045" href="#1045">1045</a>     notifyChangedReadersObservers();
+<a class="jxr_linenumber" name="1045" href="#1045">1045</a>     notifyChangedReadersObservers(sfs);
 <a class="jxr_linenumber" name="1046" href="#1046">1046</a>     <strong class="jxr_keyword">if</strong> (LOG.isTraceEnabled()) {
 <a class="jxr_linenumber" name="1047" href="#1047">1047</a>       <strong class="jxr_keyword">long</strong> totalSize = 0;
 <a class="jxr_linenumber" name="1048" href="#1048">1048</a>       <strong class="jxr_keyword">for</strong> (StoreFile sf : sfs) {
@@ -1070,9 +1070,9 @@
 <a class="jxr_linenumber" name="1060" href="#1060">1060</a> <em class="jxr_comment">   * Notify all observers that set of Readers has changed.</em>
 <a class="jxr_linenumber" name="1061" href="#1061">1061</a> <em class="jxr_comment">   * @throws IOException</em>
 <a class="jxr_linenumber" name="1062" href="#1062">1062</a> <em class="jxr_comment">   */</em>
-<a class="jxr_linenumber" name="1063" href="#1063">1063</a>   <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> notifyChangedReadersObservers() <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1063" href="#1063">1063</a>   <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> notifyChangedReadersObservers(List&lt;StoreFile&gt; sfs) <strong class="jxr_keyword">throws</strong> IOException {
 <a class="jxr_linenumber" name="1064" href="#1064">1064</a>     <strong class="jxr_keyword">for</strong> (ChangedReadersObserver o : <strong class="jxr_keyword">this</strong>.changedReaderObservers) {
-<a class="jxr_linenumber" name="1065" href="#1065">1065</a>       o.updateReaders();
+<a class="jxr_linenumber" name="1065" href="#1065">1065</a>       o.updateReaders(sfs);
 <a class="jxr_linenumber" name="1066" href="#1066">1066</a>     }
 <a class="jxr_linenumber" name="1067" href="#1067">1067</a>   }
 <a class="jxr_linenumber" name="1068" href="#1068">1068</a> 
@@ -1112,1283 +1112,1307 @@
 <a class="jxr_linenumber" name="1102" href="#1102">1102</a>   }
 <a class="jxr_linenumber" name="1103" href="#1103">1103</a> 
 <a class="jxr_linenumber" name="1104" href="#1104">1104</a>   @Override
-<a class="jxr_linenumber" name="1105" href="#1105">1105</a>   <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> addChangedReaderObserver(<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.html">ChangedReadersObserver</a> o) {
-<a class="jxr_linenumber" name="1106" href="#1106">1106</a>     <strong class="jxr_keyword">this</strong>.changedReaderObservers.add(o);
-<a class="jxr_linenumber" name="1107" href="#1107">1107</a>   }
-<a class="jxr_linenumber" name="1108" href="#1108">1108</a> 
-<a class="jxr_linenumber" name="1109" href="#1109">1109</a>   @Override
-<a class="jxr_linenumber" name="1110" href="#1110">1110</a>   <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> deleteChangedReaderObserver(<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.html">ChangedReadersObserver</a> o) {
-<a class="jxr_linenumber" name="1111" href="#1111">1111</a>     <em class="jxr_comment">// We don't check if observer present; it may not be (legitimately)</em>
-<a class="jxr_linenumber" name="1112" href="#1112">1112</a>     <strong class="jxr_keyword">this</strong>.changedReaderObservers.remove(o);
-<a class="jxr_linenumber" name="1113" href="#1113">1113</a>   }
-<a class="jxr_linenumber" name="1114" href="#1114">1114</a> 
-<a class="jxr_linenumber" name="1115" href="#1115">1115</a>   <em class="jxr_comment">//////////////////////////////////////////////////////////////////////////////</em>
-<a class="jxr_linenumber" name="1116" href="#1116">1116</a>   <em class="jxr_comment">// Compaction</em>
-<a class="jxr_linenumber" name="1117" href="#1117">1117</a>   <em class="jxr_comment">//////////////////////////////////////////////////////////////////////////////</em>
-<a class="jxr_linenumber" name="1118" href="#1118">1118</a> 
-<a class="jxr_linenumber" name="1119" href="#1119">1119</a>   <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1120" href="#1120">1120</a> <em class="jxr_javadoccomment">   * Compact the StoreFiles.  This method may take some time, so the calling</em>
-<a class="jxr_linenumber" name="1121" href="#1121">1121</a> <em class="jxr_javadoccomment">   * thread must be able to block for long periods.</em>
-<a class="jxr_linenumber" name="1122" href="#1122">1122</a> <em class="jxr_javadoccomment">   *</em>
-<a class="jxr_linenumber" name="1123" href="#1123">1123</a> <em class="jxr_javadoccomment">   * &lt;p&gt;During this time, the Store can work as usual, getting values from</em>
-<a class="jxr_linenumber" name="1124" href="#1124">1124</a> <em class="jxr_javadoccomment">   * StoreFiles and writing new StoreFiles from the memstore.</em>
-<a class="jxr_linenumber" name="1125" href="#1125">1125</a> <em class="jxr_javadoccomment">   *</em>
-<a class="jxr_linenumber" name="1126" href="#1126">1126</a> <em class="jxr_javadoccomment">   * Existing StoreFiles are not destroyed until the new compacted StoreFile is</em>
-<a class="jxr_linenumber" name="1127" href="#1127">1127</a> <em class="jxr_javadoccomment">   * completely written-out to disk.</em>
-<a class="jxr_linenumber" name="1128" href="#1128">1128</a> <em class="jxr_javadoccomment">   *</em>
-<a class="jxr_linenumber" name="1129" href="#1129">1129</a> <em class="jxr_javadoccomment">   * &lt;p&gt;The compactLock prevents multiple simultaneous compactions.</em>
-<a class="jxr_linenumber" name="1130" href="#1130">1130</a> <em class="jxr_javadoccomment">   * The structureLock prevents us from interfering with other write operations.</em>
-<a class="jxr_linenumber" name="1131" href="#1131">1131</a> <em class="jxr_javadoccomment">   *</em>
-<a class="jxr_linenumber" name="1132" href="#1132">1132</a> <em class="jxr_javadoccomment">   * &lt;p&gt;We don't want to hold the structureLock for the whole time, as a compact()</em>
-<a class="jxr_linenumber" name="1133" href="#1133">1133</a> <em class="jxr_javadoccomment">   * can be lengthy and we want to allow cache-flushes during this period.</em>
-<a class="jxr_linenumber" name="1134" href="#1134">1134</a> <em class="jxr_javadoccomment">   *</em>
-<a class="jxr_linenumber" name="1135" href="#1135">1135</a> <em class="jxr_javadoccomment">   * &lt;p&gt; Compaction event should be idempotent, since there is no IO Fencing for</em>
-<a class="jxr_linenumber" name="1136" href="#1136">1136</a> <em class="jxr_javadoccomment">   * the region directory in hdfs. A region server might still try to complete the</em>
-<a class="jxr_linenumber" name="1137" href="#1137">1137</a> <em class="jxr_javadoccomment">   * compaction after it lost the region. That is why the following events are carefully</em>
-<a class="jxr_linenumber" name="1138" href="#1138">1138</a> <em class="jxr_javadoccomment">   * ordered for a compaction:</em>
-<a class="jxr_linenumber" name="1139" href="#1139">1139</a> <em class="jxr_javadoccomment">   *  1. Compaction writes new files under region/.tmp directory (compaction output)</em>
-<a class="jxr_linenumber" name="1140" href="#1140">1140</a> <em class="jxr_javadoccomment">   *  2. Compaction atomically moves the temporary file under region directory</em>
-<a class="jxr_linenumber" name="1141" href="#1141">1141</a> <em class="jxr_javadoccomment">   *  3. Compaction appends a WAL edit containing the compaction input and output files.</em>
-<a class="jxr_linenumber" name="1142" href="#1142">1142</a> <em class="jxr_javadoccomment">   *  Forces sync on WAL.</em>
-<a class="jxr_linenumber" name="1143" href="#1143">1143</a> <em class="jxr_javadoccomment">   *  4. Compaction deletes the input files from the region directory.</em>
-<a class="jxr_linenumber" name="1144" href="#1144">1144</a> <em class="jxr_javadoccomment">   *</em>
-<a class="jxr_linenumber" name="1145" href="#1145">1145</a> <em class="jxr_javadoccomment">   * Failure conditions are handled like this:</em>
-<a class="jxr_linenumber" name="1146" href="#1146">1146</a> <em class="jxr_javadoccomment">   *  - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes</em>
-<a class="jxr_linenumber" name="1147" href="#1147">1147</a> <em class="jxr_javadoccomment">   *  the compaction later, it will only write the new data file to the region directory.</em>
-<a class="jxr_linenumber" name="1148" href="#1148">1148</a> <em class="jxr_javadoccomment">   *  Since we already have this data, this will be idempotent but we will have a redundant</em>
-<a class="jxr_linenumber" name="1149" href="#1149">1149</a> <em class="jxr_javadoccomment">   *  copy of the data.</em>
-<a class="jxr_linenumber" name="1150" href="#1150">1150</a> <em class="jxr_javadoccomment">   *  - If RS fails between 2 and 3, the region will have a redundant copy of the data. The</em>
-<a class="jxr_linenumber" name="1151" href="#1151">1151</a> <em class="jxr_javadoccomment">   *  RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.</em>
-<a class="jxr_linenumber" name="1152" href="#1152">1152</a> <em class="jxr_javadoccomment">   *  - If RS fails after 3, the region region server who opens the region will pick up the</em>
-<a class="jxr_linenumber" name="1153" href="#1153">1153</a> <em class="jxr_javadoccomment">   *  the compaction marker from the WAL and replay it by removing the compaction input files.</em>
-<a class="jxr_linenumber" name="1154" href="#1154">1154</a> <em class="jxr_javadoccomment">   *  Failed RS can also attempt to delete those files, but the operation will be idempotent</em>
+<a class="jxr_linenumber" name="1105" href="#1105">1105</a>   <strong class="jxr_keyword">public</strong> List&lt;KeyValueScanner&gt; getScanners(List&lt;StoreFile&gt; files, <strong class="jxr_keyword">boolean</strong> cacheBlocks,
+<a class="jxr_linenumber" name="1106" href="#1106">1106</a>       <strong class="jxr_keyword">boolean</strong> isGet, <strong class="jxr_keyword">boolean</strong> usePread, <strong class="jxr_keyword">boolean</strong> isCompaction, <a href="../../../../../org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.html">ScanQueryMatcher</a> matcher,
+<a class="jxr_linenumber" name="1107" href="#1107">1107</a>       byte[] startRow, byte[] stopRow, <strong class="jxr_keyword">long</strong> readPt, <strong class="jxr_keyword">boolean</strong> includeMemstoreScanner) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1108" href="#1108">1108</a>     List&lt;KeyValueScanner&gt; memStoreScanners = <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1109" href="#1109">1109</a>     <strong class="jxr_keyword">if</strong> (includeMemstoreScanner) {
+<a class="jxr_linenumber" name="1110" href="#1110">1110</a>       <strong class="jxr_keyword">this</strong>.lock.readLock().lock();
+<a class="jxr_linenumber" name="1111" href="#1111">1111</a>       <strong class="jxr_keyword">try</strong> {
+<a class="jxr_linenumber" name="1112" href="#1112">1112</a>         memStoreScanners = <strong class="jxr_keyword">this</strong>.memstore.getScanners(readPt);
+<a class="jxr_linenumber" name="1113" href="#1113">1113</a>       } <strong class="jxr_keyword">finally</strong> {
+<a class="jxr_linenumber" name="1114" href="#1114">1114</a>         <strong class="jxr_keyword">this</strong>.lock.readLock().unlock();
+<a class="jxr_linenumber" name="1115" href="#1115">1115</a>       }
+<a class="jxr_linenumber" name="1116" href="#1116">1116</a>     }
+<a class="jxr_linenumber" name="1117" href="#1117">1117</a>     List&lt;StoreFileScanner&gt; sfScanners = StoreFileScanner.getScannersForStoreFiles(files,
+<a class="jxr_linenumber" name="1118" href="#1118">1118</a>       cacheBlocks, usePread, isCompaction, false, matcher, readPt, isPrimaryReplicaStore());
+<a class="jxr_linenumber" name="1119" href="#1119">1119</a>     List&lt;KeyValueScanner&gt; scanners = <strong class="jxr_keyword">new</strong> ArrayList&lt;KeyValueScanner&gt;(sfScanners.size() + 1);
+<a class="jxr_linenumber" name="1120" href="#1120">1120</a>     scanners.addAll(sfScanners);
+<a class="jxr_linenumber" name="1121" href="#1121">1121</a>     <em class="jxr_comment">// Then the memstore scanners</em>
+<a class="jxr_linenumber" name="1122" href="#1122">1122</a>     <strong class="jxr_keyword">if</strong> (memStoreScanners != <strong class="jxr_keyword">null</strong>) {
+<a class="jxr_linenumber" name="1123" href="#1123">1123</a>       scanners.addAll(memStoreScanners);
+<a class="jxr_linenumber" name="1124" href="#1124">1124</a>     }
+<a class="jxr_linenumber" name="1125" href="#1125">1125</a>     <strong class="jxr_keyword">return</strong> scanners;
+<a class="jxr_linenumber" name="1126" href="#1126">1126</a>   }
+<a class="jxr_linenumber" name="1127" href="#1127">1127</a> 
+<a class="jxr_linenumber" name="1128" href="#1128">1128</a>   @Override
+<a class="jxr_linenumber" name="1129" href="#1129">1129</a>   <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> addChangedReaderObserver(<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.html">ChangedReadersObserver</a> o) {
+<a class="jxr_linenumber" name="1130" href="#1130">1130</a>     <strong class="jxr_keyword">this</strong>.changedReaderObservers.add(o);
+<a class="jxr_linenumber" name="1131" href="#1131">1131</a>   }
+<a class="jxr_linenumber" name="1132" href="#1132">1132</a> 
+<a class="jxr_linenumber" name="1133" href="#1133">1133</a>   @Override
+<a class="jxr_linenumber" name="1134" href="#1134">1134</a>   <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> deleteChangedReaderObserver(<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.html">ChangedReadersObserver</a> o) {
+<a class="jxr_linenumber" name="1135" href="#1135">1135</a>     <em class="jxr_comment">// We don't check if observer present; it may not be (legitimately)</em>
+<a class="jxr_linenumber" name="1136" href="#1136">1136</a>     <strong class="jxr_keyword">this</strong>.changedReaderObservers.remove(o);
+<a class="jxr_linenumber" name="1137" href="#1137">1137</a>   }
+<a class="jxr_linenumber" name="1138" href="#1138">1138</a> 
+<a class="jxr_linenumber" name="1139" href="#1139">1139</a>   <em class="jxr_comment">//////////////////////////////////////////////////////////////////////////////</em>
+<a class="jxr_linenumber" name="1140" href="#1140">1140</a>   <em class="jxr_comment">// Compaction</em>
+<a class="jxr_linenumber" name="1141" href="#1141">1141</a>   <em class="jxr_comment">//////////////////////////////////////////////////////////////////////////////</em>
+<a class="jxr_linenumber" name="1142" href="#1142">1142</a> 
+<a class="jxr_linenumber" name="1143" href="#1143">1143</a>   <em class="jxr_javadoccomment">/**</em>
+<a class="jxr_linenumber" name="1144" href="#1144">1144</a> <em class="jxr_javadoccomment">   * Compact the StoreFiles.  This method may take some time, so the calling</em>
+<a class="jxr_linenumber" name="1145" href="#1145">1145</a> <em class="jxr_javadoccomment">   * thread must be able to block for long periods.</em>
+<a class="jxr_linenumber" name="1146" href="#1146">1146</a> <em class="jxr_javadoccomment">   *</em>
+<a class="jxr_linenumber" name="1147" href="#1147">1147</a> <em class="jxr_javadoccomment">   * &lt;p&gt;During this time, the Store can work as usual, getting values from</em>
+<a class="jxr_linenumber" name="1148" href="#1148">1148</a> <em class="jxr_javadoccomment">   * StoreFiles and writing new StoreFiles from the memstore.</em>
+<a class="jxr_linenumber" name="1149" href="#1149">1149</a> <em class="jxr_javadoccomment">   *</em>
+<a class="jxr_linenumber" name="1150" href="#1150">1150</a> <em class="jxr_javadoccomment">   * Existing StoreFiles are not destroyed until the new compacted StoreFile is</em>
+<a class="jxr_linenumber" name="1151" href="#1151">1151</a> <em class="jxr_javadoccomment">   * completely written-out to disk.</em>
+<a class="jxr_linenumber" name="1152" href="#1152">1152</a> <em class="jxr_javadoccomment">   *</em>
+<a class="jxr_linenumber" name="1153" href="#1153">1153</a> <em class="jxr_javadoccomment">   * &lt;p&gt;The compactLock prevents multiple simultaneous compactions.</em>
+<a class="jxr_linenumber" name="1154" href="#1154">1154</a> <em class="jxr_javadoccomment">   * The structureLock prevents us from interfering with other write operations.</em>
 <a class="jxr_linenumber" name="1155" href="#1155">1155</a> <em class="jxr_javadoccomment">   *</em>
-<a class="jxr_linenumber" name="1156" href="#1156">1156</a> <em class="jxr_javadoccomment">   * See HBASE-2231 for details.</em>
-<a class="jxr_linenumber" name="1157" href="#1157">1157</a> <em class="jxr_javadoccomment">   *</em>
-<a class="jxr_linenumber" name="1158" href="#1158">1158</a> <em class="jxr_javadoccomment">   * @param compaction compaction details obtained from requestCompaction()</em>
-<a class="jxr_linenumber" name="1159" href="#1159">1159</a> <em class="jxr_javadoccomment">   * @throws IOException</em>
-<a class="jxr_linenumber" name="1160" href="#1160">1160</a> <em class="jxr_javadoccomment">   * @return Storefile we compacted into or null if we failed or opted out early.</em>
-<a class="jxr_linenumber" name="1161" href="#1161">1161</a> <em class="jxr_javadoccomment">   */</em>
-<a class="jxr_linenumber" name="1162" href="#1162">1162</a>   @Override
-<a class="jxr_linenumber" name="1163" href="#1163">1163</a>   <strong class="jxr_keyword">public</strong> List&lt;StoreFile&gt; compact(<a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionContext.html">CompactionContext</a> compaction,
-<a class="jxr_linenumber" name="1164" href="#1164">1164</a>       <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionThroughputController.html">CompactionThroughputController</a> throughputController) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1165" href="#1165">1165</a>     <strong class="jxr_keyword">return</strong> compact(compaction, throughputController, <strong class="jxr_keyword">null</strong>);
-<a class="jxr_linenumber" name="1166" href="#1166">1166</a>   }
-<a class="jxr_linenumber" name="1167" href="#1167">1167</a> 
-<a class="jxr_linenumber" name="1168" href="#1168">1168</a>   @Override
-<a class="jxr_linenumber" name="1169" href="#1169">1169</a>   <strong class="jxr_keyword">public</strong> List&lt;StoreFile&gt; compact(<a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionContext.html">CompactionContext</a> compaction,
-<a class="jxr_linenumber" name="1170" href="#1170">1170</a>     <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionThroughputController.html">CompactionThroughputController</a> throughputController, <a href="../../../../../org/apache/hadoop/hbase/security/User.html">User</a> user) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1171" href="#1171">1171</a>     assert compaction != <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1172" href="#1172">1172</a>     List&lt;StoreFile&gt; sfs = <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1173" href="#1173">1173</a>     <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr = compaction.getRequest();
-<a class="jxr_linenumber" name="1174" href="#1174">1174</a>     <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1175" href="#1175">1175</a>       <em class="jxr_comment">// Do all sanity checking in here if we have a valid CompactionRequest</em>
-<a class="jxr_linenumber" name="1176" href="#1176">1176</a>       <em class="jxr_comment">// because we need to clean up after it on the way out in a finally</em>
-<a class="jxr_linenumber" name="1177" href="#1177">1177</a>       <em class="jxr_comment">// block below</em>
-<a class="jxr_linenumber" name="1178" href="#1178">1178</a>       <strong class="jxr_keyword">long</strong> compactionStartTime = EnvironmentEdgeManager.currentTime();
-<a class="jxr_linenumber" name="1179" href="#1179">1179</a>       assert compaction.hasSelection();
-<a class="jxr_linenumber" name="1180" href="#1180">1180</a>       Collection&lt;StoreFile&gt; filesToCompact = cr.getFiles();
-<a class="jxr_linenumber" name="1181" href="#1181">1181</a>       assert !filesToCompact.isEmpty();
-<a class="jxr_linenumber" name="1182" href="#1182">1182</a>       <strong class="jxr_keyword">synchronized</strong> (filesCompacting) {
-<a class="jxr_linenumber" name="1183" href="#1183">1183</a>         <em class="jxr_comment">// sanity check: we're compacting files that this store knows about</em>
-<a class="jxr_linenumber" name="1184" href="#1184">1184</a>         <em class="jxr_comment">// TODO: change this to LOG.error() after more debugging</em>
-<a class="jxr_linenumber" name="1185" href="#1185">1185</a>         Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));
-<a class="jxr_linenumber" name="1186" href="#1186">1186</a>       }
-<a class="jxr_linenumber" name="1187" href="#1187">1187</a> 
-<a class="jxr_linenumber" name="1188" href="#1188">1188</a>       <em class="jxr_comment">// Ready to go. Have list of files to compact.</em>
-<a class="jxr_linenumber" name="1189" href="#1189">1189</a>       LOG.info(<span class="jxr_string">"Starting compaction of "</span> + filesToCompact.size() + <span class="jxr_string">" file(s) in "</span>
-<a class="jxr_linenumber" name="1190" href="#1190">1190</a>           + <strong class="jxr_keyword">this</strong> + <span class="jxr_string">" of "</span> + <strong class="jxr_keyword">this</strong>.getRegionInfo().getRegionNameAsString()
-<a class="jxr_linenumber" name="1191" href="#1191">1191</a>           + <span class="jxr_string">" into tmpdir="</span> + fs.getTempDir() + <span class="jxr_string">", totalSize="</span>
-<a class="jxr_linenumber" name="1192" href="#1192">1192</a>           + TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(cr.getSize(), <span class="jxr_string">""</span>, 1));
-<a class="jxr_linenumber" name="1193" href="#1193">1193</a> 
-<a class="jxr_linenumber" name="1194" href="#1194">1194</a>       <em class="jxr_comment">// Commence the compaction.</em>
-<a class="jxr_linenumber" name="1195" href="#1195">1195</a>       List&lt;Path&gt; newFiles = compaction.compact(throughputController, user);
-<a class="jxr_linenumber" name="1196" href="#1196">1196</a> 
-<a class="jxr_linenumber" name="1197" href="#1197">1197</a>       <em class="jxr_comment">// TODO: get rid of this!</em>
-<a class="jxr_linenumber" name="1198" href="#1198">1198</a>       <strong class="jxr_keyword">if</strong> (!<strong class="jxr_keyword">this</strong>.conf.getBoolean(<span class="jxr_string">"hbase.hstore.compaction.complete"</span>, <strong class="jxr_keyword">true</strong>)) {
-<a class="jxr_linenumber" name="1199" href="#1199">1199</a>         LOG.warn(<span class="jxr_string">"hbase.hstore.compaction.complete is set to false"</span>);
-<a class="jxr_linenumber" name="1200" href="#1200">1200</a>         sfs = <strong class="jxr_keyword">new</strong> ArrayList&lt;StoreFile&gt;(newFiles.size());
-<a class="jxr_linenumber" name="1201" href="#1201">1201</a>         <strong class="jxr_keyword">final</strong> <strong class="jxr_keyword">boolean</strong> evictOnClose =
-<a class="jxr_linenumber" name="1202" href="#1202">1202</a>             cacheConf != <strong class="jxr_keyword">null</strong>? cacheConf.shouldEvictOnClose(): <strong class="jxr_keyword">true</strong>;
-<a class="jxr_linenumber" name="1203" href="#1203">1203</a>         <strong class="jxr_keyword">for</strong> (Path newFile : newFiles) {
-<a class="jxr_linenumber" name="1204" href="#1204">1204</a>           <em class="jxr_comment">// Create storefile around what we wrote with a reader on it.</em>
-<a class="jxr_linenumber" name="1205" href="#1205">1205</a>           <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = createStoreFileAndReader(newFile);
-<a class="jxr_linenumber" name="1206" href="#1206">1206</a>           sf.closeReader(evictOnClose);
-<a class="jxr_linenumber" name="1207" href="#1207">1207</a>           sfs.add(sf);
-<a class="jxr_linenumber" name="1208" href="#1208">1208</a>         }
-<a class="jxr_linenumber" name="1209" href="#1209">1209</a>         <strong class="jxr_keyword">return</strong> sfs;
+<a class="jxr_linenumber" name="1156" href="#1156">1156</a> <em class="jxr_javadoccomment">   * &lt;p&gt;We don't want to hold the structureLock for the whole time, as a compact()</em>
+<a class="jxr_linenumber" name="1157" href="#1157">1157</a> <em class="jxr_javadoccomment">   * can be lengthy and we want to allow cache-flushes during this period.</em>
+<a class="jxr_linenumber" name="1158" href="#1158">1158</a> <em class="jxr_javadoccomment">   *</em>
+<a class="jxr_linenumber" name="1159" href="#1159">1159</a> <em class="jxr_javadoccomment">   * &lt;p&gt; Compaction event should be idempotent, since there is no IO Fencing for</em>
+<a class="jxr_linenumber" name="1160" href="#1160">1160</a> <em class="jxr_javadoccomment">   * the region directory in hdfs. A region server might still try to complete the</em>
+<a class="jxr_linenumber" name="1161" href="#1161">1161</a> <em class="jxr_javadoccomment">   * compaction after it lost the region. That is why the following events are carefully</em>
+<a class="jxr_linenumber" name="1162" href="#1162">1162</a> <em class="jxr_javadoccomment">   * ordered for a compaction:</em>
+<a class="jxr_linenumber" name="1163" href="#1163">1163</a> <em class="jxr_javadoccomment">   *  1. Compaction writes new files under region/.tmp directory (compaction output)</em>
+<a class="jxr_linenumber" name="1164" href="#1164">1164</a> <em class="jxr_javadoccomment">   *  2. Compaction atomically moves the temporary file under region directory</em>
+<a class="jxr_linenumber" name="1165" href="#1165">1165</a> <em class="jxr_javadoccomment">   *  3. Compaction appends a WAL edit containing the compaction input and output files.</em>
+<a class="jxr_linenumber" name="1166" href="#1166">1166</a> <em class="jxr_javadoccomment">   *  Forces sync on WAL.</em>
+<a class="jxr_linenumber" name="1167" href="#1167">1167</a> <em class="jxr_javadoccomment">   *  4. Compaction deletes the input files from the region directory.</em>
+<a class="jxr_linenumber" name="1168" href="#1168">1168</a> <em class="jxr_javadoccomment">   *</em>
+<a class="jxr_linenumber" name="1169" href="#1169">1169</a> <em class="jxr_javadoccomment">   * Failure conditions are handled like this:</em>
+<a class="jxr_linenumber" name="1170" href="#1170">1170</a> <em class="jxr_javadoccomment">   *  - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes</em>
+<a class="jxr_linenumber" name="1171" href="#1171">1171</a> <em class="jxr_javadoccomment">   *  the compaction later, it will only write the new data file to the region directory.</em>
+<a class="jxr_linenumber" name="1172" href="#1172">1172</a> <em class="jxr_javadoccomment">   *  Since we already have this data, this will be idempotent but we will have a redundant</em>
+<a class="jxr_linenumber" name="1173" href="#1173">1173</a> <em class="jxr_javadoccomment">   *  copy of the data.</em>
+<a class="jxr_linenumber" name="1174" href="#1174">1174</a> <em class="jxr_javadoccomment">   *  - If RS fails between 2 and 3, the region will have a redundant copy of the data. The</em>
+<a class="jxr_linenumber" name="1175" href="#1175">1175</a> <em class="jxr_javadoccomment">   *  RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.</em>
+<a class="jxr_linenumber" name="1176" href="#1176">1176</a> <em class="jxr_javadoccomment">   *  - If RS fails after 3, the region region server who opens the region will pick up the</em>
+<a class="jxr_linenumber" name="1177" href="#1177">1177</a> <em class="jxr_javadoccomment">   *  the compaction marker from the WAL and replay it by removing the compaction input files.</em>
+<a class="jxr_linenumber" name="1178" href="#1178">1178</a> <em class="jxr_javadoccomment">   *  Failed RS can also attempt to delete those files, but the operation will be idempotent</em>
+<a class="jxr_linenumber" name="1179" href="#1179">1179</a> <em class="jxr_javadoccomment">   *</em>
+<a class="jxr_linenumber" name="1180" href="#1180">1180</a> <em class="jxr_javadoccomment">   * See HBASE-2231 for details.</em>
+<a class="jxr_linenumber" name="1181" href="#1181">1181</a> <em class="jxr_javadoccomment">   *</em>
+<a class="jxr_linenumber" name="1182" href="#1182">1182</a> <em class="jxr_javadoccomment">   * @param compaction compaction details obtained from requestCompaction()</em>
+<a class="jxr_linenumber" name="1183" href="#1183">1183</a> <em class="jxr_javadoccomment">   * @throws IOException</em>
+<a class="jxr_linenumber" name="1184" href="#1184">1184</a> <em class="jxr_javadoccomment">   * @return Storefile we compacted into or null if we failed or opted out early.</em>
+<a class="jxr_linenumber" name="1185" href="#1185">1185</a> <em class="jxr_javadoccomment">   */</em>
+<a class="jxr_linenumber" name="1186" href="#1186">1186</a>   @Override
+<a class="jxr_linenumber" name="1187" href="#1187">1187</a>   <strong class="jxr_keyword">public</strong> List&lt;StoreFile&gt; compact(<a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionContext.html">CompactionContext</a> compaction,
+<a class="jxr_linenumber" name="1188" href="#1188">1188</a>       <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionThroughputController.html">CompactionThroughputController</a> throughputController) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1189" href="#1189">1189</a>     <strong class="jxr_keyword">return</strong> compact(compaction, throughputController, <strong class="jxr_keyword">null</strong>);
+<a class="jxr_linenumber" name="1190" href="#1190">1190</a>   }
+<a class="jxr_linenumber" name="1191" href="#1191">1191</a> 
+<a class="jxr_linenumber" name="1192" href="#1192">1192</a>   @Override
+<a class="jxr_linenumber" name="1193" href="#1193">1193</a>   <strong class="jxr_keyword">public</strong> List&lt;StoreFile&gt; compact(<a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionContext.html">CompactionContext</a> compaction,
+<a class="jxr_linenumber" name="1194" href="#1194">1194</a>     <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionThroughputController.html">CompactionThroughputController</a> throughputController, <a href="../../../../../org/apache/hadoop/hbase/security/User.html">User</a> user) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1195" href="#1195">1195</a>     assert compaction != <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1196" href="#1196">1196</a>     List&lt;StoreFile&gt; sfs = <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1197" href="#1197">1197</a>     <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr = compaction.getRequest();
+<a class="jxr_linenumber" name="1198" href="#1198">1198</a>     <strong class="jxr_keyword">try</strong> {
+<a class="jxr_linenumber" name="1199" href="#1199">1199</a>       <em class="jxr_comment">// Do all sanity checking in here if we have a valid CompactionRequest</em>
+<a class="jxr_linenumber" name="1200" href="#1200">1200</a>       <em class="jxr_comment">// because we need to clean up after it on the way out in a finally</em>
+<a class="jxr_linenumber" name="1201" href="#1201">1201</a>       <em class="jxr_comment">// block below</em>
+<a class="jxr_linenumber" name="1202" href="#1202">1202</a>       <strong class="jxr_keyword">long</strong> compactionStartTime = EnvironmentEdgeManager.currentTime();
+<a class="jxr_linenumber" name="1203" href="#1203">1203</a>       assert compaction.hasSelection();
+<a class="jxr_linenumber" name="1204" href="#1204">1204</a>       Collection&lt;StoreFile&gt; filesToCompact = cr.getFiles();
+<a class="jxr_linenumber" name="1205" href="#1205">1205</a>       assert !filesToCompact.isEmpty();
+<a class="jxr_linenumber" name="1206" href="#1206">1206</a>       <strong class="jxr_keyword">synchronized</strong> (filesCompacting) {
+<a class="jxr_linenumber" name="1207" href="#1207">1207</a>         <em class="jxr_comment">// sanity check: we're compacting files that this store knows about</em>
+<a class="jxr_linenumber" name="1208" href="#1208">1208</a>         <em class="jxr_comment">// TODO: change this to LOG.error() after more debugging</em>
+<a class="jxr_linenumber" name="1209" href="#1209">1209</a>         Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));
 <a class="jxr_linenumber" name="1210" href="#1210">1210</a>       }
-<a class="jxr_linenumber" name="1211" href="#1211">1211</a>       <em class="jxr_comment">// Do the steps necessary to complete the compaction.</em>
-<a class="jxr_linenumber" name="1212" href="#1212">1212</a>       sfs = moveCompatedFilesIntoPlace(cr, newFiles, user);
-<a class="jxr_linenumber" name="1213" href="#1213">1213</a>       writeCompactionWalRecord(filesToCompact, sfs);
-<a class="jxr_linenumber" name="1214" href="#1214">1214</a>       replaceStoreFiles(filesToCompact, sfs);
-<a class="jxr_linenumber" name="1215" href="#1215">1215</a>       <strong class="jxr_keyword">if</strong> (cr.isMajor()) {
-<a class="jxr_linenumber" name="1216" href="#1216">1216</a>         majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;
-<a class="jxr_linenumber" name="1217" href="#1217">1217</a>         majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;
-<a class="jxr_linenumber" name="1218" href="#1218">1218</a>       } <strong class="jxr_keyword">else</strong> {
-<a class="jxr_linenumber" name="1219" href="#1219">1219</a>         compactedCellsCount += getCompactionProgress().totalCompactingKVs;
-<a class="jxr_linenumber" name="1220" href="#1220">1220</a>         compactedCellsSize += getCompactionProgress().totalCompactedSize;
-<a class="jxr_linenumber" name="1221" href="#1221">1221</a>       }
-<a class="jxr_linenumber" name="1222" href="#1222">1222</a>       <em class="jxr_comment">// At this point the store will use new files for all new scanners.</em>
-<a class="jxr_linenumber" name="1223" href="#1223">1223</a>       completeCompaction(filesToCompact); <em class="jxr_comment">// update store size.</em>
-<a class="jxr_linenumber" name="1224" href="#1224">1224</a> 
-<a class="jxr_linenumber" name="1225" href="#1225">1225</a>       logCompactionEndMessage(cr, sfs, compactionStartTime);
-<a class="jxr_linenumber" name="1226" href="#1226">1226</a>       <strong class="jxr_keyword">return</strong> sfs;
-<a class="jxr_linenumber" name="1227" href="#1227">1227</a>     } <strong class="jxr_keyword">finally</strong> {
-<a class="jxr_linenumber" name="1228" href="#1228">1228</a>       finishCompactionRequest(cr);
-<a class="jxr_linenumber" name="1229" href="#1229">1229</a>     }
-<a class="jxr_linenumber" name="1230" href="#1230">1230</a>   }
-<a class="jxr_linenumber" name="1231" href="#1231">1231</a> 
-<a class="jxr_linenumber" name="1232" href="#1232">1232</a>   <strong class="jxr_keyword">private</strong> List&lt;StoreFile&gt; moveCompatedFilesIntoPlace(
-<a class="jxr_linenumber" name="1233" href="#1233">1233</a>       <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr, List&lt;Path&gt; newFiles, <a href="../../../../../org/apache/hadoop/hbase/security/User.html">User</a> user) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1234" href="#1234">1234</a>     List&lt;StoreFile&gt; sfs = <strong class="jxr_keyword">new</strong> ArrayList&lt;StoreFile&gt;(newFiles.size());
-<a class="jxr_linenumber" name="1235" href="#1235">1235</a>     <strong class="jxr_keyword">for</strong> (Path newFile : newFiles) {
-<a class="jxr_linenumber" name="1236" href="#1236">1236</a>       assert newFile != <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1237" href="#1237">1237</a>       <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = moveFileIntoPlace(newFile);
-<a class="jxr_linenumber" name="1238" href="#1238">1238</a>       <strong class="jxr_keyword">if</strong> (<strong class="jxr_keyword">this</strong>.getCoprocessorHost() != <strong class="jxr_keyword">null</strong>) {
-<a class="jxr_linenumber" name="1239" href="#1239">1239</a>         <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/Store.html">Store</a> thisStore = <strong class="jxr_keyword">this</strong>;
-<a class="jxr_linenumber" name="1240" href="#1240">1240</a>         <strong class="jxr_keyword">if</strong> (user == <strong class="jxr_keyword">null</strong>) {
-<a class="jxr_linenumber" name="1241" href="#1241">1241</a>           getCoprocessorHost().postCompact(thisStore, sf, cr);
-<a class="jxr_linenumber" name="1242" href="#1242">1242</a>         } <strong class="jxr_keyword">else</strong> {
-<a class="jxr_linenumber" name="1243" href="#1243">1243</a>           <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1244" href="#1244">1244</a>             user.getUGI().doAs(<strong class="jxr_keyword">new</strong> PrivilegedExceptionAction&lt;Void&gt;() {
-<a class="jxr_linenumber" name="1245" href="#1245">1245</a>               @Override
-<a class="jxr_linenumber" name="1246" href="#1246">1246</a>               <strong class="jxr_keyword">public</strong> Void run() <strong class="jxr_keyword">throws</strong> Exception {
-<a class="jxr_linenumber" name="1247" href="#1247">1247</a>                 getCoprocessorHost().postCompact(thisStore, sf, cr);
-<a class="jxr_linenumber" name="1248" href="#1248">1248</a>                 <strong class="jxr_keyword">return</strong> <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1249" href="#1249">1249</a>               }
-<a class="jxr_linenumber" name="1250" href="#1250">1250</a>             });
-<a class="jxr_linenumber" name="1251" href="#1251">1251</a>           } <strong class="jxr_keyword">catch</strong> (InterruptedException ie) {
-<a class="jxr_linenumber" name="1252" href="#1252">1252</a>             InterruptedIOException iioe = <strong class="jxr_keyword">new</strong> InterruptedIOException();
-<a class="jxr_linenumber" name="1253" href="#1253">1253</a>             iioe.initCause(ie);
-<a class="jxr_linenumber" name="1254" href="#1254">1254</a>             <strong class="jxr_keyword">throw</strong> iioe;
-<a class="jxr_linenumber" name="1255" href="#1255">1255</a>           }
-<a class="jxr_linenumber" name="1256" href="#1256">1256</a>         }
-<a class="jxr_linenumber" name="1257" href="#1257">1257</a>       }
-<a class="jxr_linenumber" name="1258" href="#1258">1258</a>       assert sf != <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1259" href="#1259">1259</a>       sfs.add(sf);
-<a class="jxr_linenumber" name="1260" href="#1260">1260</a>     }
-<a class="jxr_linenumber" name="1261" href="#1261">1261</a>     <strong class="jxr_keyword">return</strong> sfs;
-<a class="jxr_linenumber" name="1262" href="#1262">1262</a>   }
-<a class="jxr_linenumber" name="1263" href="#1263">1263</a> 
-<a class="jxr_linenumber" name="1264" href="#1264">1264</a>   <em class="jxr_comment">// Package-visible for tests</em>
-<a class="jxr_linenumber" name="1265" href="#1265">1265</a>   <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> moveFileIntoPlace(<strong class="jxr_keyword">final</strong> Path newFile) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1266" href="#1266">1266</a>     validateStoreFile(newFile);
-<a class="jxr_linenumber" name="1267" href="#1267">1267</a>     <em class="jxr_comment">// Move the file into the right spot</em>
-<a class="jxr_linenumber" name="1268" href="#1268">1268</a>     Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);
-<a class="jxr_linenumber" name="1269" href="#1269">1269</a>     <strong class="jxr_keyword">return</strong> createStoreFileAndReader(destPath);
-<a class="jxr_linenumber" name="1270" href="#1270">1270</a>   }
-<a class="jxr_linenumber" name="1271" href="#1271">1271</a> 
-<a class="jxr_linenumber" name="1272" href="#1272">1272</a>   <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1273" href="#1273">1273</a> <em class="jxr_javadoccomment">   * Writes the compaction WAL record.</em>
-<a class="jxr_linenumber" name="1274" href="#1274">1274</a> <em class="jxr_javadoccomment">   * @param filesCompacted Files compacted (input).</em>
-<a class="jxr_linenumber" name="1275" href="#1275">1275</a> <em class="jxr_javadoccomment">   * @param newFiles Files from compaction.</em>
-<a class="jxr_linenumber" name="1276" href="#1276">1276</a> <em class="jxr_javadoccomment">   */</em>
-<a class="jxr_linenumber" name="1277" href="#1277">1277</a>   <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> writeCompactionWalRecord(Collection&lt;StoreFile&gt; filesCompacted,
-<a class="jxr_linenumber" name="1278" href="#1278">1278</a>       Collection&lt;StoreFile&gt; newFiles) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1279" href="#1279">1279</a>     <strong class="jxr_keyword">if</strong> (region.getWAL() == <strong class="jxr_keyword">null</strong>) <strong class="jxr_keyword">return</strong>;
-<a class="jxr_linenumber" name="1280" href="#1280">1280</a>     List&lt;Path&gt; inputPaths = <strong class="jxr_keyword">new</strong> ArrayList&lt;Path&gt;(filesCompacted.size());
-<a class="jxr_linenumber" name="1281" href="#1281">1281</a>     <strong class="jxr_keyword">for</strong> (StoreFile f : filesCompacted) {
-<a class="jxr_linenumber" name="1282" href="#1282">1282</a>       inputPaths.add(f.getPath());
-<a class="jxr_linenumber" name="1283" href="#1283">1283</a>     }
-<a class="jxr_linenumber" name="1284" href="#1284">1284</a>     List&lt;Path&gt; outputPaths = <strong class="jxr_keyword">new</strong> ArrayList&lt;Path&gt;(newFiles.size());
-<a class="jxr_linenumber" name="1285" href="#1285">1285</a>     <strong class="jxr_keyword">for</strong> (StoreFile f : newFiles) {
-<a class="jxr_linenumber" name="1286" href="#1286">1286</a>       outputPaths.add(f.getPath());
-<a class="jxr_linenumber" name="1287" href="#1287">1287</a>     }
-<a class="jxr_linenumber" name="1288" href="#1288">1288</a>     <a href="../../../../../org/apache/hadoop/hbase/HRegionInfo.html">HRegionInfo</a> info = <strong class="jxr_keyword">this</strong>.region.getRegionInfo();
-<a class="jxr_linenumber" name="1289" href="#1289">1289</a>     CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
-<a class="jxr_linenumber" name="1290" href="#1290">1290</a>         family.getName(), inputPaths, outputPaths, fs.getStoreDir(getFamily().getNameAsString()));
-<a class="jxr_linenumber" name="1291" href="#1291">1291</a>     WALUtil.writeCompactionMarker(region.getWAL(), <strong class="jxr_keyword">this</strong>.region.getTableDesc(),
-<a class="jxr_linenumber" name="1292" href="#1292">1292</a>         <strong class="jxr_keyword">this</strong>.region.getRegionInfo(), compactionDescriptor, region.getMVCC());
-<a class="jxr_linenumber" name="1293" href="#1293">1293</a>   }
-<a class="jxr_linenumber" name="1294" href="#1294">1294</a> 
-<a class="jxr_linenumber" name="1295" href="#1295">1295</a>   @VisibleForTesting
-<a class="jxr_linenumber" name="1296" href="#1296">1296</a>   <strong class="jxr_keyword">void</strong> replaceStoreFiles(<strong class="jxr_keyword">final</strong> Collection&lt;StoreFile&gt; compactedFiles,
-<a class="jxr_linenumber" name="1297" href="#1297">1297</a>       <strong class="jxr_keyword">final</strong> Collection&lt;StoreFile&gt; result) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1298" href="#1298">1298</a>     <strong class="jxr_keyword">this</strong>.lock.writeLock().lock();
-<a class="jxr_linenumber" name="1299" href="#1299">1299</a>     <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1300" href="#1300">1300</a>       <strong class="jxr_keyword">this</strong>.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result);
-<a class="jxr_linenumber" name="1301" href="#1301">1301</a>       filesCompacting.removeAll(compactedFiles); <em class="jxr_comment">// safe bc: lock.writeLock();</em>
-<a class="jxr_linenumber" name="1302" href="#1302">1302</a>     } <strong class="jxr_keyword">finally</strong> {
-<a class="jxr_linenumber" name="1303" href="#1303">1303</a>       <strong class="jxr_keyword">this</strong>.lock.writeLock().unlock();
-<a class="jxr_linenumber" name="1304" href="#1304">1304</a>     }
-<a class="jxr_linenumber" name="1305" href="#1305">1305</a>   }
-<a class="jxr_linenumber" name="1306" href="#1306">1306</a> 
-<a class="jxr_linenumber" name="1307" href="#1307">1307</a>   <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1308" href="#1308">1308</a> <em class="jxr_javadoccomment">   * Log a very elaborate compaction completion message.</em>
-<a class="jxr_linenumber" name="1309" href="#1309">1309</a> <em class="jxr_javadoccomment">   * @param cr Request.</em>
-<a class="jxr_linenumber" name="1310" href="#1310">1310</a> <em class="jxr_javadoccomment">   * @param sfs Resulting files.</em>
-<a class="jxr_linenumber" name="1311" href="#1311">1311</a> <em class="jxr_javadoccomment">   * @param compactionStartTime Start time.</em>
-<a class="jxr_linenumber" name="1312" href="#1312">1312</a> <em class="jxr_javadoccomment">   */</em>
-<a class="jxr_linenumber" name="1313" href="#1313">1313</a>   <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> logCompactionEndMessage(
-<a class="jxr_linenumber" name="1314" href="#1314">1314</a>       <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr, List&lt;StoreFile&gt; sfs, <strong class="jxr_keyword">long</strong> compactionStartTime) {
-<a class="jxr_linenumber" name="1315" href="#1315">1315</a>     <strong class="jxr_keyword">long</strong> now = EnvironmentEdgeManager.currentTime();
-<a class="jxr_linenumber" name="1316" href="#1316">1316</a>     StringBuilder message = <strong class="jxr_keyword">new</strong> StringBuilder(
-<a class="jxr_linenumber" name="1317" href="#1317">1317</a>       <span class="jxr_string">"Completed"</span> + (cr.isMajor() ? <span class="jxr_string">" major"</span> : <span class="jxr_string">""</span>) + <span class="jxr_string">" compaction of "</span>
-<a class="jxr_linenumber" name="1318" href="#1318">1318</a>       + cr.getFiles().size() + (cr.isAllFiles() ? <span class="jxr_string">" (all)"</span> : <span class="jxr_string">""</span>) + <span class="jxr_string">" file(s) in "</span>
-<a class="jxr_linenumber" name="1319" href="#1319">1319</a>       + <strong class="jxr_keyword">this</strong> + <span class="jxr_string">" of "</span> + <strong class="jxr_keyword">this</strong>.getRegionInfo().getRegionNameAsString() + <span class="jxr_string">" into "</span>);
-<a class="jxr_linenumber" name="1320" href="#1320">1320</a>     <strong class="jxr_keyword">if</strong> (sfs.isEmpty()) {
-<a class="jxr_linenumber" name="1321" href="#1321">1321</a>       message.append(<span class="jxr_string">"none, "</span>);
-<a class="jxr_linenumber" name="1322" href="#1322">1322</a>     } <strong class="jxr_keyword">else</strong> {
-<a class="jxr_linenumber" name="1323" href="#1323">1323</a>       <strong class="jxr_keyword">for</strong> (StoreFile sf: sfs) {
-<a class="jxr_linenumber" name="1324" href="#1324">1324</a>         message.append(sf.getPath().getName());
-<a class="jxr_linenumber" name="1325" href="#1325">1325</a>         message.append(<span class="jxr_string">"(size="</span>);
-<a class="jxr_linenumber" name="1326" href="#1326">1326</a>         message.append(TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(sf.getReader().length(), <span class="jxr_string">""</span>, 1));
-<a class="jxr_linenumber" name="1327" href="#1327">1327</a>         message.append(<span class="jxr_string">"), "</span>);
-<a class="jxr_linenumber" name="1328" href="#1328">1328</a>       }
-<a class="jxr_linenumber" name="1329" href="#1329">1329</a>     }
-<a class="jxr_linenumber" name="1330" href="#1330">1330</a>     message.append(<span class="jxr_string">"total size for store is "</span>)
-<a class="jxr_linenumber" name="1331" href="#1331">1331</a>       .append(StringUtils.TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(storeSize, <span class="jxr_string">""</span>, 1))
-<a class="jxr_linenumber" name="1332" href="#1332">1332</a>       .append(<span class="jxr_string">". This selection was in queue for "</span>)
-<a class="jxr_linenumber" name="1333" href="#1333">1333</a>       .append(StringUtils.formatTimeDiff(compactionStartTime, cr.getSelectionTime()))
-<a class="jxr_linenumber" name="1334" href="#1334">1334</a>       .append(<span class="jxr_string">", and took "</span>).append(StringUtils.formatTimeDiff(now, compactionStartTime))
-<a class="jxr_linenumber" name="1335" href="#1335">1335</a>       .append(<span class="jxr_string">" to execute."</span>);
-<a class="jxr_linenumber" name="1336" href="#1336">1336</a>     LOG.info(message.toString());
-<a class="jxr_linenumber" name="1337" href="#1337">1337</a>     <strong class="jxr_keyword">if</strong> (LOG.isTraceEnabled()) {
-<a class="jxr_linenumber" name="1338" href="#1338">1338</a>       <strong class="jxr_keyword">int</strong> fileCount = storeEngine.getStoreFileManager().getStorefileCount();
-<a class="jxr_linenumber" name="1339" href="#1339">1339</a>       <strong class="jxr_keyword">long</strong> resultSize = 0;
-<a class="jxr_linenumber" name="1340" href="#1340">1340</a>       <strong class="jxr_keyword">for</strong> (StoreFile sf : sfs) {
-<a class="jxr_linenumber" name="1341" href="#1341">1341</a>         resultSize += sf.getReader().length();
-<a class="jxr_linenumber" name="1342" href="#1342">1342</a>       }
-<a class="jxr_linenumber" name="1343" href="#1343">1343</a>       String traceMessage = <span class="jxr_string">"COMPACTION start,end,size out,files in,files out,store size,"</span>
-<a class="jxr_linenumber" name="1344" href="#1344">1344</a>         + <span class="jxr_string">"store files ["</span> + compactionStartTime + <span class="jxr_string">","</span> + now + <span class="jxr_string">","</span> + resultSize + <span class="jxr_string">","</span>
-<a class="jxr_linenumber" name="1345" href="#1345">1345</a>           + cr.getFiles().size() + <span class="jxr_string">","</span> + sfs.size() + <span class="jxr_string">","</span> +  storeSize + <span class="jxr_string">","</span> + fileCount + <span class="jxr_string">"]"</span>;
-<a class="jxr_linenumber" name="1346" href="#1346">1346</a>       LOG.trace(traceMessage);
-<a class="jxr_linenumber" name="1347" href="#1347">1347</a>     }
-<a class="jxr_linenumber" name="1348" href="#1348">1348</a>   }
-<a class="jxr_linenumber" name="1349" href="#1349">1349</a> 
-<a class="jxr_linenumber" name="1350" href="#1350">1350</a>   <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1351" href="#1351">1351</a> <em class="jxr_javadoccomment">   * Call to complete a compaction. Its for the case where we find in the WAL a compaction</em>
-<a class="jxr_linenumber" name="1352" href="#1352">1352</a> <em class="jxr_javadoccomment">   * that was not finished.  We could find one recovering a WAL after a regionserver crash.</em>
-<a class="jxr_linenumber" name="1353" href="#1353">1353</a> <em class="jxr_javadoccomment">   * See HBASE-2231.</em>
-<a class="jxr_linenumber" name="1354" href="#1354">1354</a> <em class="jxr_javadoccomment">   * @param compaction</em>
-<a class="jxr_linenumber" name="1355" href="#1355">1355</a> <em class="jxr_javadoccomment">   */</em>
-<a class="jxr_linenumber" name="1356" href="#1356">1356</a>   @Override
-<a class="jxr_linenumber" name="1357" href="#1357">1357</a>   <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> replayCompactionMarker(CompactionDescriptor compaction,
-<a class="jxr_linenumber" name="1358" href="#1358">1358</a>       <strong class="jxr_keyword">boolean</strong> pickCompactionFiles, <strong class="jxr_keyword">boolean</strong> removeFiles)
-<a class="jxr_linenumber" name="1359" href="#1359">1359</a>       <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1360" href="#1360">1360</a>     LOG.debug(<span class="jxr_string">"Completing compaction from the WAL marker"</span>);
-<a class="jxr_linenumber" name="1361" href="#1361">1361</a>     List&lt;String&gt; compactionInputs = compaction.getCompactionInputList();
-<a class="jxr_linenumber" name="1362" href="#1362">1362</a>     List&lt;String&gt; compactionOutputs = Lists.newArrayList(compaction.getCompactionOutputList());
-<a class="jxr_linenumber" name="1363" href="#1363">1363</a> 
-<a class="jxr_linenumber" name="1364" href="#1364">1364</a>     <em class="jxr_comment">// The Compaction Marker is written after the compaction is completed,</em>
-<a class="jxr_linenumber" name="1365" href="#1365">1365</a>     <em class="jxr_comment">// and the files moved into the region/family folder.</em>
-<a class="jxr_linenumber" name="1366" href="#1366">1366</a>     <em class="jxr_comment">//</em>
-<a class="jxr_linenumber" name="1367" href="#1367">1367</a>     <em class="jxr_comment">// If we crash after the entry is written, we may not have removed the</em>
-<a class="jxr_linenumber" name="1368" href="#1368">1368</a>     <em class="jxr_comment">// input files, but the output file is present.</em>
-<a class="jxr_linenumber" name="1369" href="#1369">1369</a>     <em class="jxr_comment">// (The unremoved input files will be removed by this function)</em>
-<a class="jxr_linenumber" name="1370" href="#1370">1370</a>     <em class="jxr_comment">//</em>
-<a class="jxr_linenumber" name="1371" href="#1371">1371</a>     <em class="jxr_comment">// If we scan the directory and the file is not present, it can mean that:</em>
-<a class="jxr_linenumber" name="1372" href="#1372">1372</a>     <em class="jxr_comment">//   - The file was manually removed by the user</em>
-<a class="jxr_linenumber" name="1373" href="#1373">1373</a>     <em class="jxr_comment">//   - The file was removed as consequence of subsequent compaction</em>
-<a class="jxr_linenumber" name="1374" href="#1374">1374</a>     <em class="jxr_comment">// so, we can't do anything with the "compaction output list" because those</em>
-<a class="jxr_linenumber" name="1375" href="#1375">1375</a>     <em class="jxr_comment">// files have already been loaded when opening the region (by virtue of</em>
-<a class="jxr_linenumber" name="1376" href="#1376">1376</a>     <em class="jxr_comment">// being in the store's folder) or they may be missing due to a compaction.</em>
-<a class="jxr_linenumber" name="1377" href="#1377">1377</a> 
-<a class="jxr_linenumber" name="1378" href="#1378">1378</a>     String familyName = <strong class="jxr_keyword">this</strong>.getColumnFamilyName();
-<a class="jxr_linenumber" name="1379" href="#1379">1379</a>     List&lt;String&gt; inputFiles = <strong class="jxr_keyword">new</strong> ArrayList&lt;String&gt;(compactionInputs.size());
-<a class="jxr_linenumber" name="1380" href="#1380">1380</a>     <strong class="jxr_keyword">for</strong> (String compactionInput : compactionInputs) {
-<a class="jxr_linenumber" name="1381" href="#1381">1381</a>       Path inputPath = fs.getStoreFilePath(familyName, compactionInput);
-<a class="jxr_linenumber" name="1382" href="#1382">1382</a>       inputFiles.add(inputPath.getName());
-<a class="jxr_linenumber" name="1383" href="#1383">1383</a>     }
-<a class="jxr_linenumber" name="1384" href="#1384">1384</a> 
-<a class="jxr_linenumber" name="1385" href="#1385">1385</a>     <em class="jxr_comment">//some of the input files might already be deleted</em>
-<a class="jxr_linenumber" name="1386" href="#1386">1386</a>     List&lt;StoreFile&gt; inputStoreFiles = <strong class="jxr_keyword">new</strong> ArrayList&lt;StoreFile&gt;(compactionInputs.size());
-<a class="jxr_linenumber" name="1387" href="#1387">1387</a>     <strong class="jxr_keyword">for</strong> (StoreFile sf : <strong class="jxr_keyword">this</strong>.getStorefiles()) {
-<a class="jxr_linenumber" name="1388" href="#1388">1388</a>       <strong class="jxr_keyword">if</strong> (inputFiles.contains(sf.getPath().getName())) {
-<a class="jxr_linenumber" name="1389" href="#1389">1389</a>         inputStoreFiles.add(sf);
-<a class="jxr_linenumber" name="1390" href="#1390">1390</a>       }
-<a class="jxr_linenumber" name="1391" href="#1391">1391</a>     }
-<a class="jxr_linenumber" name="1392" href="#1392">1392</a> 
-<a class="jxr_linenumber" name="1393" href="#1393">1393</a>     <em class="jxr_comment">// check whether we need to pick up the new files</em>
-<a class="jxr_linenumber" name="1394" href="#1394">1394</a>     List&lt;StoreFile&gt; outputStoreFiles = <strong class="jxr_keyword">new</strong> ArrayList&lt;StoreFile&gt;(compactionOutputs.size());
-<a class="jxr_linenumber" name="1395" href="#1395">1395</a> 
-<a class="jxr_linenumber" name="1396" href="#1396">1396</a>     <strong class="jxr_keyword">if</strong> (pickCompactionFiles) {
-<a class="jxr_linenumber" name="1397" href="#1397">1397</a>       <strong class="jxr_keyword">for</strong> (StoreFile sf : <strong class="jxr_keyword">this</strong>.getStorefiles()) {
-<a class="jxr_linenumber" name="1398" href="#1398">1398</a>         compactionOutputs.remove(sf.getPath().getName());
-<a class="jxr_linenumber" name="1399" href="#1399">1399</a>       }
-<a class="jxr_linenumber" name="1400" href="#1400">1400</a>       <strong class="jxr_keyword">for</strong> (String compactionOutput : compactionOutputs) {
-<a class="jxr_linenumber" name="1401" href="#1401">1401</a>         <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFileInfo.html">StoreFileInfo</a> storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), compactionOutput);
-<a class="jxr_linenumber" name="1402" href="#1402">1402</a>         <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> storeFile = createStoreFileAndReader(storeFileInfo);
-<a class="jxr_linenumber" name="1403" href="#1403">1403</a>         outputStoreFiles.add(storeFile);
-<a class="jxr_linenumber" name="1404" href="#1404">1404</a>       }
-<a class="jxr_linenumber" name="1405" href="#1405">1405</a>     }
-<a class="jxr_linenumber" name="1406" href="#1406">1406</a> 
-<a class="jxr_linenumber" name="1407" href="#1407">1407</a>     <strong class="jxr_keyword">if</strong> (!inputStoreFiles.isEmpty() || !outputStoreFiles.isEmpty()) {
-<a class="jxr_linenumber" name="1408" href="#1408">1408</a>       LOG.info(<span class="jxr_string">"Replaying compaction marker, replacing input files: "</span> +
-<a class="jxr_linenumber" name="1409" href="#1409">1409</a>           inputStoreFiles + <span class="jxr_string">" with output files : "</span> + outputStoreFiles);
-<a class="jxr_linenumber" name="1410" href="#1410">1410</a>       <strong class="jxr_keyword">this</strong>.replaceStoreFiles(inputStoreFiles, outputStoreFiles);
-<a class="jxr_linenumber" name="1411" href="#1411">1411</a>       <strong class="jxr_keyword">this</strong>.completeCompaction(inputStoreFiles);
-<a class="jxr_linenumber" name="1412" href="#1412">1412</a>     }
-<a class="jxr_linenumber" name="1413" href="#1413">1413</a>   }
-<a class="jxr_linenumber" name="1414" href="#1414">1414</a> 
-<a class="jxr_linenumber" name="1415" href="#1415">1415</a>   <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1416" href="#1416">1416</a> <em class="jxr_javadoccomment">   * This method tries to compact N recent files for testing.</em>
-<a class="jxr_linenumber" name="1417" href="#1417">1417</a> <em class="jxr_javadoccomment">   * Note that because compacting "recent" files only makes sense for some policies,</em>
-<a class="jxr_linenumber" name="1418" href="#1418">1418</a> <em class="jxr_javadoccomment">   * e.g. the default one, it assumes default policy is used. It doesn't use policy,</em>
-<a class="jxr_linenumber" name="1419" href="#1419">1419</a> <em class="jxr_javadoccomment">   * but instead makes a compaction candidate list by itself.</em>
-<a class="jxr_linenumber" name="1420" href="#1420">1420</a> <em class="jxr_javadoccomment">   * @param N Number of files.</em>
-<a class="jxr_linenumber" name="1421" href="#1421">1421</a> <em class="jxr_javadoccomment">   */</em>
-<a class="jxr_linenumber" name="1422" href="#1422">1422</a>   <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> compactRecentForTestingAssumingDefaultPolicy(<strong class="jxr_keyword">int</strong> N) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1423" href="#1423">1423</a>     List&lt;StoreFile&gt; filesToCompact;
-<a class="jxr_linenumber" name="1424" href="#1424">1424</a>     <strong class="jxr_keyword">boolean</strong> isMajor;
-<a class="jxr_linenumber" name="1425" href="#1425">1425</a> 
-<a class="jxr_linenumber" name="1426" href="#1426">1426</a>     <strong class="jxr_keyword">this</strong>.lock.readLock().lock();
-<a class="jxr_linenumber" name="1427" href="#1427">1427</a>     <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1428" href="#1428">1428</a>       <strong class="jxr_keyword">synchronized</strong> (filesCompacting) {
-<a class="jxr_linenumber" name="1429" href="#1429">1429</a>         filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles());
-<a class="jxr_linenumber" name="1430" href="#1430">1430</a>         <strong class="jxr_keyword">if</strong> (!filesCompacting.isEmpty()) {
-<a class="jxr_linenumber" name="1431" href="#1431">1431</a>           <em class="jxr_comment">// exclude all files older than the newest file we're currently</em>
-<a class="jxr_linenumber" name="1432" href="#1432">1432</a>           <em class="jxr_comment">// compacting. this allows us to preserve contiguity (HBASE-2856)</em>
-<a class="jxr_linenumber" name="1433" href="#1433">1433</a>           <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> last = filesCompacting.get(filesCompacting.size() - 1);
-<a class="jxr_linenumber" name="1434" href="#1434">1434</a>           <strong class="jxr_keyword">int</strong> idx = filesToCompact.indexOf(last);
-<a class="jxr_linenumber" name="1435" href="#1435">1435</a>           Preconditions.checkArgument(idx != -1);
-<a class="jxr_linenumber" name="1436" href="#1436">1436</a>           filesToCompact.subList(0, idx + 1).clear();
-<a class="jxr_linenumber" name="1437" href="#1437">1437</a>         }
-<a class="jxr_linenumber" name="1438" href="#1438">1438</a>         <strong class="jxr_keyword">int</strong> count = filesToCompact.size();
-<a class="jxr_linenumber" name="1439" href="#1439">1439</a>         <strong class="jxr_keyword">if</strong> (N &gt; count) {
-<a class="jxr_linenumber" name="1440" href="#1440">1440</a>           <strong class="jxr_keyword">throw</strong> <strong class="jxr_keyword">new</strong> RuntimeException(<span class="jxr_string">"Not enough files"</span>);
-<a class="jxr_linenumber" name="1441" href="#1441">1441</a>         }
-<a class="jxr_linenumber" name="1442" href="#1442">1442</a> 
-<a class="jxr_linenumber" name="1443" href="#1443">1443</a>         filesToCompact = filesToCompact.subList(count - N, count);
-<a class="jxr_linenumber" name="1444" href="#1444">1444</a>         isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount());
-<a class="jxr_linenumber" name="1445" href="#1445">1445</a>         filesCompacting.addAll(filesToCompact);
-<a class="jxr_linenumber" name="1446" href="#1446">1446</a>         Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
-<a class="jxr_linenumber" name="1447" href="#1447">1447</a>       }
-<a class="jxr_linenumber" name="1448" href="#1448">1448</a>     } <strong class="jxr_keyword">finally</strong> {
-<a class="jxr_linenumber" name="1449" href="#1449">1449</a>       <strong class="jxr_keyword">this</strong>.lock.readLock().unlock();
-<a class="jxr_linenumber" name="1450" href="#1450">1450</a>     }
-<a class="jxr_linenumber" name="1451" href="#1451">1451</a> 
-<a class="jxr_linenumber" name="1452" href="#1452">1452</a>     <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1453" href="#1453">1453</a>       <em class="jxr_comment">// Ready to go. Have list of files to compact.</em>
-<a class="jxr_linenumber" name="1454" href="#1454">1454</a>       List&lt;Path&gt; newFiles = ((DefaultCompactor)<strong class="jxr_keyword">this</strong>.storeEngine.getCompactor())
-<a class="jxr_linenumber" name="1455" href="#1455">1455</a>           .compactForTesting(filesToCompact, isMajor);
-<a class="jxr_linenumber" name="1456" href="#1456">1456</a>       <strong class="jxr_keyword">for</strong> (Path newFile: newFiles) {
-<a class="jxr_linenumber" name="1457" href="#1457">1457</a>         <em class="jxr_comment">// Move the compaction into place.</em>
-<a class="jxr_linenumber" name="1458" href="#1458">1458</a>         <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = moveFileIntoPlace(newFile);
-<a class="jxr_linenumber" name="1459" href="#1459">1459</a>         <strong class="jxr_keyword">if</strong> (<strong class="jxr_keyword">this</strong>.getCoprocessorHost() != <strong class="jxr_keyword">null</strong>) {
-<a class="jxr_linenumber" name="1460" href="#1460">1460</a>           <strong class="jxr_keyword">this</strong>.getCoprocessorHost().postCompact(<strong class="jxr_keyword">this</strong>, sf, <strong class="jxr_keyword">null</strong>);
+<a class="jxr_linenumber" name="1211" href="#1211">1211</a> 
+<a class="jxr_linenumber" name="1212" href="#1212">1212</a>       <em class="jxr_comment">// Ready to go. Have list of files to compact.</em>
+<a class="jxr_linenumber" name="1213" href="#1213">1213</a>       LOG.info(<span class="jxr_string">"Starting compaction of "</span> + filesToCompact.size() + <span class="jxr_string">" file(s) in "</span>
+<a class="jxr_linenumber" name="1214" href="#1214">1214</a>           + <strong class="jxr_keyword">this</strong> + <span class="jxr_string">" of "</span> + <strong class="jxr_keyword">this</strong>.getRegionInfo().getRegionNameAsString()
+<a class="jxr_linenumber" name="1215" href="#1215">1215</a>           + <span class="jxr_string">" into tmpdir="</span> + fs.getTempDir() + <span class="jxr_string">", totalSize="</span>
+<a class="jxr_linenumber" name="1216" href="#1216">1216</a>           + TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(cr.getSize(), <span class="jxr_string">""</span>, 1));
+<a class="jxr_linenumber" name="1217" href="#1217">1217</a> 
+<a class="jxr_linenumber" name="1218" href="#1218">1218</a>       <em class="jxr_comment">// Commence the compaction.</em>
+<a class="jxr_linenumber" name="1219" href="#1219">1219</a>       List&lt;Path&gt; newFiles = compaction.compact(throughputController, user);
+<a class="jxr_linenumber" name="1220" href="#1220">1220</a> 
+<a class="jxr_linenumber" name="1221" href="#1221">1221</a>       <em class="jxr_comment">// TODO: get rid of this!</em>
+<a class="jxr_linenumber" name="1222" href="#1222">1222</a>       <strong class="jxr_keyword">if</strong> (!<strong class="jxr_keyword">this</strong>.conf.getBoolean(<span class="jxr_string">"hbase.hstore.compaction.complete"</span>, <strong class="jxr_keyword">true</strong>)) {
+<a class="jxr_linenumber" name="1223" href="#1223">1223</a>         LOG.warn(<span class="jxr_string">"hbase.hstore.compaction.complete is set to false"</span>);
+<a class="jxr_linenumber" name="1224" href="#1224">1224</a>         sfs = <strong class="jxr_keyword">new</strong> ArrayList&lt;StoreFile&gt;(newFiles.size());
+<a class="jxr_linenumber" name="1225" href="#1225">1225</a>         <strong class="jxr_keyword">final</strong> <strong class="jxr_keyword">boolean</strong> evictOnClose =
+<a class="jxr_linenumber" name="1226" href="#1226">1226</a>             cacheConf != <strong class="jxr_keyword">null</strong>? cacheConf.shouldEvictOnClose(): <strong class="jxr_keyword">true</strong>;
+<a class="jxr_linenumber" name="1227" href="#1227">1227</a>         <strong class="jxr_keyword">for</strong> (Path newFile : newFiles) {
+<a class="jxr_linenumber" name="1228" href="#1228">1228</a>           <em class="jxr_comment">// Create storefile around what we wrote with a reader on it.</em>
+<a class="jxr_linenumber" name="1229" href="#1229">1229</a>           <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = createStoreFileAndReader(newFile);
+<a class="jxr_linenumber" name="1230" href="#1230">1230</a>           sf.closeReader(evictOnClose);
+<a class="jxr_linenumber" name="1231" href="#1231">1231</a>           sfs.add(sf);
+<a class="jxr_linenumber" name="1232" href="#1232">1232</a>         }
+<a class="jxr_linenumber" name="1233" href="#1233">1233</a>         <strong class="jxr_keyword">return</strong> sfs;
+<a class="jxr_linenumber" name="1234" href="#1234">1234</a>       }
+<a class="jxr_linenumber" name="1235" href="#1235">1235</a>       <em class="jxr_comment">// Do the steps necessary to complete the compaction.</em>
+<a class="jxr_linenumber" name="1236" href="#1236">1236</a>       sfs = moveCompatedFilesIntoPlace(cr, newFiles, user);
+<a class="jxr_linenumber" name="1237" href="#1237">1237</a>       writeCompactionWalRecord(filesToCompact, sfs);
+<a class="jxr_linenumber" name="1238" href="#1238">1238</a>       replaceStoreFiles(filesToCompact, sfs);
+<a class="jxr_linenumber" name="1239" href="#1239">1239</a>       <strong class="jxr_keyword">if</strong> (cr.isMajor()) {
+<a class="jxr_linenumber" name="1240" href="#1240">1240</a>         majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;
+<a class="jxr_linenumber" name="1241" href="#1241">1241</a>         majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;
+<a class="jxr_linenumber" name="1242" href="#1242">1242</a>       } <strong class="jxr_keyword">else</strong> {
+<a class="jxr_linenumber" name="1243" href="#1243">1243</a>         compactedCellsCount += getCompactionProgress().totalCompactingKVs;
+<a class="jxr_linenumber" name="1244" href="#1244">1244</a>         compactedCellsSize += getCompactionProgress().totalCompactedSize;
+<a class="jxr_linenumber" name="1245" href="#1245">1245</a>       }
+<a class="jxr_linenumber" name="1246" href="#1246">1246</a>       <em class="jxr_comment">// At this point the store will use new files for all new scanners.</em>
+<a class="jxr_linenumber" name="1247" href="#1247">1247</a>       completeCompaction(filesToCompact); <em class="jxr_comment">// update store size.</em>
+<a class="jxr_linenumber" name="1248" href="#1248">1248</a> 
+<a class="jxr_linenumber" name="1249" href="#1249">1249</a>       logCompactionEndMessage(cr, sfs, compactionStartTime);
+<a class="jxr_linenumber" name="1250" href="#1250">1250</a>       <strong class="jxr_keyword">return</strong> sfs;
+<a class="jxr_linenumber" name="1251" href="#1251">1251</a>     } <strong class="jxr_keyword">finally</strong> {
+<a class="jxr_linenumber" name="1252" href="#1252">1252</a>       finishCompactionRequest(cr);
+<a class="jxr_linenumber" name="1253" href="#1253">1253</a>     }
+<a class="jxr_linenumber" name="1254" href="#1254">1254</a>   }
+<a class="jxr_linenumber" name="1255" href="#1255">1255</a> 
+<a class="jxr_linenumber" name="1256" href="#1256">1256</a>   <strong class="jxr_keyword">private</strong> List&lt;StoreFile&gt; moveCompatedFilesIntoPlace(
+<a class="jxr_linenumber" name="1257" href="#1257">1257</a>       <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr, List&lt;Path&gt; newFiles, <a href="../../../../../org/apache/hadoop/hbase/security/User.html">User</a> user) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1258" href="#1258">1258</a>     List&lt;StoreFile&gt; sfs = <strong class="jxr_keyword">new</strong> ArrayList&lt;StoreFile&gt;(newFiles.size());
+<a class="jxr_linenumber" name="1259" href="#1259">1259</a>     <strong class="jxr_keyword">for</strong> (Path newFile : newFiles) {
+<a class="jxr_linenumber" name="1260" href="#1260">1260</a>       assert newFile != <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1261" href="#1261">1261</a>       <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = moveFileIntoPlace(newFile);
+<a class="jxr_linenumber" name="1262" href="#1262">1262</a>       <strong class="jxr_keyword">if</strong> (<strong class="jxr_keyword">this</strong>.getCoprocessorHost() != <strong class="jxr_keyword">null</strong>) {
+<a class="jxr_linenumber" name="1263" href="#1263">1263</a>         <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/Store.html">Store</a> thisStore = <strong class="jxr_keyword">this</strong>;
+<a class="jxr_linenumber" name="1264" href="#1264">1264</a>         <strong class="jxr_keyword">if</strong> (user == <strong class="jxr_keyword">null</strong>) {
+<a class="jxr_linenumber" name="1265" href="#1265">1265</a>           getCoprocessorHost().postCompact(thisStore, sf, cr);
+<a class="jxr_linenumber" name="1266" href="#1266">1266</a>         } <strong class="jxr_keyword">else</strong> {
+<a class="jxr_linenumber" name="1267" href="#1267">1267</a>           <strong class="jxr_keyword">try</strong> {
+<a class="jxr_linenumber" name="1268" href="#1268">1268</a>             user.getUGI().doAs(<strong class="jxr_keyword">new</strong> PrivilegedExceptionAction&lt;Void&gt;() {
+<a class="jxr_linenumber" name="1269" href="#1269">1269</a>               @Override
+<a class="jxr_linenumber" name="1270" href="#1270">1270</a>               <strong class="jxr_keyword">public</strong> Void run() <strong class="jxr_keyword">throws</strong> Exception {
+<a class="jxr_linenumber" name="1271" href="#1271">1271</a>                 getCoprocessorHost().postCompact(thisStore, sf, cr);
+<a class="jxr_linenumber" name="1272" href="#1272">1272</a>                 <strong class="jxr_keyword">return</strong> <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1273" href="#1273">1273</a>               }
+<a class="jxr_linenumber" name="1274" href="#1274">1274</a>             });
+<a class="jxr_linenumber" name="1275" href="#1275">1275</a>           } <strong class="jxr_keyword">catch</strong> (InterruptedException ie) {
+<a class="jxr_linenumber" name="1276" href="#1276">1276</a>             InterruptedIOException iioe = <strong class="jxr_keyword">new</strong> InterruptedIOException();
+<a class="jxr_linenumber" name="1277" href="#1277">1277</a>             iioe.initCause(ie);
+<a class="jxr_linenumber" name="1278" href="#1278">1278</a>             <strong class="jxr_keyword">throw</strong> iioe;
+<a class="jxr_linenumber" name="1279" href="#1279">1279</a>           }
+<a class="jxr_linenumber" name="1280" href="#1280">1280</a>         }
+<a class="jxr_linenumber" name="1281" href="#1281">1281</a>       }
+<a class="jxr_linenumber" name="1282" href="#1282">1282</a>       assert sf != <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1283" href="#1283">1283</a>       sfs.add(sf);
+<a class="jxr_linenumber" name="1284" href="#1284">1284</a>     }
+<a class="jxr_linenumber" name="1285" href="#1285">1285</a>     <strong class="jxr_keyword">return</strong> sfs;
+<a class="jxr_linenumber" name="1286" href="#1286">1286</a>   }
+<a class="jxr_linenumber" name="1287" href="#1287">1287</a> 
+<a class="jxr_linenumber" name="1288" href="#1288">1288</a>   <em class="jxr_comment">// Package-visible for tests</em>
+<a class="jxr_linenumber" name="1289" href="#1289">1289</a>   <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> moveFileIntoPlace(<strong class="jxr_keyword">final</strong> Path newFile) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1290" href="#1290">1290</a>     validateStoreFile(newFile);
+<a class="jxr_linenumber" name="1291" href="#1291">1291</a>     <em class="jxr_comment">// Move the file into the right spot</em>
+<a class="jxr_linenumber" name="1292" href="#1292">1292</a>     Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);
+<a class="jxr_linenumber" name="1293" href="#1293">1293</a>     <strong class="jxr_keyword">return</strong> createStoreFileAndReader(destPath);
+<a class="jxr_linenumber" name="1294" href="#1294">1294</a>   }
+<a class="jxr_linenumber" name="1295" href="#1295">1295</a> 
+<a class="jxr_linenumber" name="1296" href="#1296">1296</a>   <em class="jxr_javadoccomment">/**</em>
+<a class="jxr_linenumber" name="1297" href="#1297">1297</a> <em class="jxr_javadoccomment">   * Writes the compaction WAL record.</em>
+<a class="jxr_linenumber" name="1298" href="#1298">1298</a> <em class="jxr_javadoccomment">   * @param filesCompacted Files compacted (input).</em>
+<a class="jxr_linenumber" name="1299" href="#1299">1299</a> <em class="jxr_javadoccomment">   * @param newFiles Files from compaction.</em>
+<a class="jxr_linenumber" name="1300" href="#1300">1300</a> <em class="jxr_javadoccomment">   */</em>
+<a class="jxr_linenumber" name="1301" href="#1301">1301</a>   <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> writeCompactionWalRecord(Collection&lt;StoreFile&gt; filesCompacted,
+<a class="jxr_linenumber" name="1302" href="#1302">1302</a>       Collection&lt;StoreFile&gt; newFiles) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1303" href="#1303">1303</a>     <strong class="jxr_keyword">if</strong> (region.getWAL() == <strong class="jxr_keyword">null</strong>) <strong class="jxr_keyword">return</strong>;
+<a class="jxr_linenumber" name="1304" href="#1304">1304</a>     List&lt;Path&gt; inputPaths = <strong class="jxr_keyword">new</strong> ArrayList&lt;Path&gt;(filesCompacted.size());
+<a class="jxr_linenumber" name="1305" href="#1305">1305</a>     <strong class="jxr_keyword">for</strong> (StoreFile f : filesCompacted) {
+<a class="jxr_linenumber" name="1306" href="#1306">1306</a>       inputPaths.add(f.getPath());
+<a class="jxr_linenumber" name="1307" href="#1307">1307</a>     }
+<a class="jxr_linenumber" name="1308" href="#1308">1308</a>     List&lt;Path&gt; outputPaths = <strong class="jxr_keyword">new</strong> ArrayList&lt;Path&gt;(newFiles.size());
+<a class="jxr_linenumber" name="1309" href="#1309">1309</a>     <strong class="jxr_keyword">for</strong> (StoreFile f : newFiles) {
+<a class="jxr_linenumber" name="1310" href="#1310">1310</a>       outputPaths.add(f.getPath());
+<a class="jxr_linenumber" name="1311" href="#1311">1311</a>     }
+<a class="jxr_linenumber" name="1312" href="#1312">1312</a>     <a href="../../../../../org/apache/hadoop/hbase/HRegionInfo.html">HRegionInfo</a> info = <strong class="jxr_keyword">this</strong>.region.getRegionInfo();
+<a class="jxr_linenumber" name="1313" href="#1313">1313</a>     CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
+<a class="jxr_linenumber" name="1314" href="#1314">1314</a>         family.getName(), inputPaths, outputPaths, fs.getStoreDir(getFamily().getNameAsString()));
+<a class="jxr_linenumber" name="1315" href="#1315">1315</a>     WALUtil.writeCompactionMarker(region.getWAL(), <strong class="jxr_keyword">this</strong>.region.getTableDesc(),
+<a class="jxr_linenumber" name="1316" href="#1316">1316</a>         <strong class="jxr_keyword">this</strong>.region.getRegionInfo(), compactionDescriptor, region.getMVCC());
+<a class="jxr_linenumber" name="1317" href="#1317">1317</a>   }
+<a class="jxr_linenumber" name="1318" href="#1318">1318</a> 
+<a class="jxr_linenumber" name="1319" href="#1319">1319</a>   @VisibleForTesting
+<a class="jxr_linenumber" name="1320" href="#1320">1320</a>   <strong class="jxr_keyword">void</strong> replaceStoreFiles(<strong class="jxr_keyword">final</strong> Collection&lt;StoreFile&gt; compactedFiles,
+<a class="jxr_linenumber" name="1321" href="#1321">1321</a>       <strong class="jxr_keyword">final</strong> Collection&lt;StoreFile&gt; result) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1322" href="#1322">1322</a>     <strong class="jxr_keyword">this</strong>.lock.writeLock().lock();
+<a class="jxr_linenumber" name="1323" href="#1323">1323</a>     <strong class="jxr_keyword">try</strong> {
+<a class="jxr_linenumber" name="1324" href="#1324">1324</a>       <strong class="jxr_keyword">this</strong>.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result);
+<a class="jxr_linenumber" name="1325" href="#1325">1325</a>       filesCompacting.removeAll(compactedFiles); <em class="jxr_comment">// safe bc: lock.writeLock();</em>
+<a class="jxr_linenumber" name="1326" href="#1326">1326</a>     } <strong class="jxr_keyword">finally</strong> {
+<a class="jxr_linenumber" name="1327" href="#1327">1327</a>       <strong class="jxr_keyword">this</strong>.lock.writeLock().unlock();
+<a class="jxr_linenumber" name="1328" href="#1328">1328</a>     }
+<a class="jxr_linenumber" name="1329" href="#1329">1329</a>   }
+<a class="jxr_linenumber" name="1330" href="#1330">1330</a> 
+<a class="jxr_linenumber" name="1331" href="#1331">1331</a>   <em class="jxr_javadoccomment">/**</em>
+<a class="jxr_linenumber" name="1332" href="#1332">1332</a> <em class="jxr_javadoccomment">   * Log a very elaborate compaction completion message.</em>
+<a class="jxr_linenumber" name="1333" href="#1333">1333</a> <em class="jxr_javadoccomment">   * @param cr Request.</em>
+<a class="jxr_linenumber" name="1334" href="#1334">1334</a> <em class="jxr_javadoccomment">   * @param sfs Resulting files.</em>
+<a class="jxr_linenumber" name="1335" href="#1335">1335</a> <em class="jxr_javadoccomment">   * @param compactionStartTime Start time.</em>
+<a class="jxr_linenumber" name="1336" href="#1336">1336</a> <em class="jxr_javadoccomment">   */</em>
+<a class="jxr_linenumber" name="1337" href="#1337">1337</a>   <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> logCompactionEndMessage(
+<a class="jxr_linenumber" name="1338" href="#1338">1338</a>       <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr, List&lt;StoreFile&gt; sfs, <strong class="jxr_keyword">long</strong> compactionStartTime) {
+<a class="jxr_linenumber" name="1339" href="#1339">1339</a>     <strong class="jxr_keyword">long</strong> now = EnvironmentEdgeManager.currentTime();
+<a class="jxr_linenumber" name="1340" href="#1340">1340</a>     StringBuilder message = <strong class="jxr_keyword">new</strong> StringBuilder(
+<a class="jxr_linenumber" name="1341" href="#1341">1341</a>       <span class="jxr_string">"Completed"</span> + (cr.isMajor() ? <span class="jxr_string">" major"</span> : <span class="jxr_string">""</span>) + <span class="jxr_string">" compaction of "</span>
+<a class="jxr_linenumber" name="1342" href="#1342">1342</a>       + cr.getFiles().size() + (cr.isAllFiles() ? <span class="jxr_string">" (all)"</span> : <span class="jxr_string">""</span>) + <span class="jxr_string">" file(s) in "</span>
+<a class="jxr_linenumber" name="1343" href="#1343">1343</a>       + <strong class="jxr_keyword">this</strong> + <span class="jxr_string">" of "</span> + <strong class="jxr_keyword">this</strong>.getRegionInfo().getRegionNameAsString() + <span class="jxr_string">" into "</span>);
+<a class="jxr_linenumber" name="1344" href="#1344">1344</a>     <strong class="jxr_keyword">if</strong> (sfs.isEmpty()) {
+<a class="jxr_linenumber" name="1345" href="#1345">1345</a>       message.append(<span class="jxr_string">"none, "</span>);
+<a class="jxr_linenumber" name="1346" href="#1346">1346</a>     } <strong class="jxr_keyword">else</strong> {
+<a class="jxr_linenumber" name="1347" href="#1347">1347</a>       <strong class="jxr_keyword">for</strong> (StoreFile sf: sfs) {
+<a class="jxr_linenumber" name="1348" href="#1348">1348</a>         message.append(sf.getPath().getName());
+<a class="jxr_linenumber" name="1349" href="#1349">1349</a>         message.append(<span class="jxr_string">"(size="</span>);
+<a class="jxr_linenumber" name="1350" href="#1350">1350</a>         message.append(TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(sf.getReader().length(), <span class="jxr_string">""</span>, 1));
+<a class="jxr_linenumber" name="1351" href="#1351">1351</a>         message.append(<span class="jxr_string">"), "</span>);
+<a class="jxr_linenumber" name="1352" href="#1352">1352</a>       }
+<a class="jxr_linenumber" name="1353" href="#1353">1353</a>     }
+<a class="jxr_linenumber" name="1354" href="#1354">1354</a>     message.append(<span class="jxr_string">"total size for store is "</span>)
+<a class="jxr_linenumber" name="1355" href="#1355">1355</a>       .append(StringUtils.TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(storeSize, <span class="jxr_string">""</span>, 1))
+<a class="jxr_linenumber" name="1356" href="#1356">1356</a>       .append(<span class="jxr_string">". This selection was in queue for "</span>)
+<a class="jxr_linenumber" name="1357" href="#1357">1357</a>       .append(StringUtils.formatTimeDiff(compactionStartTime, cr.getSelectionTime()))
+<a class="jxr_linenumber" name="1358" href="#1358">1358</a>       .append(<span class="jxr_string">", and took "</span>).append(StringUtils.formatTimeDiff(now, compactionStartTime))
+<a class="jxr_linenumber" name="1359" href="#1359">1359</a>       .append(<span class="jxr_string">" to execute."</span>);
+<a class="jxr_linenumber" name="1360" href="#1360">1360</a>     LOG.info(message.toString());
+<a class="jxr_linenumber" name="1361" href="#1361">1361</a>     <strong class="jxr_keyword">if</strong> (LOG.isTraceEnabled()) {
+<a class="jxr_linenumber" name="1362" href="#1362">1362</a>       <strong class="jxr_keyword">int</strong> fileCount = storeEngine.getStoreFileManager().getStorefileCount();
+<a class="jxr_linenumber" name="1363" href="#1363">1363</a>       <strong class="jxr_keyword">long</strong> resultSize = 0;
+<a class="jxr_linenumber" name="1364" href="#1364">1364</a>       <strong class="jxr_keyword">for</strong> (StoreFile sf : sfs) {
+<a class="jxr_linenumber" name="1365" href="#1365">1365</a>         resultSize += sf.getReader().length();
+<a class="jxr_linenumber" name="1366" href="#1366">1366</a>       }
+<a class="jxr_linenumber" name="1367" href="#1367">1367</a>   

<TRUNCATED>

Mime
View raw message