Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F246E200D1A for ; Mon, 9 Oct 2017 17:12:56 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id F0AE01609E0; Mon, 9 Oct 2017 15:12:56 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B8561160BEA for ; Mon, 9 Oct 2017 17:12:54 +0200 (CEST) Received: (qmail 56483 invoked by uid 500); 9 Oct 2017 15:12:47 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 55576 invoked by uid 99); 9 Oct 2017 15:12:46 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 09 Oct 2017 15:12:46 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 24BBEF5D46; Mon, 9 Oct 2017 15:12:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Mon, 09 Oct 2017 15:13:04 -0000 Message-Id: <5087668168ab4fac9c83da11570380f5@git.apache.org> In-Reply-To: <6334b53dd3a74dd2b4983f07fc142826@git.apache.org> References: <6334b53dd3a74dd2b4983f07fc142826@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [20/29] hbase-site git commit: Published site at . archived-at: Mon, 09 Oct 2017 15:12:57 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e6ae7c3e/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/StoreFileWriter.Builder.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/StoreFileWriter.Builder.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/StoreFileWriter.Builder.html index 02426b3..679955d 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/StoreFileWriter.Builder.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/StoreFileWriter.Builder.html @@ -79,462 +79,413 @@ 071 private long deleteFamilyCnt = 0; 072 private BloomContext bloomContext = null; 073 private BloomContext deleteFamilyBloomContext = null; -074 -075 /** -076 * timeRangeTrackerSet is used to figure if we were passed a filled-out TimeRangeTracker or not. -077 * When flushing a memstore, we set the TimeRangeTracker that it accumulated during updates to -078 * memstore in here into this Writer and use this variable to indicate that we do not need to -079 * recalculate the timeRangeTracker bounds; it was done already as part of add-to-memstore. -080 * A completed TimeRangeTracker is not set in cases of compactions when it is recalculated. -081 */ -082 private final boolean timeRangeTrackerSet; -083 final TimeRangeTracker timeRangeTracker; -084 -085 protected HFile.Writer writer; -086 -087 /** -088 * Creates an HFile.Writer that also write helpful meta data. -089 * @param fs file system to write to -090 * @param path file name to create -091 * @param conf user configuration -092 * @param comparator key comparator -093 * @param bloomType bloom filter setting -094 * @param maxKeys the expected maximum number of keys to be added. Was used -095 * for Bloom filter size in {@link HFile} format version 1. -096 * @param fileContext - The HFile context -097 * @param shouldDropCacheBehind Drop pages written to page cache after writing the store file. -098 * @throws IOException problem writing to FS -099 */ -100 StoreFileWriter(FileSystem fs, Path path, final Configuration conf, CacheConfig cacheConf, -101 final CellComparator comparator, BloomType bloomType, long maxKeys, -102 InetSocketAddress[] favoredNodes, HFileContext fileContext, boolean shouldDropCacheBehind) -103 throws IOException { -104 this(fs, path, conf, cacheConf, comparator, bloomType, maxKeys, favoredNodes, fileContext, -105 shouldDropCacheBehind, null); -106 } -107 -108 /** -109 * Creates an HFile.Writer that also write helpful meta data. -110 * @param fs file system to write to -111 * @param path file name to create -112 * @param conf user configuration -113 * @param comparator key comparator -114 * @param bloomType bloom filter setting -115 * @param maxKeys the expected maximum number of keys to be added. Was used -116 * for Bloom filter size in {@link HFile} format version 1. -117 * @param favoredNodes -118 * @param fileContext - The HFile context -119 * @param shouldDropCacheBehind Drop pages written to page cache after writing the store file. -120 * @param trt Ready-made timetracker to use. -121 * @throws IOException problem writing to FS -122 */ -123 private StoreFileWriter(FileSystem fs, Path path, -124 final Configuration conf, -125 CacheConfig cacheConf, -126 final CellComparator comparator, BloomType bloomType, long maxKeys, -127 InetSocketAddress[] favoredNodes, HFileContext fileContext, -128 boolean shouldDropCacheBehind, final TimeRangeTracker trt) -129 throws IOException { -130 // If passed a TimeRangeTracker, use it. Set timeRangeTrackerSet so we don't destroy it. -131 // TODO: put the state of the TRT on the TRT; i.e. make a read-only version (TimeRange) when -132 // it no longer writable. -133 this.timeRangeTrackerSet = trt != null; -134 this.timeRangeTracker = this.timeRangeTrackerSet? trt: TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); -135 // TODO : Change all writers to be specifically created for compaction context -136 writer = HFile.getWriterFactory(conf, cacheConf) -137 .withPath(fs, path) -138 .withComparator(comparator) -139 .withFavoredNodes(favoredNodes) -140 .withFileContext(fileContext) -141 .withShouldDropCacheBehind(shouldDropCacheBehind) -142 .create(); -143 -144 generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite( -145 conf, cacheConf, bloomType, -146 (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); -147 -148 if (generalBloomFilterWriter != null) { -149 this.bloomType = bloomType; -150 if (LOG.isTraceEnabled()) { -151 LOG.trace("Bloom filter type for " + path + ": " + this.bloomType + ", " + -152 generalBloomFilterWriter.getClass().getSimpleName()); -153 } -154 // init bloom context -155 switch (bloomType) { -156 case ROW: -157 bloomContext = new RowBloomContext(generalBloomFilterWriter, comparator); -158 break; -159 case ROWCOL: -160 bloomContext = new RowColBloomContext(generalBloomFilterWriter, comparator); -161 break; -162 default: -163 throw new IOException( -164 "Invalid Bloom filter type: " + bloomType + " (ROW or ROWCOL expected)"); -165 } -166 } else { -167 // Not using Bloom filters. -168 this.bloomType = BloomType.NONE; -169 } -170 -171 // initialize delete family Bloom filter when there is NO RowCol Bloom -172 // filter -173 if (this.bloomType != BloomType.ROWCOL) { -174 this.deleteFamilyBloomFilterWriter = BloomFilterFactory -175 .createDeleteBloomAtWrite(conf, cacheConf, -176 (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); -177 deleteFamilyBloomContext = new RowBloomContext(deleteFamilyBloomFilterWriter, comparator); -178 } else { -179 deleteFamilyBloomFilterWriter = null; -180 } -181 if (deleteFamilyBloomFilterWriter != null && LOG.isTraceEnabled()) { -182 LOG.trace("Delete Family Bloom filter type for " + path + ": " + -183 deleteFamilyBloomFilterWriter.getClass().getSimpleName()); -184 } -185 } -186 -187 /** -188 * Writes meta data. -189 * Call before {@link #close()} since its written as meta data to this file. -190 * @param maxSequenceId Maximum sequence id. -191 * @param majorCompaction True if this file is product of a major compaction -192 * @throws IOException problem writing to FS -193 */ -194 public void appendMetadata(final long maxSequenceId, final boolean majorCompaction) -195 throws IOException { -196 writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId)); -197 writer.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(majorCompaction)); -198 appendTrackedTimestampsToMetadata(); -199 } -200 -201 /** -202 * Writes meta data. -203 * Call before {@link #close()} since its written as meta data to this file. -204 * @param maxSequenceId Maximum sequence id. -205 * @param majorCompaction True if this file is product of a major compaction -206 * @param mobCellsCount The number of mob cells. -207 * @throws IOException problem writing to FS -208 */ -209 public void appendMetadata(final long maxSequenceId, final boolean majorCompaction, -210 final long mobCellsCount) throws IOException { -211 writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId)); -212 writer.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(majorCompaction)); -213 writer.appendFileInfo(MOB_CELLS_COUNT, Bytes.toBytes(mobCellsCount)); -214 appendTrackedTimestampsToMetadata(); +074 private final TimeRangeTracker timeRangeTracker; +075 +076 protected HFile.Writer writer; +077 +078 /** +079 * Creates an HFile.Writer that also write helpful meta data. +080 * @param fs file system to write to +081 * @param path file name to create +082 * @param conf user configuration +083 * @param comparator key comparator +084 * @param bloomType bloom filter setting +085 * @param maxKeys the expected maximum number of keys to be added. Was used +086 * for Bloom filter size in {@link HFile} format version 1. +087 * @param favoredNodes +088 * @param fileContext - The HFile context +089 * @param shouldDropCacheBehind Drop pages written to page cache after writing the store file. +090 * @throws IOException problem writing to FS +091 */ +092 private StoreFileWriter(FileSystem fs, Path path, +093 final Configuration conf, +094 CacheConfig cacheConf, +095 final CellComparator comparator, BloomType bloomType, long maxKeys, +096 InetSocketAddress[] favoredNodes, HFileContext fileContext, +097 boolean shouldDropCacheBehind) +098 throws IOException { +099 this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); +100 // TODO : Change all writers to be specifically created for compaction context +101 writer = HFile.getWriterFactory(conf, cacheConf) +102 .withPath(fs, path) +103 .withComparator(comparator) +104 .withFavoredNodes(favoredNodes) +105 .withFileContext(fileContext) +106 .withShouldDropCacheBehind(shouldDropCacheBehind) +107 .create(); +108 +109 generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite( +110 conf, cacheConf, bloomType, +111 (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); +112 +113 if (generalBloomFilterWriter != null) { +114 this.bloomType = bloomType; +115 if (LOG.isTraceEnabled()) { +116 LOG.trace("Bloom filter type for " + path + ": " + this.bloomType + ", " + +117 generalBloomFilterWriter.getClass().getSimpleName()); +118 } +119 // init bloom context +120 switch (bloomType) { +121 case ROW: +122 bloomContext = new RowBloomContext(generalBloomFilterWriter, comparator); +123 break; +124 case ROWCOL: +125 bloomContext = new RowColBloomContext(generalBloomFilterWriter, comparator); +126 break; +127 default: +128 throw new IOException( +129 "Invalid Bloom filter type: " + bloomType + " (ROW or ROWCOL expected)"); +130 } +131 } else { +132 // Not using Bloom filters. +133 this.bloomType = BloomType.NONE; +134 } +135 +136 // initialize delete family Bloom filter when there is NO RowCol Bloom +137 // filter +138 if (this.bloomType != BloomType.ROWCOL) { +139 this.deleteFamilyBloomFilterWriter = BloomFilterFactory +140 .createDeleteBloomAtWrite(conf, cacheConf, +141 (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); +142 deleteFamilyBloomContext = new RowBloomContext(deleteFamilyBloomFilterWriter, comparator); +143 } else { +144 deleteFamilyBloomFilterWriter = null; +145 } +146 if (deleteFamilyBloomFilterWriter != null && LOG.isTraceEnabled()) { +147 LOG.trace("Delete Family Bloom filter type for " + path + ": " + +148 deleteFamilyBloomFilterWriter.getClass().getSimpleName()); +149 } +150 } +151 +152 /** +153 * Writes meta data. +154 * Call before {@link #close()} since its written as meta data to this file. +155 * @param maxSequenceId Maximum sequence id. +156 * @param majorCompaction True if this file is product of a major compaction +157 * @throws IOException problem writing to FS +158 */ +159 public void appendMetadata(final long maxSequenceId, final boolean majorCompaction) +160 throws IOException { +161 writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId)); +162 writer.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(majorCompaction)); +163 appendTrackedTimestampsToMetadata(); +164 } +165 +166 /** +167 * Writes meta data. +168 * Call before {@link #close()} since its written as meta data to this file. +169 * @param maxSequenceId Maximum sequence id. +170 * @param majorCompaction True if this file is product of a major compaction +171 * @param mobCellsCount The number of mob cells. +172 * @throws IOException problem writing to FS +173 */ +174 public void appendMetadata(final long maxSequenceId, final boolean majorCompaction, +175 final long mobCellsCount) throws IOException { +176 writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId)); +177 writer.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(majorCompaction)); +178 writer.appendFileInfo(MOB_CELLS_COUNT, Bytes.toBytes(mobCellsCount)); +179 appendTrackedTimestampsToMetadata(); +180 } +181 +182 /** +183 * Add TimestampRange and earliest put timestamp to Metadata +184 */ +185 public void appendTrackedTimestampsToMetadata() throws IOException { +186 appendFileInfo(TIMERANGE_KEY, WritableUtils.toByteArray(timeRangeTracker)); +187 appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs)); +188 } +189 +190 /** +191 * Record the earlest Put timestamp. +192 * +193 * If the timeRangeTracker is not set, +194 * update TimeRangeTracker to include the timestamp of this key +195 */ +196 public void trackTimestamps(final Cell cell) { +197 if (KeyValue.Type.Put.getCode() == cell.getTypeByte()) { +198 earliestPutTs = Math.min(earliestPutTs, cell.getTimestamp()); +199 } +200 timeRangeTracker.includeTimestamp(cell); +201 } +202 +203 private void appendGeneralBloomfilter(final Cell cell) throws IOException { +204 if (this.generalBloomFilterWriter != null) { +205 /* +206 * http://2.bp.blogspot.com/_Cib_A77V54U/StZMrzaKufI/AAAAAAAAADo/ZhK7bGoJdMQ/s400/KeyValue.png +207 * Key = RowLen + Row + FamilyLen + Column [Family + Qualifier] + TimeStamp +208 * +209 * 2 Types of Filtering: +210 * 1. Row = Row +211 * 2. RowCol = Row + Qualifier +212 */ +213 bloomContext.writeBloom(cell); +214 } 215 } 216 -217 /** -218 * Add TimestampRange and earliest put timestamp to Metadata -219 */ -220 public void appendTrackedTimestampsToMetadata() throws IOException { -221 appendFileInfo(TIMERANGE_KEY, WritableUtils.toByteArray(timeRangeTracker)); -222 appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs)); -223 } -224 -225 /** -226 * Record the earlest Put timestamp. -227 * -228 * If the timeRangeTracker is not set, -229 * update TimeRangeTracker to include the timestamp of this key -230 */ -231 public void trackTimestamps(final Cell cell) { -232 if (KeyValue.Type.Put.getCode() == cell.getTypeByte()) { -233 earliestPutTs = Math.min(earliestPutTs, cell.getTimestamp()); -234 } -235 if (!timeRangeTrackerSet) { -236 timeRangeTracker.includeTimestamp(cell); -237 } -238 } -239 -240 private void appendGeneralBloomfilter(final Cell cell) throws IOException { -241 if (this.generalBloomFilterWriter != null) { -242 /* -243 * http://2.bp.blogspot.com/_Cib_A77V54U/StZMrzaKufI/AAAAAAAAADo/ZhK7bGoJdMQ/s400/KeyValue.png -244 * Key = RowLen + Row + FamilyLen + Column [Family + Qualifier] + TimeStamp -245 * -246 * 2 Types of Filtering: -247 * 1. Row = Row -248 * 2. RowCol = Row + Qualifier -249 */ -250 bloomContext.writeBloom(cell); -251 } -252 } -253 -254 private void appendDeleteFamilyBloomFilter(final Cell cell) -255 throws IOException { -256 if (!CellUtil.isDeleteFamily(cell) && !CellUtil.isDeleteFamilyVersion(cell)) { -257 return; -258 } -259 -260 // increase the number of delete family in the store file -261 deleteFamilyCnt++; -262 if (this.deleteFamilyBloomFilterWriter != null) { -263 deleteFamilyBloomContext.writeBloom(cell); -264 } -265 } -266 -267 @Override -268 public void append(final Cell cell) throws IOException { -269 appendGeneralBloomfilter(cell); -270 appendDeleteFamilyBloomFilter(cell); -271 writer.append(cell); -272 trackTimestamps(cell); -273 } -274 -275 @Override -276 public void beforeShipped() throws IOException { -277 // For now these writer will always be of type ShipperListener true. -278 // TODO : Change all writers to be specifically created for compaction context -279 writer.beforeShipped(); -280 if (generalBloomFilterWriter != null) { -281 generalBloomFilterWriter.beforeShipped(); -282 } -283 if (deleteFamilyBloomFilterWriter != null) { -284 deleteFamilyBloomFilterWriter.beforeShipped(); -285 } +217 private void appendDeleteFamilyBloomFilter(final Cell cell) +218 throws IOException { +219 if (!CellUtil.isDeleteFamily(cell) && !CellUtil.isDeleteFamilyVersion(cell)) { +220 return; +221 } +222 +223 // increase the number of delete family in the store file +224 deleteFamilyCnt++; +225 if (this.deleteFamilyBloomFilterWriter != null) { +226 deleteFamilyBloomContext.writeBloom(cell); +227 } +228 } +229 +230 @Override +231 public void append(final Cell cell) throws IOException { +232 appendGeneralBloomfilter(cell); +233 appendDeleteFamilyBloomFilter(cell); +234 writer.append(cell); +235 trackTimestamps(cell); +236 } +237 +238 @Override +239 public void beforeShipped() throws IOException { +240 // For now these writer will always be of type ShipperListener true. +241 // TODO : Change all writers to be specifically created for compaction context +242 writer.beforeShipped(); +243 if (generalBloomFilterWriter != null) { +244 generalBloomFilterWriter.beforeShipped(); +245 } +246 if (deleteFamilyBloomFilterWriter != null) { +247 deleteFamilyBloomFilterWriter.beforeShipped(); +248 } +249 } +250 +251 public Path getPath() { +252 return this.writer.getPath(); +253 } +254 +255 public boolean hasGeneralBloom() { +256 return this.generalBloomFilterWriter != null; +257 } +258 +259 /** +260 * For unit testing only. +261 * +262 * @return the Bloom filter used by this writer. +263 */ +264 BloomFilterWriter getGeneralBloomWriter() { +265 return generalBloomFilterWriter; +266 } +267 +268 private boolean closeBloomFilter(BloomFilterWriter bfw) throws IOException { +269 boolean haveBloom = (bfw != null && bfw.getKeyCount() > 0); +270 if (haveBloom) { +271 bfw.compactBloom(); +272 } +273 return haveBloom; +274 } +275 +276 private boolean closeGeneralBloomFilter() throws IOException { +277 boolean hasGeneralBloom = closeBloomFilter(generalBloomFilterWriter); +278 +279 // add the general Bloom filter writer and append file info +280 if (hasGeneralBloom) { +281 writer.addGeneralBloomFilter(generalBloomFilterWriter); +282 writer.appendFileInfo(BLOOM_FILTER_TYPE_KEY, Bytes.toBytes(bloomType.toString())); +283 bloomContext.addLastBloomKey(writer); +284 } +285 return hasGeneralBloom; 286 } 287 -288 public Path getPath() { -289 return this.writer.getPath(); -290 } -291 -292 public boolean hasGeneralBloom() { -293 return this.generalBloomFilterWriter != null; -294 } +288 private boolean closeDeleteFamilyBloomFilter() throws IOException { +289 boolean hasDeleteFamilyBloom = closeBloomFilter(deleteFamilyBloomFilterWriter); +290 +291 // add the delete family Bloom filter writer +292 if (hasDeleteFamilyBloom) { +293 writer.addDeleteFamilyBloomFilter(deleteFamilyBloomFilterWriter); +294 } 295 -296 /** -297 * For unit testing only. -298 * -299 * @return the Bloom filter used by this writer. -300 */ -301 BloomFilterWriter getGeneralBloomWriter() { -302 return generalBloomFilterWriter; -303 } -304 -305 private boolean closeBloomFilter(BloomFilterWriter bfw) throws IOException { -306 boolean haveBloom = (bfw != null && bfw.getKeyCount() > 0); -307 if (haveBloom) { -308 bfw.compactBloom(); -309 } -310 return haveBloom; -311 } -312 -313 private boolean closeGeneralBloomFilter() throws IOException { -314 boolean hasGeneralBloom = closeBloomFilter(generalBloomFilterWriter); -315 -316 // add the general Bloom filter writer and append file info -317 if (hasGeneralBloom) { -318 writer.addGeneralBloomFilter(generalBloomFilterWriter); -319 writer.appendFileInfo(BLOOM_FILTER_TYPE_KEY, Bytes.toBytes(bloomType.toString())); -320 bloomContext.addLastBloomKey(writer); -321 } -322 return hasGeneralBloom; -323 } -324 -325 private boolean closeDeleteFamilyBloomFilter() throws IOException { -326 boolean hasDeleteFamilyBloom = closeBloomFilter(deleteFamilyBloomFilterWriter); -327 -328 // add the delete family Bloom filter writer -329 if (hasDeleteFamilyBloom) { -330 writer.addDeleteFamilyBloomFilter(deleteFamilyBloomFilterWriter); -331 } -332 -333 // append file info about the number of delete family kvs -334 // even if there is no delete family Bloom. -335 writer.appendFileInfo(DELETE_FAMILY_COUNT, Bytes.toBytes(this.deleteFamilyCnt)); -336 -337 return hasDeleteFamilyBloom; -338 } -339 -340 public void close() throws IOException { -341 boolean hasGeneralBloom = this.closeGeneralBloomFilter(); -342 boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter(); -343 -344 writer.close(); -345 -346 // Log final Bloom filter statistics. This needs to be done after close() -347 // because compound Bloom filters might be finalized as part of closing. -348 if (LOG.isTraceEnabled()) { -349 LOG.trace((hasGeneralBloom ? "" : "NO ") + "General Bloom and " + -350 (hasDeleteFamilyBloom ? "" : "NO ") + "DeleteFamily" + " was added to HFile " + -351 getPath()); -352 } -353 -354 } -355 -356 public void appendFileInfo(byte[] key, byte[] value) throws IOException { -357 writer.appendFileInfo(key, value); -358 } -359 -360 /** For use in testing. -361 */ -362 HFile.Writer getHFileWriter() { -363 return writer; -364 } -365 -366 /** -367 * @param fs -368 * @param dir Directory to create file in. -369 * @return random filename inside passed <code>dir</code> -370 */ -371 static Path getUniqueFile(final FileSystem fs, final Path dir) throws IOException { -372 if (!fs.getFileStatus(dir).isDirectory()) { -373 throw new IOException("Expecting " + dir.toString() + " to be a directory"); -374 } -375 return new Path(dir, dash.matcher(UUID.randomUUID().toString()).replaceAll("")); -376 } -377 -378 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="ICAST_INTEGER_MULTIPLY_CAST_TO_LONG", -379 justification="Will not overflow") -380 public static class Builder { -381 private final Configuration conf; -382 private final CacheConfig cacheConf; -383 private final FileSystem fs; -384 -385 private CellComparator comparator = CellComparator.COMPARATOR; -386 private BloomType bloomType = BloomType.NONE; -387 private long maxKeyCount = 0; -388 private Path dir; -389 private Path filePath; -390 private InetSocketAddress[] favoredNodes; -391 private HFileContext fileContext; -392 private TimeRangeTracker trt; -393 private boolean shouldDropCacheBehind; -394 -395 public Builder(Configuration conf, CacheConfig cacheConf, -396 FileSystem fs) { -397 this.conf = conf; -398 this.cacheConf = cacheConf; -399 this.fs = fs; -400 } -401 -402 /** -403 * Creates Builder with cache configuration disabled -404 */ -405 public Builder(Configuration conf, FileSystem fs) { -406 this.conf = conf; -407 this.cacheConf = CacheConfig.DISABLED; -408 this.fs = fs; -409 } -410 -411 /** -412 * @param trt A premade TimeRangeTracker to use rather than build one per append (building one -413 * of these is expensive so good to pass one in if you have one). -414 * @return this (for chained invocation) -415 */ -416 public Builder withTimeRangeTracker(final TimeRangeTracker trt) { -417 Preconditions.checkNotNull(trt); -418 this.trt = trt; -419 return this; -420 } -421 -422 /** -423 * Use either this method or {@link #withFilePath}, but not both. -424 * @param dir Path to column family directory. The directory is created if -425 * does not exist. The file is given a unique name within this -426 * directory. -427 * @return this (for chained invocation) -428 */ -429 public Builder withOutputDir(Path dir) { -430 Preconditions.checkNotNull(dir); -431 this.dir = dir; -432 return this; -433 } -434 -435 /** -436 * Use either this method or {@link #withOutputDir}, but not both. -437 * @param filePath the StoreFile path to write -438 * @return this (for chained invocation) -439 */ -440 public Builder withFilePath(Path filePath) { -441 Preconditions.checkNotNull(filePath); -442 this.filePath = filePath; -443 return this; -444 } -445 -446 /** -447 * @param favoredNodes an array of favored nodes or possibly null -448 * @return this (for chained invocation) -449 */ -450 public Builder withFavoredNodes(InetSocketAddress[] favoredNodes) { -451 this.favoredNodes = favoredNodes; -452 return this; -453 } -454 -455 public Builder withComparator(CellComparator comparator) { -456 Preconditions.checkNotNull(comparator); -457 this.comparator = comparator; -458 return this; -459 } -460 -461 public Builder withBloomType(BloomType bloomType) { -462 Preconditions.checkNotNull(bloomType); -463 this.bloomType = bloomType; -464 return this; -465 } -466 -467 /** -468 * @param maxKeyCount estimated maximum number of keys we expect to add -469 * @return this (for chained invocation) -470 */ -471 public Builder withMaxKeyCount(long maxKeyCount) { -472 this.maxKeyCount = maxKeyCount; -473 return this; -474 } -475 -476 public Builder withFileContext(HFileContext fileContext) { -477 this.fileContext = fileContext; -478 return this; -479 } -480 -481 public Builder withShouldDropCacheBehind(boolean shouldDropCacheBehind) { -482 this.shouldDropCacheBehind = shouldDropCacheBehind; -483 return this; -484 } -485 -486 /** -487 * Create a store file writer. Client is responsible for closing file when -488 * done. If metadata, add BEFORE closing using -489 * {@link StoreFileWriter#appendMetadata}. -490 */ -491 public StoreFileWriter build() throws IOException { -492 if ((dir == null ? 0 : 1) + (filePath == null ? 0 : 1) != 1) { -493 throw new IllegalArgumentException("Either specify parent directory " + -494 "or file path"); -495 } -496 -497 if (dir == null) { -498 dir = filePath.getParent(); -499 } -500 -501 if (!fs.exists(dir)) { -502 // Handle permission for non-HDFS filesystem properly -503 // See HBASE-17710 -504 HRegionFileSystem.mkdirs(fs, conf, dir); -505 } -506 -507 // set block storage policy for temp path -508 String policyName = this.conf.get(ColumnFamilyDescriptorBuilder.STORAGE_POLICY); -509 if (null == policyName) { -510 policyName = this.conf.get(HStore.BLOCK_STORAGE_POLICY_KEY); -511 } -512 FSUtils.setStoragePolicy(this.fs, dir, policyName); -513 -514 if (filePath == null) { -515 filePath = getUniqueFile(fs, dir); -516 if (!BloomFilterFactory.isGeneralBloomEnabled(conf)) { -517 bloomType = BloomType.NONE; -518 } -519 } -520 -521 if (comparator == null) { -522 comparator = CellComparator.COMPARATOR; -523 } -524 return new StoreFileWriter(fs, filePath, -525 conf, cacheConf, comparator, bloomType, maxKeyCount, favoredNodes, fileContext, -526 shouldDropCacheBehind, trt); -527 } -528 } -529} +296 // append file info about the number of delete family kvs +297 // even if there is no delete family Bloom. +298 writer.appendFileInfo(DELETE_FAMILY_COUNT, Bytes.toBytes(this.deleteFamilyCnt)); +299 +300 return hasDeleteFamilyBloom; +301 } +302 +303 public void close() throws IOException { +304 boolean hasGeneralBloom = this.closeGeneralBloomFilter(); +305 boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter(); +306 +307 writer.close(); +308 +309 // Log final Bloom filter statistics. This needs to be done after close() +310 // because compound Bloom filters might be finalized as part of closing. +311 if (LOG.isTraceEnabled()) { +312 LOG.trace((hasGeneralBloom ? "" : "NO ") + "General Bloom and " + +313 (hasDeleteFamilyBloom ? "" : "NO ") + "DeleteFamily" + " was added to HFile " + +314 getPath()); +315 } +316 +317 } +318 +319 public void appendFileInfo(byte[] key, byte[] value) throws IOException { +320 writer.appendFileInfo(key, value); +321 } +322 +323 /** For use in testing. +324 */ +325 HFile.Writer getHFileWriter() { +326 return writer; +327 } +328 +329 /** +330 * @param fs +331 * @param dir Directory to create file in. +332 * @return random filename inside passed <code>dir</code> +333 */ +334 static Path getUniqueFile(final FileSystem fs, final Path dir) throws IOException { +335 if (!fs.getFileStatus(dir).isDirectory()) { +336 throw new IOException("Expecting " + dir.toString() + " to be a directory"); +337 } +338 return new Path(dir, dash.matcher(UUID.randomUUID().toString()).replaceAll("")); +339 } +340 +341 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="ICAST_INTEGER_MULTIPLY_CAST_TO_LONG", +342 justification="Will not overflow") +343 public static class Builder { +344 private final Configuration conf; +345 private final CacheConfig cacheConf; +346 private final FileSystem fs; +347 +348 private CellComparator comparator = CellComparator.COMPARATOR; +349 private BloomType bloomType = BloomType.NONE; +350 private long maxKeyCount = 0; +351 private Path dir; +352 private Path filePath; +353 private InetSocketAddress[] favoredNodes; +354 private HFileContext fileContext; +355 private boolean shouldDropCacheBehind; +356 +357 public Builder(Configuration conf, CacheConfig cacheConf, +358 FileSystem fs) { +359 this.conf = conf; +360 this.cacheConf = cacheConf; +361 this.fs = fs; +362 } +363 +364 /** +365 * Creates Builder with cache configuration disabled +366 */ +367 public Builder(Configuration conf, FileSystem fs) { +368 this.conf = conf; +369 this.cacheConf = CacheConfig.DISABLED; +370 this.fs = fs; +371 } +372 +373 /** +374 * Use either this method or {@link #withFilePath}, but not both. +375 * @param dir Path to column family directory. The directory is created if +376 * does not exist. The file is given a unique name within this +377 * directory. +378 * @return this (for chained invocation) +379 */ +380 public Builder withOutputDir(Path dir) { +381 Preconditions.checkNotNull(dir); +382 this.dir = dir; +383 return this; +384 } +385 +386 /** +387 * Use either this method or {@link #withOutputDir}, but not both. +388 * @param filePath the StoreFile path to write +389 * @return this (for chained invocation) +390 */ +391 public Builder withFilePath(Path filePath) { +392 Preconditions.checkNotNull(filePath); +393 this.filePath = filePath; +394 return this; +395 } +396 +397 /** +398 * @param favoredNodes an array of favored nodes or possibly null +399 * @return this (for chained invocation) +400 */ +401 public Builder withFavoredNodes(InetSocketAddress[] favoredNodes) { +402 this.favoredNodes = favoredNodes; +403 return this; +404 } +405 +406 public Builder withComparator(CellComparator comparator) { +407 Preconditions.checkNotNull(comparator); +408 this.comparator = comparator; +409 return this; +410 } +411 +412 public Builder withBloomType(BloomType bloomType) { +413 Preconditions.checkNotNull(bloomType); +414 this.bloomType = bloomType; +415 return this; +416 } +417 +418 /** +419 * @param maxKeyCount estimated maximum number of keys we expect to add +420 * @return this (for chained invocation) +421 */ +422 public Builder withMaxKeyCount(long maxKeyCount) { +423 this.maxKeyCount = maxKeyCount; +424 return this; +425 } +426 +427 public Builder withFileContext(HFileContext fileContext) { +428 this.fileContext = fileContext; +429 return this; +430 } +431 +432 public Builder withShouldDropCacheBehind(boolean shouldDropCacheBehind) { +433 this.shouldDropCacheBehind = shouldDropCacheBehind; +434 return this; +435 } +436 +437 /** +438 * Create a store file writer. Client is responsible for closing file when +439 * done. If metadata, add BEFORE closing using +440 * {@link StoreFileWriter#appendMetadata}. +441 */ +442 public StoreFileWriter build() throws IOException { +443 if ((dir == null ? 0 : 1) + (filePath == null ? 0 : 1) != 1) { +444 throw new IllegalArgumentException("Either specify parent directory " + +445 "or file path"); +446 } +447 +448 if (dir == null) { +449 dir = filePath.getParent(); +450 } +451 +452 if (!fs.exists(dir)) { +453 // Handle permission for non-HDFS filesystem properly +454 // See HBASE-17710 +455 HRegionFileSystem.mkdirs(fs, conf, dir); +456 } +457 +458 // set block storage policy for temp path +459 String policyName = this.conf.get(ColumnFamilyDescriptorBuilder.STORAGE_POLICY); +460 if (null == policyName) { +461 policyName = this.conf.get(HStore.BLOCK_STORAGE_POLICY_KEY); +462 } +463 FSUtils.setStoragePolicy(this.fs, dir, policyName); +464 +465 if (filePath == null) { +466 filePath = getUniqueFile(fs, dir); +467 if (!BloomFilterFactory.isGeneralBloomEnabled(conf)) { +468 bloomType = BloomType.NONE; +469 } +470 } +471 +472 if (comparator == null) { +473 comparator = CellComparator.COMPARATOR; +474 } +475 return new StoreFileWriter(fs, filePath, +476 conf, cacheConf, comparator, bloomType, maxKeyCount, favoredNodes, fileContext, +477 shouldDropCacheBehind); +478 } +479 } +480}