Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C71B3182B0 for ; Thu, 26 Nov 2015 21:52:28 +0000 (UTC) Received: (qmail 19955 invoked by uid 500); 26 Nov 2015 21:52:27 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 19701 invoked by uid 500); 26 Nov 2015 21:52:27 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 18444 invoked by uid 99); 26 Nov 2015 21:52:26 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Nov 2015 21:52:26 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B8298E2C66; Thu, 26 Nov 2015 21:52:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Thu, 26 Nov 2015 21:52:40 -0000 Message-Id: <218cd312b8b84b2d9b843bb6bb01ebce@git.apache.org> In-Reply-To: <357cad06470c4dbbad171d1776f70ffe@git.apache.org> References: <357cad06470c4dbbad171d1776f70ffe@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [16/24] hbase git commit: Published site at 0d72849581c1e5a50a2aa7757b909e5531b8d12c. http://git-wip-us.apache.org/repos/asf/hbase/blob/2fa570dd/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.MatchCode.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.MatchCode.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.MatchCode.html index bddf5c5..80b5066 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.MatchCode.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.MatchCode.html @@ -168,513 +168,518 @@ 160 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns, 161 ScanType scanType, long readPointToUse, long earliestPutTs, long oldestUnexpiredTS, 162 long now, RegionCoprocessorHost regionCoprocessorHost) throws IOException { -163 this.tr = scan.getTimeRange(); -164 this.rowComparator = scanInfo.getComparator(); -165 this.regionCoprocessorHost = regionCoprocessorHost; -166 this.deletes = instantiateDeleteTracker(); -167 this.stopRow = scan.getStopRow(); -168 this.startKey = CellUtil.createFirstDeleteFamilyCellOnRow(scan.getStartRow(), -169 scanInfo.getFamily()); -170 this.filter = scan.getFilter(); -171 this.earliestPutTs = earliestPutTs; -172 this.oldestUnexpiredTS = oldestUnexpiredTS; -173 this.now = now; -174 -175 this.maxReadPointToTrackVersions = readPointToUse; -176 this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes(); -177 this.ttl = oldestUnexpiredTS; -178 -179 /* how to deal with deletes */ -180 this.isUserScan = scanType == ScanType.USER_SCAN; -181 // keep deleted cells: if compaction or raw scan -182 this.keepDeletedCells = scan.isRaw() ? KeepDeletedCells.TRUE : -183 isUserScan ? KeepDeletedCells.FALSE : scanInfo.getKeepDeletedCells(); -184 // retain deletes: if minor compaction or raw scanisDone -185 this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw(); -186 // seePastDeleteMarker: user initiated scans -187 this.seePastDeleteMarkers = -188 scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE && isUserScan; -189 -190 int maxVersions = -191 scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(), -192 scanInfo.getMaxVersions()); -193 -194 // Single branch to deal with two types of reads (columns vs all in family) -195 if (columns == null || columns.size() == 0) { -196 // there is always a null column in the wildcard column query. -197 hasNullColumn = true; +163 TimeRange timeRange = scan.getColumnFamilyTimeRange().get(scanInfo.getFamily()); +164 if (timeRange == null) { +165 this.tr = scan.getTimeRange(); +166 } else { +167 this.tr = timeRange; +168 } +169 this.rowComparator = scanInfo.getComparator(); +170 this.regionCoprocessorHost = regionCoprocessorHost; +171 this.deletes = instantiateDeleteTracker(); +172 this.stopRow = scan.getStopRow(); +173 this.startKey = CellUtil.createFirstDeleteFamilyCellOnRow(scan.getStartRow(), +174 scanInfo.getFamily()); +175 this.filter = scan.getFilter(); +176 this.earliestPutTs = earliestPutTs; +177 this.oldestUnexpiredTS = oldestUnexpiredTS; +178 this.now = now; +179 +180 this.maxReadPointToTrackVersions = readPointToUse; +181 this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes(); +182 this.ttl = oldestUnexpiredTS; +183 +184 /* how to deal with deletes */ +185 this.isUserScan = scanType == ScanType.USER_SCAN; +186 // keep deleted cells: if compaction or raw scan +187 this.keepDeletedCells = scan.isRaw() ? KeepDeletedCells.TRUE : +188 isUserScan ? KeepDeletedCells.FALSE : scanInfo.getKeepDeletedCells(); +189 // retain deletes: if minor compaction or raw scanisDone +190 this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw(); +191 // seePastDeleteMarker: user initiated scans +192 this.seePastDeleteMarkers = +193 scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE && isUserScan; +194 +195 int maxVersions = +196 scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(), +197 scanInfo.getMaxVersions()); 198 -199 // use a specialized scan for wildcard column tracker. -200 this.columns = new ScanWildcardColumnTracker( -201 scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS); -202 } else { -203 // whether there is null column in the explicit column query -204 hasNullColumn = (columns.first().length == 0); -205 -206 // We can share the ExplicitColumnTracker, diff is we reset -207 // between rows, not between storefiles. -208 this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions, -209 oldestUnexpiredTS); -210 } -211 this.isReversed = scan.isReversed(); -212 } -213 -214 private DeleteTracker instantiateDeleteTracker() throws IOException { -215 DeleteTracker tracker = new ScanDeleteTracker(); -216 if (regionCoprocessorHost != null) { -217 tracker = regionCoprocessorHost.postInstantiateDeleteTracker(tracker); -218 } -219 return tracker; -220 } -221 -222 /** -223 * Construct a QueryMatcher for a scan that drop deletes from a limited range of rows. -224 * @param scan -225 * @param scanInfo The store's immutable scan info -226 * @param columns -227 * @param earliestPutTs Earliest put seen in any of the store files. -228 * @param oldestUnexpiredTS the oldest timestamp we are interested in, based on TTL -229 * @param now the current server time -230 * @param dropDeletesFromRow The inclusive left bound of the range; can be EMPTY_START_ROW. -231 * @param dropDeletesToRow The exclusive right bound of the range; can be EMPTY_END_ROW. -232 * @param regionCoprocessorHost -233 * @throws IOException -234 */ -235 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns, -236 long readPointToUse, long earliestPutTs, long oldestUnexpiredTS, long now, -237 byte[] dropDeletesFromRow, byte[] dropDeletesToRow, -238 RegionCoprocessorHost regionCoprocessorHost) throws IOException { -239 this(scan, scanInfo, columns, ScanType.COMPACT_RETAIN_DELETES, readPointToUse, earliestPutTs, -240 oldestUnexpiredTS, now, regionCoprocessorHost); -241 Preconditions.checkArgument((dropDeletesFromRow != null) && (dropDeletesToRow != null)); -242 this.dropDeletesFromRow = dropDeletesFromRow; -243 this.dropDeletesToRow = dropDeletesToRow; -244 } -245 -246 /* -247 * Constructor for tests -248 */ -249 ScanQueryMatcher(Scan scan, ScanInfo scanInfo, -250 NavigableSet<byte[]> columns, long oldestUnexpiredTS, long now) throws IOException { -251 this(scan, scanInfo, columns, ScanType.USER_SCAN, -252 Long.MAX_VALUE, /* max Readpoint to track versions */ -253 HConstants.LATEST_TIMESTAMP, oldestUnexpiredTS, now, null); -254 } -255 -256 /** -257 * -258 * @return whether there is an null column in the query -259 */ -260 public boolean hasNullColumnInQuery() { -261 return hasNullColumn; -262 } -263 -264 /** -265 * Determines if the caller should do one of several things: -266 * - seek/skip to the next row (MatchCode.SEEK_NEXT_ROW) -267 * - seek/skip to the next column (MatchCode.SEEK_NEXT_COL) -268 * - include the current KeyValue (MatchCode.INCLUDE) -269 * - ignore the current KeyValue (MatchCode.SKIP) -270 * - got to the next row (MatchCode.DONE) -271 * -272 * @param cell KeyValue to check -273 * @return The match code instance. -274 * @throws IOException in case there is an internal consistency problem -275 * caused by a data corruption. -276 */ -277 public MatchCode match(Cell cell) throws IOException { -278 if (filter != null && filter.filterAllRemaining()) { -279 return MatchCode.DONE_SCAN; -280 } -281 int ret = this.rowComparator.compareRows(curCell, cell); -282 if (!this.isReversed) { -283 if (ret <= -1) { -284 return MatchCode.DONE; -285 } else if (ret >= 1) { -286 // could optimize this, if necessary? -287 // Could also be called SEEK_TO_CURRENT_ROW, but this -288 // should be rare/never happens. -289 return MatchCode.SEEK_NEXT_ROW; -290 } -291 } else { -292 if (ret <= -1) { -293 return MatchCode.SEEK_NEXT_ROW; -294 } else if (ret >= 1) { -295 return MatchCode.DONE; -296 } -297 } -298 -299 // optimize case. -300 if (this.stickyNextRow) -301 return MatchCode.SEEK_NEXT_ROW; -302 -303 if (this.columns.done()) { -304 stickyNextRow = true; -305 return MatchCode.SEEK_NEXT_ROW; -306 } +199 // Single branch to deal with two types of reads (columns vs all in family) +200 if (columns == null || columns.size() == 0) { +201 // there is always a null column in the wildcard column query. +202 hasNullColumn = true; +203 +204 // use a specialized scan for wildcard column tracker. +205 this.columns = new ScanWildcardColumnTracker( +206 scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS); +207 } else { +208 // whether there is null column in the explicit column query +209 hasNullColumn = (columns.first().length == 0); +210 +211 // We can share the ExplicitColumnTracker, diff is we reset +212 // between rows, not between storefiles. +213 this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions, +214 oldestUnexpiredTS); +215 } +216 this.isReversed = scan.isReversed(); +217 } +218 +219 private DeleteTracker instantiateDeleteTracker() throws IOException { +220 DeleteTracker tracker = new ScanDeleteTracker(); +221 if (regionCoprocessorHost != null) { +222 tracker = regionCoprocessorHost.postInstantiateDeleteTracker(tracker); +223 } +224 return tracker; +225 } +226 +227 /** +228 * Construct a QueryMatcher for a scan that drop deletes from a limited range of rows. +229 * @param scan +230 * @param scanInfo The store's immutable scan info +231 * @param columns +232 * @param earliestPutTs Earliest put seen in any of the store files. +233 * @param oldestUnexpiredTS the oldest timestamp we are interested in, based on TTL +234 * @param now the current server time +235 * @param dropDeletesFromRow The inclusive left bound of the range; can be EMPTY_START_ROW. +236 * @param dropDeletesToRow The exclusive right bound of the range; can be EMPTY_END_ROW. +237 * @param regionCoprocessorHost +238 * @throws IOException +239 */ +240 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns, +241 long readPointToUse, long earliestPutTs, long oldestUnexpiredTS, long now, +242 byte[] dropDeletesFromRow, byte[] dropDeletesToRow, +243 RegionCoprocessorHost regionCoprocessorHost) throws IOException { +244 this(scan, scanInfo, columns, ScanType.COMPACT_RETAIN_DELETES, readPointToUse, earliestPutTs, +245 oldestUnexpiredTS, now, regionCoprocessorHost); +246 Preconditions.checkArgument((dropDeletesFromRow != null) && (dropDeletesToRow != null)); +247 this.dropDeletesFromRow = dropDeletesFromRow; +248 this.dropDeletesToRow = dropDeletesToRow; +249 } +250 +251 /* +252 * Constructor for tests +253 */ +254 ScanQueryMatcher(Scan scan, ScanInfo scanInfo, +255 NavigableSet<byte[]> columns, long oldestUnexpiredTS, long now) throws IOException { +256 this(scan, scanInfo, columns, ScanType.USER_SCAN, +257 Long.MAX_VALUE, /* max Readpoint to track versions */ +258 HConstants.LATEST_TIMESTAMP, oldestUnexpiredTS, now, null); +259 } +260 +261 /** +262 * +263 * @return whether there is an null column in the query +264 */ +265 public boolean hasNullColumnInQuery() { +266 return hasNullColumn; +267 } +268 +269 /** +270 * Determines if the caller should do one of several things: +271 * - seek/skip to the next row (MatchCode.SEEK_NEXT_ROW) +272 * - seek/skip to the next column (MatchCode.SEEK_NEXT_COL) +273 * - include the current KeyValue (MatchCode.INCLUDE) +274 * - ignore the current KeyValue (MatchCode.SKIP) +275 * - got to the next row (MatchCode.DONE) +276 * +277 * @param cell KeyValue to check +278 * @return The match code instance. +279 * @throws IOException in case there is an internal consistency problem +280 * caused by a data corruption. +281 */ +282 public MatchCode match(Cell cell) throws IOException { +283 if (filter != null && filter.filterAllRemaining()) { +284 return MatchCode.DONE_SCAN; +285 } +286 int ret = this.rowComparator.compareRows(curCell, cell); +287 if (!this.isReversed) { +288 if (ret <= -1) { +289 return MatchCode.DONE; +290 } else if (ret >= 1) { +291 // could optimize this, if necessary? +292 // Could also be called SEEK_TO_CURRENT_ROW, but this +293 // should be rare/never happens. +294 return MatchCode.SEEK_NEXT_ROW; +295 } +296 } else { +297 if (ret <= -1) { +298 return MatchCode.SEEK_NEXT_ROW; +299 } else if (ret >= 1) { +300 return MatchCode.DONE; +301 } +302 } +303 +304 // optimize case. +305 if (this.stickyNextRow) +306 return MatchCode.SEEK_NEXT_ROW; 307 -308 long timestamp = cell.getTimestamp(); -309 // check for early out based on timestamp alone -310 if (columns.isDone(timestamp)) { -311 return columns.getNextRowOrNextColumn(cell); -312 } -313 // check if the cell is expired by cell TTL -314 if (HStore.isCellTTLExpired(cell, this.oldestUnexpiredTS, this.now)) { -315 return MatchCode.SKIP; -316 } -317 -318 /* -319 * The delete logic is pretty complicated now. -320 * This is corroborated by the following: -321 * 1. The store might be instructed to keep deleted rows around. -322 * 2. A scan can optionally see past a delete marker now. -323 * 3. If deleted rows are kept, we have to find out when we can -324 * remove the delete markers. -325 * 4. Family delete markers are always first (regardless of their TS) -326 * 5. Delete markers should not be counted as version -327 * 6. Delete markers affect puts of the *same* TS -328 * 7. Delete marker need to be version counted together with puts -329 * they affect -330 */ -331 byte typeByte = cell.getTypeByte(); -332 long mvccVersion = cell.getSequenceId(); -333 if (CellUtil.isDelete(cell)) { -334 if (keepDeletedCells == KeepDeletedCells.FALSE -335 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < ttl)) { -336 // first ignore delete markers if the scanner can do so, and the -337 // range does not include the marker -338 // -339 // during flushes and compactions also ignore delete markers newer -340 // than the readpoint of any open scanner, this prevents deleted -341 // rows that could still be seen by a scanner from being collected -342 boolean includeDeleteMarker = seePastDeleteMarkers ? -343 tr.withinTimeRange(timestamp) : -344 tr.withinOrAfterTimeRange(timestamp); -345 if (includeDeleteMarker -346 && mvccVersion <= maxReadPointToTrackVersions) { -347 this.deletes.add(cell); -348 } -349 // Can't early out now, because DelFam come before any other keys -350 } -351 -352 if ((!isUserScan) -353 && timeToPurgeDeletes > 0 -354 && (EnvironmentEdgeManager.currentTime() - timestamp) -355 <= timeToPurgeDeletes) { -356 return MatchCode.INCLUDE; -357 } else if (retainDeletesInOutput || mvccVersion > maxReadPointToTrackVersions) { -358 // always include or it is not time yet to check whether it is OK -359 // to purge deltes or not -360 if (!isUserScan) { -361 // if this is not a user scan (compaction), we can filter this deletemarker right here -362 // otherwise (i.e. a "raw" scan) we fall through to normal version and timerange checking -363 return MatchCode.INCLUDE; -364 } -365 } else if (keepDeletedCells == KeepDeletedCells.TRUE -366 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= ttl)) { -367 if (timestamp < earliestPutTs) { -368 // keeping delete rows, but there are no puts older than -369 // this delete in the store files. -370 return columns.getNextRowOrNextColumn(cell); -371 } -372 // else: fall through and do version counting on the -373 // delete markers -374 } else { -375 return MatchCode.SKIP; -376 } -377 // note the following next else if... -378 // delete marker are not subject to other delete markers -379 } else if (!this.deletes.isEmpty()) { -380 DeleteResult deleteResult = deletes.isDeleted(cell); -381 switch (deleteResult) { -382 case FAMILY_DELETED: -383 case COLUMN_DELETED: -384 return columns.getNextRowOrNextColumn(cell); -385 case VERSION_DELETED: -386 case FAMILY_VERSION_DELETED: -387 return MatchCode.SKIP; -388 case NOT_DELETED: -389 break; -390 default: -391 throw new RuntimeException("UNEXPECTED"); -392 } -393 } -394 -395 int timestampComparison = tr.compare(timestamp); -396 if (timestampComparison >= 1) { -397 return MatchCode.SKIP; -398 } else if (timestampComparison <= -1) { -399 return columns.getNextRowOrNextColumn(cell); -400 } -401 -402 // STEP 1: Check if the column is part of the requested columns -403 MatchCode colChecker = columns.checkColumn(cell, typeByte); -404 if (colChecker == MatchCode.INCLUDE) { -405 ReturnCode filterResponse = ReturnCode.SKIP; -406 // STEP 2: Yes, the column is part of the requested columns. Check if filter is present -407 if (filter != null) { -408 // STEP 3: Filter the key value and return if it filters out -409 filterResponse = filter.filterKeyValue(cell); -410 switch (filterResponse) { -411 case SKIP: -412 return MatchCode.SKIP; -413 case NEXT_COL: -414 return columns.getNextRowOrNextColumn(cell); -415 case NEXT_ROW: -416 stickyNextRow = true; -417 return MatchCode.SEEK_NEXT_ROW; -418 case SEEK_NEXT_USING_HINT: -419 return MatchCode.SEEK_NEXT_USING_HINT; -420 default: -421 //It means it is either include or include and seek next -422 break; -423 } -424 } -425 /* -426 * STEP 4: Reaching this step means the column is part of the requested columns and either -427 * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response. -428 * Now check the number of versions needed. This method call returns SKIP, INCLUDE, -429 * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL. -430 * -431 * FilterResponse ColumnChecker Desired behavior -432 * INCLUDE SKIP row has already been included, SKIP. -433 * INCLUDE INCLUDE INCLUDE -434 * INCLUDE INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL -435 * INCLUDE INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW -436 * INCLUDE_AND_SEEK_NEXT_COL SKIP row has already been included, SKIP. -437 * INCLUDE_AND_SEEK_NEXT_COL INCLUDE INCLUDE_AND_SEEK_NEXT_COL -438 * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL -439 * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW -440 * -441 * In all the above scenarios, we return the column checker return value except for -442 * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE) -443 */ -444 colChecker = columns.checkVersions(cell, timestamp, typeByte, -445 mvccVersion > maxReadPointToTrackVersions); -446 //Optimize with stickyNextRow -447 boolean seekNextRowFromEssential = filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW && -448 filter.isFamilyEssential(cell.getFamilyArray()); -449 if (colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW || seekNextRowFromEssential) { -450 stickyNextRow = true; -451 } -452 if (filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW) { -453 if (colChecker != MatchCode.SKIP) { -454 return MatchCode.INCLUDE_AND_SEEK_NEXT_ROW; -455 } -456 return MatchCode.SEEK_NEXT_ROW; -457 } -458 return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL && -459 colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL -460 : colChecker; -461 } -462 stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true -463 : stickyNextRow; -464 return colChecker; -465 } -466 -467 /** Handle partial-drop-deletes. As we match keys in order, when we have a range from which -468 * we can drop deletes, we can set retainDeletesInOutput to false for the duration of this -469 * range only, and maintain consistency. */ -470 private void checkPartialDropDeleteRange(Cell curCell) { -471 // If partial-drop-deletes are used, initially, dropDeletesFromRow and dropDeletesToRow -472 // are both set, and the matcher is set to retain deletes. We assume ordered keys. When -473 // dropDeletesFromRow is leq current kv, we start dropping deletes and reset -474 // dropDeletesFromRow; thus the 2nd "if" starts to apply. -475 if ((dropDeletesFromRow != null) -476 && (Arrays.equals(dropDeletesFromRow, HConstants.EMPTY_START_ROW) || -477 (CellComparator.COMPARATOR.compareRows(curCell, dropDeletesFromRow, 0, -478 dropDeletesFromRow.length) >= 0))) { -479 retainDeletesInOutput = false; -480 dropDeletesFromRow = null; -481 } -482 // If dropDeletesFromRow is null and dropDeletesToRow is set, we are inside the partial- -483 // drop-deletes range. When dropDeletesToRow is leq current kv, we stop dropping deletes, -484 // and reset dropDeletesToRow so that we don't do any more compares. -485 if ((dropDeletesFromRow == null) -486 && (dropDeletesToRow != null) -487 && !Arrays.equals(dropDeletesToRow, HConstants.EMPTY_END_ROW) -488 && (CellComparator.COMPARATOR -489 .compareRows(curCell, dropDeletesToRow, 0, dropDeletesToRow.length) >= 0)) { -490 retainDeletesInOutput = true; -491 dropDeletesToRow = null; -492 } -493 } -494 -495 public boolean moreRowsMayExistAfter(Cell kv) { -496 if (this.isReversed) { -497 if (rowComparator.compareRows(kv, stopRow, 0, stopRow.length) <= 0) { -498 return false; -499 } else { -500 return true; -501 } -502 } -503 if (!Bytes.equals(stopRow , HConstants.EMPTY_END_ROW) && -504 rowComparator.compareRows(kv, stopRow, 0, stopRow.length) >= 0) { -505 // KV >= STOPROW -506 // then NO there is nothing left. -507 return false; -508 } else { -509 return true; -510 } -511 } -512 -513 /** -514 * Set the row when there is change in row -515 * @param curCell -516 */ -517 public void setToNewRow(Cell curCell) { -518 checkPartialDropDeleteRange(curCell); -519 this.curCell = curCell; -520 reset(); -521 } -522 -523 public void reset() { -524 this.deletes.reset(); -525 this.columns.reset(); -526 -527 stickyNextRow = false; -528 } -529 -530 /** -531 * -532 * @return the start key -533 */ -534 public Cell getStartKey() { -535 return this.startKey; -536 } -537 -538 /** -539 * -540 * @return the Filter -541 */ -542 Filter getFilter() { -543 return this.filter; -544 } -545 -546 public Cell getNextKeyHint(Cell kv) throws IOException { -547 if (filter == null) { -548 return null; -549 } else { -550 return filter.getNextCellHint(kv); -551 } -552 } -553 -554 public Cell getKeyForNextColumn(Cell kv) { -555 ColumnCount nextColumn = columns.getColumnHint(); -556 if (nextColumn == null) { -557 return CellUtil.createLastOnRowCol(kv); -558 } else { -559 return CellUtil.createFirstOnRowCol(kv, nextColumn.getBuffer(), nextColumn.getOffset(), -560 nextColumn.getLength()); -561 } -562 } -563 -564 public Cell getKeyForNextRow(Cell c) { -565 return CellUtil.createLastOnRow(c); -566 } -567 -568 /** -569 * @param nextIndexed the key of the next entry in the block index (if any) -570 * @param kv The Cell we're using to calculate the seek key -571 * @return result of the compare between the indexed key and the key portion of the passed cell -572 */ -573 public int compareKeyForNextRow(Cell nextIndexed, Cell kv) { -574 return rowComparator.compareKeyBasedOnColHint(nextIndexed, kv, 0, 0, null, 0, 0, -575 HConstants.OLDEST_TIMESTAMP, Type.Minimum.getCode()); -576 } -577 -578 /** -579 * @param nextIndexed the key of the next entry in the block index (if any) -580 * @param currentCell The Cell we're using to calculate the seek key -581 * @return result of the compare between the indexed key and the key portion of the passed cell -582 */ -583 public int compareKeyForNextColumn(Cell nextIndexed, Cell currentCell) { -584 ColumnCount nextColumn = columns.getColumnHint(); -585 if (nextColumn == null) { -586 return rowComparator.compareKeyBasedOnColHint(nextIndexed, currentCell, 0, 0, null, 0, 0, -587 HConstants.OLDEST_TIMESTAMP, Type.Minimum.getCode()); -588 } else { -589 return rowComparator.compareKeyBasedOnColHint(nextIndexed, currentCell, -590 currentCell.getFamilyOffset(), currentCell.getFamilyLength(), nextColumn.getBuffer(), -591 nextColumn.getOffset(), nextColumn.getLength(), HConstants.LATEST_TIMESTAMP, -592 Type.Maximum.getCode()); -593 } -594 } -595 -596 //Used only for testing purposes -597 static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset, -598 int length, long ttl, byte type, boolean ignoreCount) throws IOException { -599 KeyValue kv = KeyValueUtil.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY, 0, 0, -600 HConstants.EMPTY_BYTE_ARRAY, 0, 0, bytes, offset, length); -601 MatchCode matchCode = columnTracker.checkColumn(kv, type); -602 if (matchCode == MatchCode.INCLUDE) { -603 return columnTracker.checkVersions(kv, ttl, type, ignoreCount); -604 } -605 return matchCode; -606 } -607 -608 /** -609 * {@link #match} return codes. These instruct the scanner moving through -610 * memstores and StoreFiles what to do with the current KeyValue. -611 * <p> -612 * Additionally, this contains "early-out" language to tell the scanner to -613 * move on to the next File (memstore or Storefile), or to return immediately. -614 */ -615 public static enum MatchCode { -616 /** -617 * Include KeyValue in the returned result -618 */ -619 INCLUDE, -620 +308 if (this.columns.done()) { +309 stickyNextRow = true; +310 return MatchCode.SEEK_NEXT_ROW; +311 } +312 +313 long timestamp = cell.getTimestamp(); +314 // check for early out based on timestamp alone +315 if (columns.isDone(timestamp)) { +316 return columns.getNextRowOrNextColumn(cell); +317 } +318 // check if the cell is expired by cell TTL +319 if (HStore.isCellTTLExpired(cell, this.oldestUnexpiredTS, this.now)) { +320 return MatchCode.SKIP; +321 } +322 +323 /* +324 * The delete logic is pretty complicated now. +325 * This is corroborated by the following: +326 * 1. The store might be instructed to keep deleted rows around. +327 * 2. A scan can optionally see past a delete marker now. +328 * 3. If deleted rows are kept, we have to find out when we can +329 * remove the delete markers. +330 * 4. Family delete markers are always first (regardless of their TS) +331 * 5. Delete markers should not be counted as version +332 * 6. Delete markers affect puts of the *same* TS +333 * 7. Delete marker need to be version counted together with puts +334 * they affect +335 */ +336 byte typeByte = cell.getTypeByte(); +337 long mvccVersion = cell.getSequenceId(); +338 if (CellUtil.isDelete(cell)) { +339 if (keepDeletedCells == KeepDeletedCells.FALSE +340 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < ttl)) { +341 // first ignore delete markers if the scanner can do so, and the +342 // range does not include the marker +343 // +344 // during flushes and compactions also ignore delete markers newer +345 // than the readpoint of any open scanner, this prevents deleted +346 // rows that could still be seen by a scanner from being collected +347 boolean includeDeleteMarker = seePastDeleteMarkers ? +348 tr.withinTimeRange(timestamp) : +349 tr.withinOrAfterTimeRange(timestamp); +350 if (includeDeleteMarker +351 && mvccVersion <= maxReadPointToTrackVersions) { +352 this.deletes.add(cell); +353 } +354 // Can't early out now, because DelFam come before any other keys +355 } +356 +357 if ((!isUserScan) +358 && timeToPurgeDeletes > 0 +359 && (EnvironmentEdgeManager.currentTime() - timestamp) +360 <= timeToPurgeDeletes) { +361 return MatchCode.INCLUDE; +362 } else if (retainDeletesInOutput || mvccVersion > maxReadPointToTrackVersions) { +363 // always include or it is not time yet to check whether it is OK +364 // to purge deltes or not +365 if (!isUserScan) { +366 // if this is not a user scan (compaction), we can filter this deletemarker right here +367 // otherwise (i.e. a "raw" scan) we fall through to normal version and timerange checking +368 return MatchCode.INCLUDE; +369 } +370 } else if (keepDeletedCells == KeepDeletedCells.TRUE +371 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= ttl)) { +372 if (timestamp < earliestPutTs) { +373 // keeping delete rows, but there are no puts older than +374 // this delete in the store files. +375 return columns.getNextRowOrNextColumn(cell); +376 } +377 // else: fall through and do version counting on the +378 // delete markers +379 } else { +380 return MatchCode.SKIP; +381 } +382 // note the following next else if... +383 // delete marker are not subject to other delete markers +384 } else if (!this.deletes.isEmpty()) { +385 DeleteResult deleteResult = deletes.isDeleted(cell); +386 switch (deleteResult) { +387 case FAMILY_DELETED: +388 case COLUMN_DELETED: +389 return columns.getNextRowOrNextColumn(cell); +390 case VERSION_DELETED: +391 case FAMILY_VERSION_DELETED: +392 return MatchCode.SKIP; +393 case NOT_DELETED: +394 break; +395 default: +396 throw new RuntimeException("UNEXPECTED"); +397 } +398 } +399 +400 int timestampComparison = tr.compare(timestamp); +401 if (timestampComparison >= 1) { +402 return MatchCode.SKIP; +403 } else if (timestampComparison <= -1) { +404 return columns.getNextRowOrNextColumn(cell); +405 } +406 +407 // STEP 1: Check if the column is part of the requested columns +408 MatchCode colChecker = columns.checkColumn(cell, typeByte); +409 if (colChecker == MatchCode.INCLUDE) { +410 ReturnCode filterResponse = ReturnCode.SKIP; +411 // STEP 2: Yes, the column is part of the requested columns. Check if filter is present +412 if (filter != null) { +413 // STEP 3: Filter the key value and return if it filters out +414 filterResponse = filter.filterKeyValue(cell); +415 switch (filterResponse) { +416 case SKIP: +417 return MatchCode.SKIP; +418 case NEXT_COL: +419 return columns.getNextRowOrNextColumn(cell); +420 case NEXT_ROW: +421 stickyNextRow = true; +422 return MatchCode.SEEK_NEXT_ROW; +423 case SEEK_NEXT_USING_HINT: +424 return MatchCode.SEEK_NEXT_USING_HINT; +425 default: +426 //It means it is either include or include and seek next +427 break; +428 } +429 } +430 /* +431 * STEP 4: Reaching this step means the column is part of the requested columns and either +432 * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response. +433 * Now check the number of versions needed. This method call returns SKIP, INCLUDE, +434 * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL. +435 * +436 * FilterResponse ColumnChecker Desired behavior +437 * INCLUDE SKIP row has already been included, SKIP. +438 * INCLUDE INCLUDE INCLUDE +439 * INCLUDE INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL +440 * INCLUDE INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW +441 * INCLUDE_AND_SEEK_NEXT_COL SKIP row has already been included, SKIP. +442 * INCLUDE_AND_SEEK_NEXT_COL INCLUDE INCLUDE_AND_SEEK_NEXT_COL +443 * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL +444 * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW +445 * +446 * In all the above scenarios, we return the column checker return value except for +447 * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE) +448 */ +449 colChecker = columns.checkVersions(cell, timestamp, typeByte, +450 mvccVersion > maxReadPointToTrackVersions); +451 //Optimize with stickyNextRow +452 boolean seekNextRowFromEssential = filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW && +453 filter.isFamilyEssential(cell.getFamilyArray()); +454 if (colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW || seekNextRowFromEssential) { +455 stickyNextRow = true; +456 } +457 if (filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW) { +458 if (colChecker != MatchCode.SKIP) { +459 return MatchCode.INCLUDE_AND_SEEK_NEXT_ROW; +460 } +461 return MatchCode.SEEK_NEXT_ROW; +462 } +463 return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL && +464 colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL +465 : colChecker; +466 } +467 stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true +468 : stickyNextRow; +469 return colChecker; +470 } +471 +472 /** Handle partial-drop-deletes. As we match keys in order, when we have a range from which +473 * we can drop deletes, we can set retainDeletesInOutput to false for the duration of this +474 * range only, and maintain consistency. */ +475 private void checkPartialDropDeleteRange(Cell curCell) { +476 // If partial-drop-deletes are used, initially, dropDeletesFromRow and dropDeletesToRow +477 // are both set, and the matcher is set to retain deletes. We assume ordered keys. When +478 // dropDeletesFromRow is leq current kv, we start dropping deletes and reset +479 // dropDeletesFromRow; thus the 2nd "if" starts to apply. +480 if ((dropDeletesFromRow != null) +481 && (Arrays.equals(dropDeletesFromRow, HConstants.EMPTY_START_ROW) || +482 (CellComparator.COMPARATOR.compareRows(curCell, dropDeletesFromRow, 0, +483 dropDeletesFromRow.length) >= 0))) { +484 retainDeletesInOutput = false; +485 dropDeletesFromRow = null; +486 } +487 // If dropDeletesFromRow is null and dropDeletesToRow is set, we are inside the partial- +488 // drop-deletes range. When dropDeletesToRow is leq current kv, we stop dropping deletes, +489 // and reset dropDeletesToRow so that we don't do any more compares. +490 if ((dropDeletesFromRow == null) +491 && (dropDeletesToRow != null) +492 && !Arrays.equals(dropDeletesToRow, HConstants.EMPTY_END_ROW) +493 && (CellComparator.COMPARATOR +494 .compareRows(curCell, dropDeletesToRow, 0, dropDeletesToRow.length) >= 0)) { +495 retainDeletesInOutput = true; +496 dropDeletesToRow = null; +497 } +498 } +499 +500 public boolean moreRowsMayExistAfter(Cell kv) { +501 if (this.isReversed) { +502 if (rowComparator.compareRows(kv, stopRow, 0, stopRow.length) <= 0) { +503 return false; +504 } else { +505 return true; +506 } +507 } +508 if (!Bytes.equals(stopRow , HConstants.EMPTY_END_ROW) && +509 rowComparator.compareRows(kv, stopRow, 0, stopRow.length) >= 0) { +510 // KV >= STOPROW +511 // then NO there is nothing left. +512 return false; +513 } else { +514 return true; +515 } +516 } +517 +518 /** +519 * Set the row when there is change in row +520 * @param curCell +521 */ +522 public void setToNewRow(Cell curCell) { +523 checkPartialDropDeleteRange(curCell); +524 this.curCell = curCell; +525 reset(); +526 } +527 +528 public void reset() { +529 this.deletes.reset(); +530 this.columns.reset(); +531 +532 stickyNextRow = false; +533 } +534 +535 /** +536 * +537 * @return the start key +538 */ +539 public Cell getStartKey() { +540 return this.startKey; +541 } +542 +543 /** +544 * +545 * @return the Filter +546 */ +547 Filter getFilter() { +548 return this.filter; +549 } +550 +551 public Cell getNextKeyHint(Cell kv) throws IOException { +552 if (filter == null) { +553 return null; +554 } else { +555 return filter.getNextCellHint(kv); +556 } +557 } +558 +559 public Cell getKeyForNextColumn(Cell kv) { +560 ColumnCount nextColumn = columns.getColumnHint(); +561 if (nextColumn == null) { +562 return CellUtil.createLastOnRowCol(kv); +563 } else { +564 return CellUtil.createFirstOnRowCol(kv, nextColumn.getBuffer(), nextColumn.getOffset(), +565 nextColumn.getLength()); +566 } +567 } +568 +569 public Cell getKeyForNextRow(Cell c) { +570 return CellUtil.createLastOnRow(c); +571 } +572 +573 /** +574 * @param nextIndexed the key of the next entry in the block index (if any) +575 * @param kv The Cell we're using to calculate the seek key +576 * @return result of the compare between the indexed key and the key portion of the passed cell +577 */ +578 public int compareKeyForNextRow(Cell nextIndexed, Cell kv