Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 57186200CF0 for ; Thu, 7 Sep 2017 17:13:55 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 56E2D1617B3; Thu, 7 Sep 2017 15:13:55 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id CECFF161A7C for ; Thu, 7 Sep 2017 17:13:52 +0200 (CEST) Received: (qmail 73330 invoked by uid 500); 7 Sep 2017 15:13:50 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 72228 invoked by uid 99); 7 Sep 2017 15:13:49 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 07 Sep 2017 15:13:49 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7E500F56C7; Thu, 7 Sep 2017 15:13:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Thu, 07 Sep 2017 15:14:07 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [20/27] hbase-site git commit: Published site at . archived-at: Thu, 07 Sep 2017 15:13:55 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/f9e8b54d/devapidocs/src-html/org/apache/hadoop/hbase/filter/FilterList.Operator.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/filter/FilterList.Operator.html b/devapidocs/src-html/org/apache/hadoop/hbase/filter/FilterList.Operator.html index 0ed7e8b..7839e75 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/filter/FilterList.Operator.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/filter/FilterList.Operator.html @@ -30,615 +30,634 @@ 022import java.util.ArrayList; 023import java.util.Arrays; 024import java.util.Collections; -025import java.util.List; -026 -027import org.apache.hadoop.hbase.Cell; -028import org.apache.hadoop.hbase.CellComparator; -029import org.apache.hadoop.hbase.CellUtil; -030import org.apache.hadoop.hbase.KeyValueUtil; -031import org.apache.hadoop.hbase.classification.InterfaceAudience; -032import org.apache.hadoop.hbase.exceptions.DeserializationException; -033import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -034import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos; -035import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException; -036 -037/** -038 * Implementation of {@link Filter} that represents an ordered List of Filters -039 * which will be evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL} -040 * (<code>AND</code>) or {@link Operator#MUST_PASS_ONE} (<code>OR</code>). -041 * Since you can use Filter Lists as children of Filter Lists, you can create a -042 * hierarchy of filters to be evaluated. -043 * -044 * -045 * {@link Operator#MUST_PASS_ALL} evaluates lazily: evaluation stops as soon as one filter does -046 * not include the KeyValue. -047 * -048 * -049 * {@link Operator#MUST_PASS_ONE} evaluates non-lazily: all filters are always evaluated. -050 * -051 * -052 * Defaults to {@link Operator#MUST_PASS_ALL}. -053 */ -054@InterfaceAudience.Public -055final public class FilterList extends FilterBase { -056 /** set operator */ -057 @InterfaceAudience.Public -058 public static enum Operator { -059 /** !AND */ -060 MUST_PASS_ALL, -061 /** !OR */ -062 MUST_PASS_ONE -063 } -064 -065 private static final int MAX_LOG_FILTERS = 5; -066 private Operator operator = Operator.MUST_PASS_ALL; -067 private final List<Filter> filters; -068 private Filter seekHintFilter = null; -069 -070 /** -071 * Save previous return code and previous cell for every filter in filter list. For MUST_PASS_ONE, -072 * we use the previous return code to decide whether we should pass current cell encountered to -073 * the filter. For MUST_PASS_ALL, the two list are meaningless. -074 */ -075 private List<ReturnCode> prevFilterRCList = null; -076 private List<Cell> prevCellList = null; -077 -078 /** Reference Cell used by {@link #transformCell(Cell)} for validation purpose. */ -079 private Cell referenceCell = null; -080 -081 /** -082 * When filtering a given Cell in {@link #filterKeyValue(Cell)}, -083 * this stores the transformed Cell to be returned by {@link #transformCell(Cell)}. -084 * -085 * Individual filters transformation are applied only when the filter includes the Cell. -086 * Transformations are composed in the order specified by {@link #filters}. -087 */ -088 private Cell transformedCell = null; -089 -090 /** -091 * Constructor that takes a set of {@link Filter}s. The default operator -092 * MUST_PASS_ALL is assumed. -093 * All filters are cloned to internal list. -094 * @param rowFilters list of filters -095 */ -096 public FilterList(final List<Filter> rowFilters) { -097 reversed = getReversed(rowFilters, reversed); -098 this.filters = new ArrayList<>(rowFilters); -099 initPrevListForMustPassOne(rowFilters.size()); -100 } -101 -102 /** -103 * Constructor that takes a var arg number of {@link Filter}s. The fefault operator -104 * MUST_PASS_ALL is assumed. -105 * @param rowFilters -106 */ -107 public FilterList(final Filter... rowFilters) { -108 this(Arrays.asList(rowFilters)); -109 } -110 -111 /** -112 * Constructor that takes an operator. -113 * -114 * @param operator Operator to process filter set with. -115 */ -116 public FilterList(final Operator operator) { -117 this.operator = operator; -118 this.filters = new ArrayList<>(); -119 initPrevListForMustPassOne(filters.size()); -120 } -121 -122 /** -123 * Constructor that takes a set of {@link Filter}s and an operator. -124 * -125 * @param operator Operator to process filter set with. -126 * @param rowFilters Set of row filters. -127 */ -128 public FilterList(final Operator operator, final List<Filter> rowFilters) { -129 this(rowFilters); -130 this.operator = operator; -131 initPrevListForMustPassOne(rowFilters.size()); -132 } -133 -134 /** -135 * Constructor that takes a var arg number of {@link Filter}s and an operator. -136 * -137 * @param operator Operator to process filter set with. -138 * @param rowFilters Filters to use -139 */ -140 public FilterList(final Operator operator, final Filter... rowFilters) { -141 this(rowFilters); -142 this.operator = operator; -143 initPrevListForMustPassOne(rowFilters.length); -144 } -145 -146 public void initPrevListForMustPassOne(int size) { -147 if (operator == Operator.MUST_PASS_ONE) { -148 if (this.prevFilterRCList == null) { -149 prevFilterRCList = new ArrayList<>(Collections.nCopies(size, null)); -150 } -151 if (this.prevCellList == null) { -152 prevCellList = new ArrayList<>(Collections.nCopies(size, null)); -153 } -154 } -155 } -156 -157 -158 /** -159 * Get the operator. -160 * -161 * @return operator -162 */ -163 public Operator getOperator() { -164 return operator; -165 } -166 -167 /** -168 * Get the filters. -169 * -170 * @return filters -171 */ -172 public List<Filter> getFilters() { -173 return filters; -174 } -175 -176 public int size() { -177 return filters.size(); -178 } -179 -180 private boolean isEmpty() { -181 return filters.isEmpty(); -182 } -183 -184 private static boolean getReversed(List<Filter> rowFilters, boolean defaultValue) { -185 boolean rval = defaultValue; -186 boolean isFirst = true; -187 for (Filter f : rowFilters) { -188 if (isFirst) { -189 rval = f.isReversed(); -190 isFirst = false; -191 continue; -192 } -193 if (rval != f.isReversed()) { -194 throw new IllegalArgumentException("Filters in the list must have the same reversed flag"); -195 } -196 } -197 return rval; -198 } -199 private static void checkReversed(List<Filter> rowFilters, boolean expected) { -200 for (Filter filter : rowFilters) { -201 if (expected != filter.isReversed()) { -202 throw new IllegalArgumentException( -203 "Filters in the list must have the same reversed flag, expected=" -204 + expected); -205 } -206 } -207 } -208 -209 public void addFilter(List<Filter> filters) { -210 checkReversed(filters, isReversed()); -211 this.filters.addAll(filters); -212 if (operator == Operator.MUST_PASS_ONE) { -213 this.prevFilterRCList.addAll(Collections.nCopies(filters.size(), null)); -214 this.prevCellList.addAll(Collections.nCopies(filters.size(), null)); -215 } -216 } -217 -218 /** -219 * Add a filter. -220 * -221 * @param filter another filter -222 */ -223 public void addFilter(Filter filter) { -224 addFilter(Collections.singletonList(filter)); -225 } -226 -227 @Override -228 public void reset() throws IOException { -229 int listize = filters.size(); -230 for (int i = 0; i < listize; i++) { -231 filters.get(i).reset(); -232 if (operator == Operator.MUST_PASS_ONE) { -233 prevFilterRCList.set(i, null); -234 prevCellList.set(i, null); -235 } -236 } -237 seekHintFilter = null; -238 } -239 -240 @Override -241 public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException { -242 if (isEmpty()) { -243 return super.filterRowKey(rowKey, offset, length); -244 } -245 boolean flag = this.operator == Operator.MUST_PASS_ONE; -246 int listize = filters.size(); -247 for (int i = 0; i < listize; i++) { -248 Filter filter = filters.get(i); -249 if (this.operator == Operator.MUST_PASS_ALL) { -250 if (filter.filterAllRemaining() || -251 filter.filterRowKey(rowKey, offset, length)) { -252 flag = true; -253 } -254 } else if (this.operator == Operator.MUST_PASS_ONE) { -255 if (!filter.filterAllRemaining() && -256 !filter.filterRowKey(rowKey, offset, length)) { -257 flag = false; -258 } -259 } -260 } -261 return flag; -262 } -263 -264 @Override -265 public boolean filterRowKey(Cell firstRowCell) throws IOException { -266 if (isEmpty()) { -267 return super.filterRowKey(firstRowCell); -268 } -269 boolean flag = this.operator == Operator.MUST_PASS_ONE; -270 int listize = filters.size(); -271 for (int i = 0; i < listize; i++) { -272 Filter filter = filters.get(i); -273 if (this.operator == Operator.MUST_PASS_ALL) { -274 if (filter.filterAllRemaining() || filter.filterRowKey(firstRowCell)) { -275 flag = true; -276 } -277 } else if (this.operator == Operator.MUST_PASS_ONE) { -278 if (!filter.filterAllRemaining() && !filter.filterRowKey(firstRowCell)) { -279 flag = false; -280 } -281 } -282 } -283 return flag; -284 } -285 -286 @Override -287 public boolean filterAllRemaining() throws IOException { -288 if (isEmpty()) { -289 return super.filterAllRemaining(); -290 } -291 int listize = filters.size(); -292 for (int i = 0; i < listize; i++) { -293 if (filters.get(i).filterAllRemaining()) { -294 if (operator == Operator.MUST_PASS_ALL) { -295 return true; -296 } -297 } else { -298 if (operator == Operator.MUST_PASS_ONE) { -299 return false; -300 } -301 } -302 } -303 return operator == Operator.MUST_PASS_ONE; -304 } -305 -306 @Override -307 public Cell transformCell(Cell c) throws IOException { -308 if (isEmpty()) { -309 return super.transformCell(c); -310 } -311 if (!CellUtil.equals(c, referenceCell)) { -312 throw new IllegalStateException("Reference Cell: " + this.referenceCell + " does not match: " -313 + c); -314 } -315 return this.transformedCell; -316 } -317 -318 /** -319 * For MUST_PASS_ONE, we cannot make sure that when filter-A in filter list return NEXT_COL then -320 * the next cell passing to filterList will be the first cell in next column, because if filter-B -321 * in filter list return SKIP, then the filter list will return SKIP. In this case, we should pass -322 * the cell following the previous cell, and it's possible that the next cell has the same column -323 * as the previous cell even if filter-A has NEXT_COL returned for the previous cell. So we should -324 * save the previous cell and the return code list when checking previous cell for every filter in -325 * filter list, and verify if currentCell fit the previous return code, if fit then pass the currentCell -326 * to the corresponding filter. (HBASE-17678) -327 */ -328 private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell, int filterIdx) -329 throws IOException { -330 ReturnCode prevCode = this.prevFilterRCList.get(filterIdx); -331 if (prevCell == null || prevCode == null) { -332 return true; -333 } -334 switch (prevCode) { -335 case INCLUDE: -336 case SKIP: -337 return true; -338 case SEEK_NEXT_USING_HINT: -339 Cell nextHintCell = getNextCellHint(prevCell); -340 return nextHintCell == null -341 || CellComparator.COMPARATOR.compare(currentCell, nextHintCell) >= 0; -342 case NEXT_COL: -343 case INCLUDE_AND_NEXT_COL: -344 return !CellUtil.matchingRowColumn(prevCell, currentCell); -345 case NEXT_ROW: -346 case INCLUDE_AND_SEEK_NEXT_ROW: -347 return !CellUtil.matchingRows(prevCell, currentCell); -348 default: -349 throw new IllegalStateException("Received code is not valid."); -350 } -351 } -352 -353 @Override -354 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="SF_SWITCH_FALLTHROUGH", -355 justification="Intentional") -356 public ReturnCode filterKeyValue(Cell c) throws IOException { -357 if (isEmpty()) { -358 return ReturnCode.INCLUDE; -359 } -360 this.referenceCell = c; -361 -362 // Accumulates successive transformation of every filter that includes the Cell: -363 Cell transformed = c; +025import java.util.HashSet; +026import java.util.List; +027import java.util.Set; +028 +029import org.apache.hadoop.hbase.Cell; +030import org.apache.hadoop.hbase.CellComparator; +031import org.apache.hadoop.hbase.CellUtil; +032import org.apache.hadoop.hbase.KeyValueUtil; +033import org.apache.hadoop.hbase.classification.InterfaceAudience; +034import org.apache.hadoop.hbase.exceptions.DeserializationException; +035import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +036import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos; +037import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException; +038 +039/** +040 * Implementation of {@link Filter} that represents an ordered List of Filters +041 * which will be evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL} +042 * (<code>AND</code>) or {@link Operator#MUST_PASS_ONE} (<code>OR</code>). +043 * Since you can use Filter Lists as children of Filter Lists, you can create a +044 * hierarchy of filters to be evaluated. +045 * +046 * +047 * {@link Operator#MUST_PASS_ALL} evaluates lazily: evaluation stops as soon as one filter does +048 * not include the KeyValue. +049 * +050 * +051 * {@link Operator#MUST_PASS_ONE} evaluates non-lazily: all filters are always evaluated. +052 * +053 * +054 * Defaults to {@link Operator#MUST_PASS_ALL}. +055 */ +056@InterfaceAudience.Public +057final public class FilterList extends FilterBase { +058 /** set operator */ +059 @InterfaceAudience.Public +060 public static enum Operator { +061 /** !AND */ +062 MUST_PASS_ALL, +063 /** !OR */ +064 MUST_PASS_ONE +065 } +066 +067 private static final int MAX_LOG_FILTERS = 5; +068 private Operator operator = Operator.MUST_PASS_ALL; +069 private final List<Filter> filters; +070 private Set<Filter> seekHintFilter = new HashSet<>(); +071 +072 /** +073 * Save previous return code and previous cell for every filter in filter list. For MUST_PASS_ONE, +074 * we use the previous return code to decide whether we should pass current cell encountered to +075 * the filter. For MUST_PASS_ALL, the two list are meaningless. +076 */ +077 private List<ReturnCode> prevFilterRCList = null; +078 private List<Cell> prevCellList = null; +079 +080 /** Reference Cell used by {@link #transformCell(Cell)} for validation purpose. */ +081 private Cell referenceCell = null; +082 +083 /** +084 * When filtering a given Cell in {@link #filterKeyValue(Cell)}, +085 * this stores the transformed Cell to be returned by {@link #transformCell(Cell)}. +086 * +087 * Individual filters transformation are applied only when the filter includes the Cell. +088 * Transformations are composed in the order specified by {@link #filters}. +089 */ +090 private Cell transformedCell = null; +091 +092 /** +093 * Constructor that takes a set of {@link Filter}s. The default operator +094 * MUST_PASS_ALL is assumed. +095 * All filters are cloned to internal list. +096 * @param rowFilters list of filters +097 */ +098 public FilterList(final List<Filter> rowFilters) { +099 reversed = getReversed(rowFilters, reversed); +100 this.filters = new ArrayList<>(rowFilters); +101 initPrevListForMustPassOne(rowFilters.size()); +102 } +103 +104 /** +105 * Constructor that takes a var arg number of {@link Filter}s. The fefault operator +106 * MUST_PASS_ALL is assumed. +107 * @param rowFilters +108 */ +109 public FilterList(final Filter... rowFilters) { +110 this(Arrays.asList(rowFilters)); +111 } +112 +113 /** +114 * Constructor that takes an operator. +115 * +116 * @param operator Operator to process filter set with. +117 */ +118 public FilterList(final Operator operator) { +119 this.operator = operator; +120 this.filters = new ArrayList<>(); +121 initPrevListForMustPassOne(filters.size()); +122 } +123 +124 /** +125 * Constructor that takes a set of {@link Filter}s and an operator. +126 * +127 * @param operator Operator to process filter set with. +128 * @param rowFilters Set of row filters. +129 */ +130 public FilterList(final Operator operator, final List<Filter> rowFilters) { +131 this(rowFilters); +132 this.operator = operator; +133 initPrevListForMustPassOne(rowFilters.size()); +134 } +135 +136 /** +137 * Constructor that takes a var arg number of {@link Filter}s and an operator. +138 * +139 * @param operator Operator to process filter set with. +140 * @param rowFilters Filters to use +141 */ +142 public FilterList(final Operator operator, final Filter... rowFilters) { +143 this(rowFilters); +144 this.operator = operator; +145 initPrevListForMustPassOne(rowFilters.length); +146 } +147 +148 public void initPrevListForMustPassOne(int size) { +149 if (operator == Operator.MUST_PASS_ONE) { +150 if (this.prevFilterRCList == null) { +151 prevFilterRCList = new ArrayList<>(Collections.nCopies(size, null)); +152 } +153 if (this.prevCellList == null) { +154 prevCellList = new ArrayList<>(Collections.nCopies(size, null)); +155 } +156 } +157 } +158 +159 +160 /** +161 * Get the operator. +162 * +163 * @return operator +164 */ +165 public Operator getOperator() { +166 return operator; +167 } +168 +169 /** +170 * Get the filters. +171 * +172 * @return filters +173 */ +174 public List<Filter> getFilters() { +175 return filters; +176 } +177 +178 public int size() { +179 return filters.size(); +180 } +181 +182 private boolean isEmpty() { +183 return filters.isEmpty(); +184 } +185 +186 private static boolean getReversed(List<Filter> rowFilters, boolean defaultValue) { +187 boolean rval = defaultValue; +188 boolean isFirst = true; +189 for (Filter f : rowFilters) { +190 if (isFirst) { +191 rval = f.isReversed(); +192 isFirst = false; +193 continue; +194 } +195 if (rval != f.isReversed()) { +196 throw new IllegalArgumentException("Filters in the list must have the same reversed flag"); +197 } +198 } +199 return rval; +200 } +201 private static void checkReversed(List<Filter> rowFilters, boolean expected) { +202 for (Filter filter : rowFilters) { +203 if (expected != filter.isReversed()) { +204 throw new IllegalArgumentException( +205 "Filters in the list must have the same reversed flag, expected=" +206 + expected); +207 } +208 } +209 } +210 +211 public void addFilter(List<Filter> filters) { +212 checkReversed(filters, isReversed()); +213 this.filters.addAll(filters); +214 if (operator == Operator.MUST_PASS_ONE) { +215 this.prevFilterRCList.addAll(Collections.nCopies(filters.size(), null)); +216 this.prevCellList.addAll(Collections.nCopies(filters.size(), null)); +217 } +218 } +219 +220 /** +221 * Add a filter. +222 * +223 * @param filter another filter +224 */ +225 public void addFilter(Filter filter) { +226 addFilter(Collections.singletonList(filter)); +227 } +228 +229 @Override +230 public void reset() throws IOException { +231 int listize = filters.size(); +232 for (int i = 0; i < listize; i++) { +233 filters.get(i).reset(); +234 if (operator == Operator.MUST_PASS_ONE) { +235 prevFilterRCList.set(i, null); +236 prevCellList.set(i, null); +237 } +238 } +239 seekHintFilter.clear(); +240 } +241 +242 @Override +243 public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException { +244 if (isEmpty()) { +245 return super.filterRowKey(rowKey, offset, length); +246 } +247 boolean flag = this.operator == Operator.MUST_PASS_ONE; +248 int listize = filters.size(); +249 for (int i = 0; i < listize; i++) { +250 Filter filter = filters.get(i); +251 if (this.operator == Operator.MUST_PASS_ALL) { +252 if (filter.filterAllRemaining() || +253 filter.filterRowKey(rowKey, offset, length)) { +254 flag = true; +255 } +256 } else if (this.operator == Operator.MUST_PASS_ONE) { +257 if (!filter.filterAllRemaining() && +258 !filter.filterRowKey(rowKey, offset, length)) { +259 flag = false; +260 } +261 } +262 } +263 return flag; +264 } +265 +266 @Override +267 public boolean filterRowKey(Cell firstRowCell) throws IOException { +268 if (isEmpty()) { +269 return super.filterRowKey(firstRowCell); +270 } +271 boolean flag = this.operator == Operator.MUST_PASS_ONE; +272 int listize = filters.size(); +273 for (int i = 0; i < listize; i++) { +274 Filter filter = filters.get(i); +275 if (this.operator == Operator.MUST_PASS_ALL) { +276 if (filter.filterAllRemaining() || filter.filterRowKey(firstRowCell)) { +277 flag = true; +278 } +279 } else if (this.operator == Operator.MUST_PASS_ONE) { +280 if (!filter.filterAllRemaining() && !filter.filterRowKey(firstRowCell)) { +281 flag = false; +282 } +283 } +284 } +285 return flag; +286 } +287 +288 @Override +289 public boolean filterAllRemaining() throws IOException { +290 if (isEmpty()) { +291 return super.filterAllRemaining(); +292 } +293 int listize = filters.size(); +294 for (int i = 0; i < listize; i++) { +295 if (filters.get(i).filterAllRemaining()) { +296 if (operator == Operator.MUST_PASS_ALL) { +297 return true; +298 } +299 } else { +300 if (operator == Operator.MUST_PASS_ONE) { +301 return false; +302 } +303 } +304 } +305 return operator == Operator.MUST_PASS_ONE; +306 } +307 +308 @Override +309 public Cell transformCell(Cell c) throws IOException { +310 if (isEmpty()) { +311 return super.transformCell(c); +312 } +313 if (!CellUtil.equals(c, referenceCell)) { +314 throw new IllegalStateException("Reference Cell: " + this.referenceCell + " does not match: " +315 + c); +316 } +317 return this.transformedCell; +318 } +319 +320 /** +321 * For MUST_PASS_ONE, we cannot make sure that when filter-A in filter list return NEXT_COL then +322 * the next cell passing to filterList will be the first cell in next column, because if filter-B +323 * in filter list return SKIP, then the filter list will return SKIP. In this case, we should pass +324 * the cell following the previous cell, and it's possible that the next cell has the same column +325 * as the previous cell even if filter-A has NEXT_COL returned for the previous cell. So we should +326 * save the previous cell and the return code list when checking previous cell for every filter in +327 * filter list, and verify if currentCell fit the previous return code, if fit then pass the currentCell +328 * to the corresponding filter. (HBASE-17678) +329 */ +330 private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell, int filterIdx) +331 throws IOException { +332 ReturnCode prevCode = this.prevFilterRCList.get(filterIdx); +333 if (prevCell == null || prevCode == null) { +334 return true; +335 } +336 switch (prevCode) { +337 case INCLUDE: +338 case SKIP: +339 return true; +340 case SEEK_NEXT_USING_HINT: +341 Cell nextHintCell = getNextCellHint(prevCell); +342 return nextHintCell == null +343 || CellComparator.COMPARATOR.compare(currentCell, nextHintCell) >= 0; +344 case NEXT_COL: +345 case INCLUDE_AND_NEXT_COL: +346 return !CellUtil.matchingRowColumn(prevCell, currentCell); +347 case NEXT_ROW: +348 case INCLUDE_AND_SEEK_NEXT_ROW: +349 return !CellUtil.matchingRows(prevCell, currentCell); +350 default: +351 throw new IllegalStateException("Received code is not valid."); +352 } +353 } +354 +355 @Override +356 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="SF_SWITCH_FALLTHROUGH", +357 justification="Intentional") +358 public ReturnCode filterKeyValue(Cell c) throws IOException { +359 if (isEmpty()) { +360 return ReturnCode.INCLUDE; +361 } +362 this.referenceCell = c; +363 seekHintFilter.clear(); 364 -365 ReturnCode rc = operator == Operator.MUST_PASS_ONE? -366 ReturnCode.SKIP: ReturnCode.INCLUDE; -367 int listize = filters.size(); -368 /* -369 * When all filters in a MUST_PASS_ONE FilterList return a SEEK_USING_NEXT_HINT code, -370 * we should return SEEK_NEXT_USING_HINT from the FilterList to utilize the lowest seek value. -371 * -372 * The following variable tracks whether any of the Filters returns ReturnCode other than -373 * SEEK_NEXT_USING_HINT for MUST_PASS_ONE FilterList, in which case the optimization would -374 * be skipped. -375 */ -376 boolean seenNonHintReturnCode = false; -377 for (int i = 0; i < listize; i++) { -378 Filter filter = filters.get(i); -379 if (operator == Operator.MUST_PASS_ALL) { -380 if (filter.filterAllRemaining()) { -381 return ReturnCode.NEXT_ROW; -382 } -383 ReturnCode code = filter.filterKeyValue(c); -384 switch (code) { -385 // Override INCLUDE and continue to evaluate. -386 case INCLUDE_AND_NEXT_COL: -387 rc = ReturnCode.INCLUDE_AND_NEXT_COL; // FindBugs SF_SWITCH_FALLTHROUGH -388 case INCLUDE: -389 transformed = filter.transformCell(transformed); -390 continue; -391 case SEEK_NEXT_USING_HINT: -392 seekHintFilter = filter; -393 return code; -394 default: -395 return code; -396 } -397 } else if (operator == Operator.MUST_PASS_ONE) { -398 Cell prevCell = this.prevCellList.get(i); -399 if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) { -400 seenNonHintReturnCode = true; -401 continue; -402 } -403 -404 ReturnCode localRC = filter.filterKeyValue(c); -405 // Update previous cell and return code we encountered. -406 prevFilterRCList.set(i, localRC); -407 if (c == null || localRC == ReturnCode.INCLUDE || localRC == ReturnCode.SKIP) { -408 // If previous return code is INCLUDE or SKIP, we should always pass the next cell to the -409 // corresponding sub-filter(need not test shouldPassCurrentCellToFilter() method), So we -410 // need not save current cell to prevCellList for saving heap memory. -411 prevCellList.set(i, null); -412 } else { -413 prevCellList.set(i, KeyValueUtil.toNewKeyCell(c)); -414 } -415 -416 if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) { -417 seenNonHintReturnCode = true; -418 } -419 switch (localRC) { -420 case INCLUDE: -421 if (rc != ReturnCode.INCLUDE_AND_NEXT_COL) { -422 rc = ReturnCode.INCLUDE; -423 } -424 transformed = filter.transformCell(transformed); -425 break; -426 case INCLUDE_AND_NEXT_COL: -427 rc = ReturnCode.INCLUDE_AND_NEXT_COL; -428 transformed = filter.transformCell(transformed); -429 // must continue here to evaluate all filters +365 // Accumulates successive transformation of every filter that includes the Cell: +366 Cell transformed = c; +367 +368 ReturnCode rc = operator == Operator.MUST_PASS_ONE? +369 ReturnCode.SKIP: ReturnCode.INCLUDE; +370 int listize = filters.size(); +371 /* +372 * When all filters in a MUST_PASS_ONE FilterList return a SEEK_USING_NEXT_HINT code, +373 * we should return SEEK_NEXT_USING_HINT from the FilterList to utilize the lowest seek value. +374 * +375 * The following variable tracks whether any of the Filters returns ReturnCode other than +376 * SEEK_NEXT_USING_HINT for MUST_PASS_ONE FilterList, in which case the optimization would +377 * be skipped. +378 */ +379 boolean seenNonHintReturnCode = false; +380 for (int i = 0; i < listize; i++) { +381 Filter filter = filters.get(i); +382 if (operator == Operator.MUST_PASS_ALL) { +383 if (filter.filterAllRemaining()) { +384 return ReturnCode.NEXT_ROW; +385 } +386 ReturnCode code = filter.filterKeyValue(c); +387 switch (code) { +388 // Override INCLUDE and continue to evaluate. +389 case INCLUDE_AND_NEXT_COL: +390 rc = ReturnCode.INCLUDE_AND_NEXT_COL; // FindBugs SF_SWITCH_FALLTHROUGH +391 case INCLUDE: +392 transformed = filter.transformCell(transformed); +393 continue; +394 case SEEK_NEXT_USING_HINT: +395 seekHintFilter.add(filter); +396 continue; +397 default: +398 if (seekHintFilter.isEmpty()) { +399 return code; +400 } +401 } +402 } else if (operator == Operator.MUST_PASS_ONE) { +403 Cell prevCell = this.prevCellList.get(i); +404 if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) { +405 seenNonHintReturnCode = true; +406 continue; +407 } +408 +409 ReturnCode localRC = filter.filterKeyValue(c); +410 // Update previous cell and return code we encountered. +411 prevFilterRCList.set(i, localRC); +412 if (c == null || localRC == ReturnCode.INCLUDE || localRC == ReturnCode.SKIP) { +413 // If previous return code is INCLUDE or SKIP, we should always pass the next cell to the +414 // corresponding sub-filter(need not test shouldPassCurrentCellToFilter() method), So we +415 // need not save current cell to prevCellList for saving heap memory. +416 prevCellList.set(i, null); +417 } else { +418 prevCellList.set(i, KeyValueUtil.toNewKeyCell(c)); +419 } +420 +421 if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) { +422 seenNonHintReturnCode = true; +423 } +424 switch (localRC) { +425 case INCLUDE: +426 if (rc != ReturnCode.INCLUDE_AND_NEXT_COL) { +427 rc = ReturnCode.INCLUDE; +428 } +429 transformed = filter.transformCell(transformed); 430 break; -431 case NEXT_ROW: -432 break; -433 case SKIP: -434 break; -435 case NEXT_COL: -436 break; -437 case SEEK_NEXT_USING_HINT: -438 break; -439 default: -440 throw new IllegalStateException("Received code is not valid."); -441 } -442 } -443 } -444 -445 // Save the transformed Cell for transform(): -446 this.transformedCell = transformed; -447 -448 /* -449 * The seenNonHintReturnCode flag is intended only for Operator.MUST_PASS_ONE branch. -450 * If we have seen non SEEK_NEXT_USING_HINT ReturnCode, respect that ReturnCode. -451 */ -452 if (operator == Operator.MUST_PASS_ONE && !seenNonHintReturnCode) { -453 return ReturnCode.SEEK_NEXT_USING_HINT; -454 } -455 return rc; -456 } -457 -458 /** -459 * Filters that never filter by modifying the returned List of Cells can -460 * inherit this implementation that does nothing. -461 * -462 * {@inheritDoc} -463 */ -464 @Override -465 public void filterRowCells(List<Cell> cells) throws IOException { -466 int listize = filters.size(); -467 for (int i = 0; i < listize; i++) { -468 filters.get(i).filterRowCells(cells); -469 } -470 } -471 -472 @Override -473 public boolean hasFilterRow() { -474 int listize = filters.size(); -475 for (int i = 0; i < listize; i++) { -476 if (filters.get(i).hasFilterRow()) { -477 return true; -478 } -479 } -480 return false; -481 } -482 -483 @Override -484 public boolean filterRow() throws IOException { -485 if (isEmpty()) { -486 return super.filterRow(); -487 } -488 int listize = filters.size(); -489 for (int i = 0; i < listize; i++) { -490 Filter filter = filters.get(i); -491 if (operator == Operator.MUST_PASS_ALL) { -492 if (filter.filterRow()) { -493 return true; -494 } -495 } else if (operator == Operator.MUST_PASS_ONE) { -496 if (!filter.filterRow()) { -497 return false; -498 } -499 } -500 } -501 return operator == Operator.MUST_PASS_ONE; -502 } -503 -504 /** -505 * @return The filter serialized using pb -506 */ -507 public byte[] toByteArray() throws IOException { -508 FilterProtos.FilterList.Builder builder = -509 FilterProtos.FilterList.newBuilder(); -510 builder.setOperator(FilterProtos.FilterList.Operator.valueOf(operator.name())); -511 int listize = filters.size(); -512 for (int i = 0; i < listize; i++) { -513 builder.addFilters(ProtobufUtil.toFilter(filters.get(i))); -514 } -515 return builder.build().toByteArray(); -516 } -517 -518 /** -519 * @param pbBytes A pb serialized {@link FilterList} instance -520 * @return An instance of {@link FilterList} made from <code>bytes</code> -521 * @throws DeserializationException -522 * @see #toByteArray -523 */ -524 public static FilterList parseFrom(final byte [] pbBytes) -525 throws DeserializationException { -526 FilterProtos.FilterList proto; -527 try { -528 proto = FilterProtos.FilterList.parseFrom(pbBytes); -529 } catch (InvalidProtocolBufferException e) { -530 throw new DeserializationException(e); -531 } -532 -533 List<Filter> rowFilters = new ArrayList<>(proto.getFiltersCount()); -534 try { -535 List<FilterProtos.Filter> filtersList = proto.getFiltersList(); -536 int listSize = filtersList.size(); -537 for (int i = 0; i < listSize; i++) { -538 rowFilters.add(ProtobufUtil.toFilter(filtersList.get(i))); -539 } -540 } catch (IOException ioe) { -541 throw new DeserializationException(ioe); -542 } -543 return new FilterList(Operator.valueOf(proto.getOperator().name()),rowFilters); -544 } -545 -546 /** -547 * @param other -548 * @return true if and only if the fields of the filter that are serialized -549 * are equal to the corresponding fields in other. Used for testing. -550 */ -551 boolean areSerializedFieldsEqual(Filter other) { -552 if (other == this) return true; -553 if (!(other instanceof FilterList)) return false; +431 case INCLUDE_AND_NEXT_COL: +432 rc = ReturnCode.INCLUDE_AND_NEXT_COL; +433 transformed = filter.transformCell(transformed); +434 // must continue here to evaluate all filters +435 break; +436 case NEXT_ROW: