hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhang...@apache.org
Subject hbase git commit: HBASE-17599 Use mayHaveMoreCellsInRow instead of isPartial
Date Thu, 09 Feb 2017 07:47:50 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 f61b840a3 -> e9abe0762


HBASE-17599 Use mayHaveMoreCellsInRow instead of isPartial


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e9abe076
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e9abe076
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e9abe076

Branch: refs/heads/branch-1
Commit: e9abe07629bd698b8c727903a0826cde2bf6d1d5
Parents: f61b840
Author: zhangduo <zhangduo@apache.org>
Authored: Thu Feb 9 10:51:16 2017 +0800
Committer: zhangduo <zhangduo@apache.org>
Committed: Thu Feb 9 15:38:23 2017 +0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/client/Result.java  | 47 +++++++++++++-------
 .../hadoop/hbase/protobuf/ProtobufUtil.java     |  2 +-
 .../hbase/protobuf/generated/ClientProtos.java  | 24 ++++++----
 hbase-protocol/src/main/protobuf/Client.proto   |  3 +-
 .../hadoop/hbase/regionserver/HRegion.java      | 34 +++++++-------
 .../hbase/regionserver/RSRpcServices.java       | 11 ++---
 .../hbase/regionserver/ScannerContext.java      | 23 +++-------
 7 files changed, 81 insertions(+), 63 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/e9abe076/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
index 3074d3e..d27518e 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -83,15 +83,10 @@ public class Result implements CellScannable, CellScanner {
   private boolean stale = false;
 
   /**
-   * Partial results do not contain the full row's worth of cells. The result had to be returned
in
-   * parts because the size of the cells in the row exceeded the RPC result size on the server.
-   * Partial results must be combined client side with results representing the remainder
of the
-   * row's cells to form the complete result. Partial results and RPC result size allow us
to avoid
-   * OOME on the server when servicing requests for large rows. The Scan configuration used
to
-   * control the result size on the server is {@link Scan#setMaxResultSize(long)} and the
default
-   * value can be seen here: {@link HConstants#DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE}
+   * See {@link #mayHaveMoreCellsInRow()}. And please notice that, The client side implementation
+   * should also check for row key change to determine if a Result is the last one for a
row.
    */
-  private boolean partial = false;
+  private boolean mayHaveMoreCellsInRow = false;
   // We're not using java serialization.  Transient here is just a marker to say
   // that this is where we cache row if we're ever asked for it.
   private transient byte [] row = null;
@@ -185,19 +180,20 @@ public class Result implements CellScannable, CellScanner {
     return create(cells, exists, stale, false);
   }
 
-  public static Result create(Cell[] cells, Boolean exists, boolean stale, boolean partial)
{
-    if (exists != null){
-      return new Result(null, exists, stale, partial);
+  public static Result create(Cell[] cells, Boolean exists, boolean stale,
+      boolean mayHaveMoreCellsInRow) {
+    if (exists != null) {
+      return new Result(null, exists, stale, mayHaveMoreCellsInRow);
     }
-    return new Result(cells, null, stale, partial);
+    return new Result(cells, null, stale, mayHaveMoreCellsInRow);
   }
 
   /** Private ctor. Use {@link #create(Cell[])}. */
-  private Result(Cell[] cells, Boolean exists, boolean stale, boolean partial) {
+  private Result(Cell[] cells, Boolean exists, boolean stale, boolean mayHaveMoreCellsInRow)
{
     this.cells = cells;
     this.exists = exists;
     this.stale = stale;
-    this.partial = partial;
+    this.mayHaveMoreCellsInRow = mayHaveMoreCellsInRow;
     this.readonly = false;
   }
 
@@ -890,7 +886,7 @@ public class Result implements CellScannable, CellScanner {
         // Result1: -1- -2- (2 cells, size limit reached, mark as partial)
         // Result2: -3- -4- (2 cells, size limit reached, mark as partial)
         // Result3: -5- (1 cell, size limit NOT reached, NOT marked as partial)
-        if (i != (partialResults.size() - 1) && !r.isPartial()) {
+        if (i != (partialResults.size() - 1) && !r.mayHaveMoreCellsInRow()) {
           throw new IOException(
               "Cannot form complete result. Result is missing partial flag. " +
                   "Partial Results: " + partialResults);
@@ -977,9 +973,28 @@ public class Result implements CellScannable, CellScanner {
    * for a row and should be combined with a result representing the remaining cells in that
row to
    * form a complete (non-partial) result.
    * @return Whether or not the result is a partial result
+   * @deprecated the word 'partial' ambiguous, use {@link #mayHaveMoreCellsInRow()} instead.
+   *             Deprecated since 1.4.0.
+   * @see #mayHaveMoreCellsInRow()
    */
+  @Deprecated
   public boolean isPartial() {
-    return partial;
+    return mayHaveMoreCellsInRow;
+  }
+
+  /**
+   * For scanning large rows, the RS may choose to return the cells chunk by chunk to prevent
OOM.
+   * This flag is used to tell you if the current Result is the last one of the current row.
False
+   * means this Result is the last one. True means there may still be more cells for the
current
+   * row. Notice that, 'may' have, not must have. This is because we may reach the size or
time
+   * limit just at the last cell of row at RS, so we do not know if it is the last one.
+   * <p>
+   * The Scan configuration used to control the result size on the server is
+   * {@link Scan#setMaxResultSize(long)} and the default value can be seen here:
+   * {@link HConstants#DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE}
+   */
+  public boolean mayHaveMoreCellsInRow() {
+    return mayHaveMoreCellsInRow;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/e9abe076/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 896185f..b198d53 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -1423,7 +1423,7 @@ public final class ProtobufUtil {
     }
 
     builder.setStale(result.isStale());
-    builder.setPartial(result.isPartial());
+    builder.setPartial(result.mayHaveMoreCellsInRow());
 
     return builder.build();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/e9abe076/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
index cefdd38..9133c19 100644
--- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
@@ -4636,7 +4636,8 @@ public final class ClientProtos {
      * Whether or not the entire result could be returned. Results will be split when
      * the RPC chunk size limit is reached. Partial results contain only a subset of the
      * cells for a row and must be combined with a result containing the remaining cells
-     * to form a complete result
+     * to form a complete result. The equivalent flag in o.a.h.h.client.Result is
+     * mayHaveMoreCellsInRow.
      * </pre>
      */
     boolean hasPartial();
@@ -4647,7 +4648,8 @@ public final class ClientProtos {
      * Whether or not the entire result could be returned. Results will be split when
      * the RPC chunk size limit is reached. Partial results contain only a subset of the
      * cells for a row and must be combined with a result containing the remaining cells
-     * to form a complete result
+     * to form a complete result. The equivalent flag in o.a.h.h.client.Result is
+     * mayHaveMoreCellsInRow.
      * </pre>
      */
     boolean getPartial();
@@ -4929,7 +4931,8 @@ public final class ClientProtos {
      * Whether or not the entire result could be returned. Results will be split when
      * the RPC chunk size limit is reached. Partial results contain only a subset of the
      * cells for a row and must be combined with a result containing the remaining cells
-     * to form a complete result
+     * to form a complete result. The equivalent flag in o.a.h.h.client.Result is
+     * mayHaveMoreCellsInRow.
      * </pre>
      */
     public boolean hasPartial() {
@@ -4942,7 +4945,8 @@ public final class ClientProtos {
      * Whether or not the entire result could be returned. Results will be split when
      * the RPC chunk size limit is reached. Partial results contain only a subset of the
      * cells for a row and must be combined with a result containing the remaining cells
-     * to form a complete result
+     * to form a complete result. The equivalent flag in o.a.h.h.client.Result is
+     * mayHaveMoreCellsInRow.
      * </pre>
      */
     public boolean getPartial() {
@@ -5858,7 +5862,8 @@ public final class ClientProtos {
        * Whether or not the entire result could be returned. Results will be split when
        * the RPC chunk size limit is reached. Partial results contain only a subset of the
        * cells for a row and must be combined with a result containing the remaining cells
-       * to form a complete result
+       * to form a complete result. The equivalent flag in o.a.h.h.client.Result is
+       * mayHaveMoreCellsInRow.
        * </pre>
        */
       public boolean hasPartial() {
@@ -5871,7 +5876,8 @@ public final class ClientProtos {
        * Whether or not the entire result could be returned. Results will be split when
        * the RPC chunk size limit is reached. Partial results contain only a subset of the
        * cells for a row and must be combined with a result containing the remaining cells
-       * to form a complete result
+       * to form a complete result. The equivalent flag in o.a.h.h.client.Result is
+       * mayHaveMoreCellsInRow.
        * </pre>
        */
       public boolean getPartial() {
@@ -5884,7 +5890,8 @@ public final class ClientProtos {
        * Whether or not the entire result could be returned. Results will be split when
        * the RPC chunk size limit is reached. Partial results contain only a subset of the
        * cells for a row and must be combined with a result containing the remaining cells
-       * to form a complete result
+       * to form a complete result. The equivalent flag in o.a.h.h.client.Result is
+       * mayHaveMoreCellsInRow.
        * </pre>
        */
       public Builder setPartial(boolean value) {
@@ -5900,7 +5907,8 @@ public final class ClientProtos {
        * Whether or not the entire result could be returned. Results will be split when
        * the RPC chunk size limit is reached. Partial results contain only a subset of the
        * cells for a row and must be combined with a result containing the remaining cells
-       * to form a complete result
+       * to form a complete result. The equivalent flag in o.a.h.h.client.Result is
+       * mayHaveMoreCellsInRow.
        * </pre>
        */
       public Builder clearPartial() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/e9abe076/hbase-protocol/src/main/protobuf/Client.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/protobuf/Client.proto b/hbase-protocol/src/main/protobuf/Client.proto
index 4fb64bd..d55b97a 100644
--- a/hbase-protocol/src/main/protobuf/Client.proto
+++ b/hbase-protocol/src/main/protobuf/Client.proto
@@ -112,7 +112,8 @@ message Result {
   // Whether or not the entire result could be returned. Results will be split when
   // the RPC chunk size limit is reached. Partial results contain only a subset of the
   // cells for a row and must be combined with a result containing the remaining cells
-  // to form a complete result
+  // to form a complete result. The equivalent flag in o.a.h.h.client.Result is
+  // mayHaveMoreCellsInRow.
   optional bool partial = 5 [default = false];
 }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e9abe076/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index a32fe65..1f648f7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -18,6 +18,20 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.Closeables;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+import com.google.protobuf.RpcCallback;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.Service;
+import com.google.protobuf.TextFormat;
+
 import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -65,6 +79,8 @@ import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import javax.jws.soap.SOAPBinding.Use;
+
 import org.apache.commons.lang.RandomStringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -189,20 +205,6 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.htrace.Trace;
 import org.apache.htrace.TraceScope;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.io.Closeables;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.Descriptors;
-import com.google.protobuf.Message;
-import com.google.protobuf.RpcCallback;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.Service;
-import com.google.protobuf.TextFormat;
-
 @InterfaceAudience.Private
 public class HRegion implements HeapSize, PropagatingConfigurationObserver, Region {
   private static final Log LOG = LogFactory.getLog(HRegion.class);
@@ -6054,7 +6056,9 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
       // If the size limit was reached it means a partial Result is being returned. Returning
a
       // partial Result means that we should not reset the filters; filters should only be
reset in
       // between rows
-      if (!scannerContext.midRowResultFormed()) resetFilters();
+      if (!scannerContext.mayHaveMoreCellsInRow()) {
+        resetFilters();
+      }
 
       if (isFilterDoneInternal()) {
         moreValues = false;

http://git-wip-us.apache.org/repos/asf/hbase/blob/e9abe076/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index f13d8b7..e14356a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -408,11 +408,13 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
   private void addResults(final ScanResponse.Builder builder, final List<Result> results,
       final PayloadCarryingRpcController controller, boolean isDefaultRegion) {
     builder.setStale(!isDefaultRegion);
-    if (results == null || results.isEmpty()) return;
+    if (results.isEmpty()) {
+      return;
+    }
     if (isClientCellBlockSupport()) {
       for (Result res : results) {
         builder.addCellsPerResult(res.size());
-        builder.addPartialFlagPerResult(res.isPartial());
+        builder.addPartialFlagPerResult(res.mayHaveMoreCellsInRow());
       }
       controller.setCellScanner(CellUtil.createCellScanner(results));
     } else {
@@ -2636,8 +2638,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
           moreRows = scanner.nextRaw(values, scannerContext);
 
           if (!values.isEmpty()) {
-            final boolean partial = scannerContext.partialResultFormed();
-            Result r = Result.create(values, null, stale, partial);
+            Result r = Result.create(values, null, stale, scannerContext.mayHaveMoreCellsInRow());
             lastBlock.setValue(addSize(context, r, lastBlock.getValue()));
             results.add(r);
             i++;
@@ -2856,7 +2857,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
         // is false. Can remove the isEmpty check after we get rid of the old implementation.
         moreResults = false;
       } else if (limitOfRows > 0 && results.size() >= limitOfRows &&
-          !results.get(results.size() - 1).isPartial()) {
+          !results.get(results.size() - 1).mayHaveMoreCellsInRow()) {
         // if we have reached the limit of rows
         moreResults = false;
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/e9abe076/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
index bffcdf6..4e61f1b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.regionserver;
 
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -51,7 +49,6 @@ import org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics;
 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
 @InterfaceStability.Evolving
 public class ScannerContext {
-  private static final Log LOG = LogFactory.getLog(ScannerContext.class);
 
   /**
    * Two sets of the same fields. One for the limits, another for the progress towards those
limits
@@ -221,21 +218,13 @@ public class ScannerContext {
   }
 
   /**
-   * @return true when a partial result is formed. A partial result is formed when a limit
is
-   *         reached in the middle of a row.
+   * @return true when we may have more cells for the current row. This usually because we
have
+   *         reached a limit in the middle of a row
    */
-  boolean partialResultFormed() {
-    return scannerState == NextState.SIZE_LIMIT_REACHED_MID_ROW
-        || scannerState == NextState.TIME_LIMIT_REACHED_MID_ROW;
-  }
-
-  /**
-   * @return true when a mid-row result is formed.
-   */
-  boolean midRowResultFormed() {
-    return scannerState == NextState.SIZE_LIMIT_REACHED_MID_ROW
-        || scannerState == NextState.TIME_LIMIT_REACHED_MID_ROW
-        || scannerState == NextState.BATCH_LIMIT_REACHED;
+  boolean mayHaveMoreCellsInRow() {
+    return scannerState == NextState.SIZE_LIMIT_REACHED_MID_ROW ||
+        scannerState == NextState.TIME_LIMIT_REACHED_MID_ROW ||
+        scannerState == NextState.BATCH_LIMIT_REACHED;
   }
 
   /**


Mime
View raw message