hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [1/7] hbase git commit: HBASE-18043 Institute a hard limit for individual cell size that cannot be overridden by clients
Date Tue, 16 May 2017 01:36:08 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 0a4528225 -> d8ef49506
  refs/heads/branch-1.1 26cb211e1 -> 2cff94c40
  refs/heads/branch-1.2 14ab4a9c4 -> 719ab32cc
  refs/heads/branch-1.3 36ebe05fc -> d18eb62d8
  refs/heads/master 841bb0065 -> 37650775a


HBASE-18043 Institute a hard limit for individual cell size that cannot be overridden by clients


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/29222669
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/29222669
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/29222669

Branch: refs/heads/branch-1
Commit: 292226690ac842be4513366633f31bba4fa62d34
Parents: 0a45282
Author: Andrew Purtell <apurtell@apache.org>
Authored: Mon May 15 18:03:24 2017 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Mon May 15 18:03:24 2017 -0700

----------------------------------------------------------------------
 .../src/main/resources/hbase-default.xml        |  9 +++++
 .../hadoop/hbase/regionserver/HRegion.java      | 11 +++++-
 .../hbase/regionserver/RSRpcServices.java       | 35 ++++++++++++++++--
 .../hadoop/hbase/client/TestFromClientSide.java | 37 ++++++++++++++++++++
 4 files changed, 89 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/29222669/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 2a74a6a..c571289 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -555,6 +555,15 @@ possible configurations would overwhelm and obscure the important.
     or less disables the check.</description>
   </property>
   <property>
+    <name>hbase.server.keyvalue.maxsize</name>
+    <value>10485760</value>
+    <description>Maximum allowed size of an individual cell, inclusive of value and
all key
+    components. A value of 0 or less disables the check.
+    The default value is 10MB.
+    This is a safety setting to protect the server from OOM situations.
+    </description>
+  </property>
+  <property>
     <name>hbase.client.scanner.timeout.period</name>
     <value>60000</value>
     <description>Client scanner lease period in milliseconds.</description>

http://git-wip-us.apache.org/repos/asf/hbase/blob/29222669/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 8e1306f..1c44f6a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -218,6 +218,9 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
   public static final String HREGION_UNASSIGN_FOR_FNFE = "hbase.hregion.unassign.for.fnfe";
   public static final boolean DEFAULT_HREGION_UNASSIGN_FOR_FNFE = true;
 
+  public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize";
+  public static final int DEFAULT_MAX_CELL_SIZE = 10485760;
+
   /**
    * Longest time we'll wait on a sequenceid.
    * Sequenceid comes up out of the WAL subsystem. WAL subsystem can go bad or a test might
use
@@ -333,6 +336,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
   // purge timeout, when a RPC call will be terminated by the RPC engine.
   final long maxBusyWaitDuration;
 
+  // Max cell size. If nonzero, the maximum allowed size for any given cell
+  // in bytes
+  final long maxCellSize;
+
   // negative number indicates infinite timeout
   static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L;
   final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool();
@@ -816,6 +823,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
           conf.getBoolean(HConstants.ENABLE_CLIENT_BACKPRESSURE,
               HConstants.DEFAULT_ENABLE_CLIENT_BACKPRESSURE);
 
+    this.maxCellSize = conf.getLong(HBASE_MAX_CELL_SIZE_KEY, DEFAULT_MAX_CELL_SIZE);
+
     boolean unassignForFNFE =
         conf.getBoolean(HREGION_UNASSIGN_FOR_FNFE, DEFAULT_HREGION_UNASSIGN_FOR_FNFE);
     if (unassignForFNFE) {
@@ -8137,7 +8146,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
       ClassSize.OBJECT +
       ClassSize.ARRAY +
       46 * ClassSize.REFERENCE + 3 * Bytes.SIZEOF_INT +
-      (14 * Bytes.SIZEOF_LONG) +
+      (15 * Bytes.SIZEOF_LONG) +
       5 * Bytes.SIZEOF_BOOLEAN);
 
   // woefully out of date - currently missing:

http://git-wip-us.apache.org/repos/asf/hbase/blob/29222669/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 87ff3d2..9b270c6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -454,7 +454,9 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
       }
       switch (type) {
         case PUT:
-          rm.add(ProtobufUtil.toPut(action.getMutation(), cellScanner));
+          Put put = ProtobufUtil.toPut(action.getMutation(), cellScanner);
+          checkCellSizeLimit(region, put);
+          rm.add(put);
           break;
         case DELETE:
           rm.add(ProtobufUtil.toDelete(action.getMutation(), cellScanner));
@@ -506,7 +508,9 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
       }
       switch (type) {
         case PUT:
-          rm.add(ProtobufUtil.toPut(action.getMutation(), cellScanner));
+          Put put = ProtobufUtil.toPut(action.getMutation(), cellScanner);
+          checkCellSizeLimit(region, put);
+          rm.add(put);
           break;
         case DELETE:
           rm.add(ProtobufUtil.toDelete(action.getMutation(), cellScanner));
@@ -541,6 +545,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
       throws IOException {
     long before = EnvironmentEdgeManager.currentTime();
     Append append = ProtobufUtil.toAppend(mutation, cellScanner);
+    checkCellSizeLimit(region, append);
     quota.addMutation(append);
     Result r = null;
     if (region.getCoprocessorHost() != null) {
@@ -591,6 +596,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
       throws IOException {
     long before = EnvironmentEdgeManager.currentTime();
     Increment increment = ProtobufUtil.toIncrement(mutation, cellScanner);
+    checkCellSizeLimit(region, increment);
     quota.addMutation(increment);
     Result r = null;
     if (region.getCoprocessorHost() != null) {
@@ -791,6 +797,26 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
     return cellsToReturn;
   }
 
+  private void checkCellSizeLimit(final Region region, final Mutation m) throws IOException
{
+    if (!(region instanceof HRegion)) {
+      return;
+    }
+    HRegion r = (HRegion)region;
+    if (r.maxCellSize > 0) {
+      CellScanner cells = m.cellScanner();
+      while (cells.advance()) {
+        int size = CellUtil.estimatedSerializedSizeOf(cells.current());
+        if (size > r.maxCellSize) {
+          String msg = "Cell with size " + size + " exceeds limit of " + r.maxCellSize +
" bytes";
+          if (LOG.isDebugEnabled()) {
+            LOG.debug(msg);
+          }
+          throw new DoNotRetryIOException(msg);
+        }
+      }
+    }
+  }
+
   /**
    * Execute a list of Put/Delete mutations.
    *
@@ -826,15 +852,18 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
         }
         mutationActionMap.put(mutation, action);
         mArray[i++] = mutation;
+        checkCellSizeLimit(region, mutation);
         quota.addMutation(mutation);
       }
 
       if (!region.getRegionInfo().isMetaTable()) {
         regionServer.cacheFlusher.reclaimMemStoreMemory();
       }
+
       // HBASE-17924
       // sort to improve lock efficiency
       Arrays.sort(mArray);
+
       OperationStatus[] codes = region.batchMutate(mArray, HConstants.NO_NONCE,
         HConstants.NO_NONCE);
       for (i = 0; i < codes.length; i++) {
@@ -2358,6 +2387,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
         break;
       case PUT:
         Put put = ProtobufUtil.toPut(mutation, cellScanner);
+        checkCellSizeLimit(region, put);
         quota.addMutation(put);
         if (request.hasCondition()) {
           Condition condition = request.getCondition();
@@ -2387,6 +2417,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
         break;
       case DELETE:
         Delete delete = ProtobufUtil.toDelete(mutation, cellScanner);
+        checkCellSizeLimit(region, delete);
         quota.addMutation(delete);
         if (request.hasCondition()) {
           Condition condition = request.getCondition();

http://git-wip-us.apache.org/repos/asf/hbase/blob/29222669/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 6b831b1..e21e51a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -101,6 +101,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.Mut
 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
 import org.apache.hadoop.hbase.regionserver.DelegatingKeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
@@ -6462,4 +6463,40 @@ public class TestFromClientSide {
       .getNumberOfCachedRegionLocations(htd.getTableName());
     assertEquals(results.size(), number);
   }
+
+  @Test
+  public void testCellSizeLimit() throws IOException {
+    final TableName tableName = TableName.valueOf("testCellSizeLimit");
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.setConfiguration(HRegion.HBASE_MAX_CELL_SIZE_KEY, Integer.toString(10 * 1024)); //
10K
+    HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
+    htd.addFamily(fam);
+    Admin admin = TEST_UTIL.getHBaseAdmin();
+    admin.createTable(htd);
+    // Will succeed
+    try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
+      t.put(new Put(ROW).add(FAMILY, QUALIFIER, Bytes.toBytes(0L)));
+      t.increment(new Increment(ROW).addColumn(FAMILY, QUALIFIER, 1L));
+    }
+    // Will succeed
+    try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
+      t.put(new Put(ROW).add(FAMILY, QUALIFIER, new byte[9*1024]));
+    }
+    // Will fail
+    try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
+      try {
+        t.put(new Put(ROW).add(FAMILY, QUALIFIER, new byte[10 * 1024]));
+        fail("Oversize cell failed to trigger exception");
+      } catch (IOException e) {
+        // expected
+      }
+      try {
+        t.append(new Append(ROW).add(FAMILY, QUALIFIER, new byte[10 * 1024]));
+        fail("Oversize cell failed to trigger exception");
+      } catch (IOException e) {
+        // expected
+      }
+    }
+  }
+
 }


Mime
View raw message