hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject svn commit: r1173335 - in /hbase/trunk: CHANGES.txt src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
Date Tue, 20 Sep 2011 20:21:27 GMT
Author: tedyu
Date: Tue Sep 20 20:21:27 2011
New Revision: 1173335

URL: http://svn.apache.org/viewvc?rev=1173335&view=rev
Log:
HBASE-3421  Very wide rows -- 30M plus -- cause us OOME (Nate Putnam)

Modified:
    hbase/trunk/CHANGES.txt
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java

Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1173335&r1=1173334&r2=1173335&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Tue Sep 20 20:21:27 2011
@@ -288,6 +288,7 @@ Release 0.91.0 - Unreleased
    HBASE-4428  Two methods in CacheTestUtils don't call setDaemon() on the threads
    HBASE-4400  .META. getting stuck if RS hosting it is dead and znode state is in
                RS_ZK_REGION_OPENED (Ramkrishna)
+   HBASE-3421  Very wide rows -- 30M plus -- cause us OOME (Nate Putnam)
 
   IMPROVEMENTS
    HBASE-3290  Max Compaction Size (Nicolas Spiegelberg via Stack)  

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1173335&r1=1173334&r2=1173335&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Sep 20 20:21:27
2011
@@ -118,6 +118,7 @@ public class Store implements HeapSize {
   private final String storeNameStr;
   private final boolean inMemory;
   private CompactionProgress progress;
+  private final int compactionKVMax;
 
   /*
    * List of store files inside this store. This is an immutable list that
@@ -210,6 +211,7 @@ public class Store implements HeapSize {
     this.maxCompactSize
       = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
     this.compactRatio = conf.getFloat("hbase.hstore.compaction.ratio", 1.2F);
+    this.compactionKVMax = conf.getInt("hbase.hstore.compaction.kv.max", 10);
 
     if (Store.closeCheckInterval == 0) {
       Store.closeCheckInterval = conf.getInt(
@@ -1146,7 +1148,8 @@ public class Store implements HeapSize {
         // since scanner.next() can return 'false' but still be delivering data,
         // we have to use a do/while loop.
         ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
-        while (scanner.next(kvs)) {
+        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
+        while (scanner.next(kvs,this.compactionKVMax)) {
           if (writer == null && !kvs.isEmpty()) {
             writer = createWriterInTmp(maxKeyCount,
               this.compactionCompression);



Mime
View raw message