hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1567269 - in /hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver: HRegion.java MemStore.java Store.java
Date Tue, 11 Feb 2014 18:28:37 GMT
Author: liyin
Date: Tue Feb 11 18:28:36 2014
New Revision: 1567269

URL: http://svn.apache.org/r1567269
Log:
[master] Let HRegion get correct flushable size from each Store

Author: fan

Summary:
Current calculation of flushable size in HRegion is wrong when previous flush attempt fails.
The bug messes up HRegion's total memstore size, which leads to a very serious data loss for
all data in memstore(s) when region is closing.
This fix corrects the size change after flush succeeds.

Test Plan: TBD

Reviewers: aaiyer, liyintang, jiqingt

Reviewed By: jiqingt

CC: HBase Eng@, hbase-dev@, manukranthk

Differential Revision: https://phabricator.fb.com/D1158429

Task ID: 3621385

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1567269&r1=1567268&r2=1567269&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Tue Feb 11 18:28:36 2014
@@ -1506,7 +1506,7 @@ public class HRegion implements HeapSize
       for (Store store : storesToFlush) {
         LOG.debug("Flushing Column Family: " + store.getColumnFamilyName() +
                   " which was occupying " +
-                  StringUtils.humanReadableInt(store.getMemStoreSize()) +
+                  StringUtils.humanReadableInt(store.getFlushableMemstoreSize()) +
                   " of memstore.");
       }
     }
@@ -1561,7 +1561,7 @@ public class HRegion implements HeapSize
         sequenceId = myseqid;
       }
       for (Store s : storesToFlush) {
-        totalMemstoreSizeOfFlushableStores += s.getMemStoreSize();
+        totalMemstoreSizeOfFlushableStores += s.getFlushableMemstoreSize();
         storeFlushers.add(s.getStoreFlusher(sequenceId));
       }
 
@@ -2621,8 +2621,8 @@ public class HRegion implements HeapSize
    * @return true if the size of the store is above the flush threshold for column families
    */
   private boolean shouldFlushStore(Store store) {
-    return (store.getMemStoreSize() > this.columnfamilyMemstoreFlushSize) ?
-            true : false;
+    return store.getSnapshotSize() > 0 ||
+        store.getMemStoreSize() > this.columnfamilyMemstoreFlushSize;
   }
 
   /**

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java?rev=1567269&r1=1567268&r2=1567269&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
Tue Feb 11 18:28:36 2014
@@ -84,6 +84,7 @@ public class MemStore implements HeapSiz
 
   // Used to track own heapSize
   final AtomicLong size;
+  private long snapshotSize;
 
   private AtomicLong numDeletesInKvSet;
   private AtomicLong numDeletesInSnapshot;
@@ -122,6 +123,7 @@ public class MemStore implements HeapSiz
     timeRangeTracker = new TimeRangeTracker();
     snapshotTimeRangeTracker = new TimeRangeTracker();
     this.size = new AtomicLong(DEEP_OVERHEAD);
+    this.snapshotSize = 0;
     this.numDeletesInKvSet = new AtomicLong(0);
     this.numDeletesInSnapshot = new AtomicLong(0);
 
@@ -201,6 +203,7 @@ public class MemStore implements HeapSiz
           }
 
           // Reset heap to not include any keys
+          this.snapshotSize = keySize();
           this.size.set(DEEP_OVERHEAD);
           this.numDeletesInSnapshot = numDeletesInKvSet;
           this.numDeletesInKvSet.set(0);
@@ -248,6 +251,7 @@ public class MemStore implements HeapSiz
         tmpAllocator = this.snapshotAllocator;
         this.snapshotAllocator = null;
       }
+      this.snapshotSize = 0;
     } finally {
       this.lock.writeLock().unlock();
     }
@@ -883,6 +887,20 @@ public class MemStore implements HeapSiz
     return heapSize() - DEEP_OVERHEAD;
   }
 
+  public long getSnapshotSize() {
+    return snapshotSize;
+  }
+
+  /**
+   * Flush will first clear out the data in snapshot if any. If snapshot is
+   * empty, current keyvalue set will be flushed.
+   *
+   * @return size of data that is going to be flushed
+   */
+  public long getFlushableSize() {
+    return snapshotSize > 0 ? snapshotSize : keySize();
+  }
+
   /**
    * Code to help figure if our approximation of object heap sizes is close
    * enough.  See hbase-900.  Fills memstores then waits so user can heap

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1567269&r1=1567268&r2=1567269&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue
Feb 11 18:28:36 2014
@@ -1980,6 +1980,14 @@ public class Store extends SchemaConfigu
     return this.memstore.keySize();
   }
 
+  public long getSnapshotSize() {
+    return this.memstore.getSnapshotSize();
+  }
+
+  public long getFlushableMemstoreSize() {
+    return this.memstore.getFlushableSize();
+  }
+
   /**
    * A helper function to get the smallest LSN in the mestore.
    * @return



Mime
View raw message