hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anasta...@apache.org
Subject [1/2] hbase git commit: HBASE-18010: CellChunkMap integration into CompactingMemStore, merge to Release 2.0
Date Sun, 01 Oct 2017 08:01:47 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-2 7ee44a820 -> 95405fbd8


http://git-wip-us.apache.org/repos/asf/hbase/blob/95405fbd/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index 1dc943f..f4cd742 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -788,7 +788,8 @@ public class TestHRegionReplayEvents {
     HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
     long newFlushableSize = store.getFlushableSize().getHeapSize();
     if (droppableMemstore) {
-      assertTrue(newFlushableSize == 0); // assert that the memstore is dropped
+      // assert that the memstore is dropped
+      assertTrue(newFlushableSize == MutableSegment.DEEP_OVERHEAD);
     } else {
       assertTrue(newFlushableSize > 0); // assert that the memstore is not dropped
     }
@@ -876,9 +877,9 @@ public class TestHRegionReplayEvents {
     for (HStore s : secondaryRegion.getStores()) {
       assertEquals(expectedStoreFileCount, s.getStorefilesCount());
     }
-    HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
+    Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
     long newFlushableSize = store.getFlushableSize().getHeapSize();
-    assertTrue(newFlushableSize == 0);
+    assertTrue(newFlushableSize == MutableSegment.DEEP_OVERHEAD);
 
     // assert that the region memstore is empty
     long newRegionMemstoreSize = secondaryRegion.getMemstoreSize();

http://git-wip-us.apache.org/repos/asf/hbase/blob/95405fbd/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
index 9b78f41..815166b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
@@ -253,7 +253,9 @@ public class TestHStore {
         assertEquals(0, size.getDataSize());
         LOG.info("Adding some data");
         MemstoreSize kvSize = new MemstoreSize();
-        store.add(new KeyValue(row, family, qf1, 1, (byte[])null), kvSize);
+        store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize);
+        // add the heap size of active (mutable) segment
+        kvSize.incMemstoreSize(0, MutableSegment.DEEP_OVERHEAD);
         size = store.memstore.getFlushableSize();
         assertEquals(kvSize, size);
         // Flush.  Bug #1 from HBASE-10466.  Make sure size calculation on failed flush is
right.
@@ -264,10 +266,14 @@ public class TestHStore {
         } catch (IOException ioe) {
           assertTrue(ioe.getMessage().contains("Fault injected"));
         }
+        // due to snapshot, change mutable to immutable segment
+        kvSize.incMemstoreSize(0,
+            CSLMImmutableSegment.DEEP_OVERHEAD_CSLM-MutableSegment.DEEP_OVERHEAD);
         size = store.memstore.getFlushableSize();
         assertEquals(kvSize, size);
         MemstoreSize kvSize2 = new MemstoreSize();
         store.add(new KeyValue(row, family, qf2, 2, (byte[])null), kvSize2);
+        kvSize2.incMemstoreSize(0, MutableSegment.DEEP_OVERHEAD);
         // Even though we add a new kv, we expect the flushable size to be 'same' since we
have
         // not yet cleared the snapshot -- the above flush failed.
         assertEquals(kvSize, size);
@@ -279,7 +285,7 @@ public class TestHStore {
         flushStore(store, id++);
         size = store.memstore.getFlushableSize();
         assertEquals(0, size.getDataSize());
-        assertEquals(0, size.getHeapSize());
+        assertEquals(MutableSegment.DEEP_OVERHEAD, size.getHeapSize());
         return null;
       }
     });

http://git-wip-us.apache.org/repos/asf/hbase/blob/95405fbd/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
index e1a52a6..a0d953e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
@@ -194,7 +194,7 @@ public class TestPerColumnFamilyFlush {
     // We should have cleared out only CF1, since we chose the flush thresholds
     // and number of puts accordingly.
     assertEquals(0, cf1MemstoreSize.getDataSize());
-    assertEquals(0, cf1MemstoreSize.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf1MemstoreSize.getHeapSize());
     // Nothing should have happened to CF2, ...
     assertEquals(cf2MemstoreSize, oldCF2MemstoreSize);
     // ... or CF3
@@ -231,9 +231,9 @@ public class TestPerColumnFamilyFlush {
 
     // CF1 and CF2, both should be absent.
     assertEquals(0, cf1MemstoreSize.getDataSize());
-    assertEquals(0, cf1MemstoreSize.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf1MemstoreSize.getHeapSize());
     assertEquals(0, cf2MemstoreSize.getDataSize());
-    assertEquals(0, cf2MemstoreSize.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf2MemstoreSize.getHeapSize());
     // CF3 shouldn't have been touched.
     assertEquals(cf3MemstoreSize, oldCF3MemstoreSize);
     assertEquals(totalMemstoreSize, cf3MemstoreSize.getDataSize());
@@ -314,11 +314,11 @@ public class TestPerColumnFamilyFlush {
 
     // Everything should have been cleared
     assertEquals(0, cf1MemstoreSize.getDataSize());
-    assertEquals(0, cf1MemstoreSize.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf1MemstoreSize.getHeapSize());
     assertEquals(0, cf2MemstoreSize.getDataSize());
-    assertEquals(0, cf2MemstoreSize.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf2MemstoreSize.getHeapSize());
     assertEquals(0, cf3MemstoreSize.getDataSize());
-    assertEquals(0, cf3MemstoreSize.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf3MemstoreSize.getHeapSize());
     assertEquals(0, totalMemstoreSize);
     assertEquals(HConstants.NO_SEQNUM, smallestSeqInRegionCurrentMemstore);
     HBaseTestingUtility.closeRegionAndWAL(region);
@@ -525,9 +525,12 @@ public class TestPerColumnFamilyFlush {
       });
       LOG.info("Finished waiting on flush after too many WALs...");
       // Individual families should have been flushed.
-      assertEquals(0, desiredRegion.getStore(FAMILY1).getMemStoreSize().getHeapSize());
-      assertEquals(0, desiredRegion.getStore(FAMILY2).getMemStoreSize().getHeapSize());
-      assertEquals(0, desiredRegion.getStore(FAMILY3).getMemStoreSize().getHeapSize());
+      assertEquals(MutableSegment.DEEP_OVERHEAD,
+        desiredRegion.getStore(FAMILY1).getMemStoreSize().getHeapSize());
+      assertEquals(MutableSegment.DEEP_OVERHEAD,
+        desiredRegion.getStore(FAMILY2).getMemStoreSize().getHeapSize());
+      assertEquals(MutableSegment.DEEP_OVERHEAD,
+        desiredRegion.getStore(FAMILY3).getMemStoreSize().getHeapSize());
       // let WAL cleanOldLogs
       assertNull(getWAL(desiredRegion).rollWriter(true));
       assertTrue(getNumRolledLogFiles(desiredRegion) < maxLogs);

http://git-wip-us.apache.org/repos/asf/hbase/blob/95405fbd/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
index 60afa3d..6dc1a1b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
@@ -241,7 +241,7 @@ public class TestWalAndCompactingMemStoreFlush {
 
     // CF2 should become empty
     assertEquals(0, cf2MemstoreSizePhaseII.getDataSize());
-    assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf2MemstoreSizePhaseII.getHeapSize());
 
     // verify that CF3 was flushed to memory and was compacted (this is approximation check)
     assertTrue(cf3MemstoreSizePhaseI.getDataSize() > cf3MemstoreSizePhaseII.getDataSize());
@@ -302,7 +302,7 @@ public class TestWalAndCompactingMemStoreFlush {
     // CF2 should be flushed to disk
     assertTrue(cf1MemstoreSizePhaseIII.getDataSize() > cf1MemstoreSizePhaseIV.getDataSize());
     assertEquals(0, cf2MemstoreSizePhaseIV.getDataSize());
-    assertEquals(0, cf2MemstoreSizePhaseIV.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf2MemstoreSizePhaseIV.getHeapSize());
 
     // CF3 shouldn't have been touched.
     assertEquals(cf3MemstoreSizePhaseIV, cf3MemstoreSizePhaseII);
@@ -326,11 +326,11 @@ public class TestWalAndCompactingMemStoreFlush {
         .getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes());
 
     assertEquals(0, cf1MemstoreSizePhaseV.getDataSize());
-    assertEquals(0, cf1MemstoreSizePhaseV.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf1MemstoreSizePhaseV.getHeapSize());
     assertEquals(0, cf2MemstoreSizePhaseV.getDataSize());
-    assertEquals(0, cf2MemstoreSizePhaseV.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf2MemstoreSizePhaseV.getHeapSize());
     assertEquals(0, cf3MemstoreSizePhaseV.getDataSize());
-    assertEquals(0, cf3MemstoreSizePhaseV.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf3MemstoreSizePhaseV.getHeapSize());
 
     // What happens when we hit the memstore limit, but we are not able to find
     // any Column Family above the threshold?
@@ -476,7 +476,7 @@ public class TestWalAndCompactingMemStoreFlush {
     assertTrue(cf1MemstoreSizePhaseII.getHeapSize() < cf1MemstoreSizePhaseI.getHeapSize());
     // CF2 should become empty
     assertEquals(0, cf2MemstoreSizePhaseII.getDataSize());
-    assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf2MemstoreSizePhaseII.getHeapSize());
     // verify that CF3 was flushed to memory and was not compacted (this is an approximation
check)
     // if compacted CF# should be at least twice less because its every key was duplicated
     assertEquals(cf3MemstoreSizePhaseII.getDataSize() , cf3MemstoreSizePhaseI.getDataSize());
@@ -544,7 +544,7 @@ public class TestWalAndCompactingMemStoreFlush {
     // CF2 should remain empty
     assertTrue(cf1MemstoreSizePhaseIII.getDataSize() > cf1MemstoreSizePhaseIV.getDataSize());
     assertEquals(0, cf2MemstoreSizePhaseIV.getDataSize());
-    assertEquals(0, cf2MemstoreSizePhaseIV.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf2MemstoreSizePhaseIV.getHeapSize());
     // CF3 shouldn't have been touched.
     assertEquals(cf3MemstoreSizePhaseIV, cf3MemstoreSizePhaseII);
     // the smallest LSN of CF3 shouldn't change
@@ -573,11 +573,11 @@ public class TestWalAndCompactingMemStoreFlush {
     /*------------------------------------------------------------------------------*/
     /* PHASE V - validation */
     assertEquals(0, cf1MemstoreSizePhaseV.getDataSize());
-    assertEquals(0, cf1MemstoreSizePhaseV.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf1MemstoreSizePhaseV.getHeapSize());
     assertEquals(0, cf2MemstoreSizePhaseV.getDataSize());
-    assertEquals(0, cf2MemstoreSizePhaseV.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf2MemstoreSizePhaseV.getHeapSize());
     assertEquals(0, cf3MemstoreSizePhaseV.getDataSize());
-    assertEquals(0, cf3MemstoreSizePhaseV.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf3MemstoreSizePhaseV.getHeapSize());
     // The total memstores size should be empty
     assertEquals(0, totalMemstoreSizePhaseV);
     // Because there is nothing in any memstore the WAL's LSN should be -1
@@ -699,7 +699,7 @@ public class TestWalAndCompactingMemStoreFlush {
 
     // CF2 should have been cleared
     assertEquals(0, cf2MemstoreSizePhaseII.getDataSize());
-    assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize());
+    assertEquals(MutableSegment.DEEP_OVERHEAD, cf2MemstoreSizePhaseII.getHeapSize());
 
     String s = "\n\n----------------------------------\n"
         + "Upon initial insert and flush, LSN of CF1 is:"
@@ -875,9 +875,13 @@ public class TestWalAndCompactingMemStoreFlush {
     MemstoreSize cf2MemstoreSizePhaseIV = region.getStore(FAMILY2).getMemStoreSize();
 
     assertEquals(2*cf1MemstoreSizePhaseI.getDataSize(), cf1MemstoreSizePhaseIV.getDataSize());
+    // the decrease in the heap size due to usage of CellArrayMap instead of CSLM
+    // should be the same in flattening and in merge (first and second in-memory-flush)
+    // but in phase 1 we do not yet have immutable segment
     assertEquals(
         cf1MemstoreSizePhaseI.getHeapSize() - cf1MemstoreSizePhaseII.getHeapSize(),
-        cf1MemstoreSizePhaseIII.getHeapSize() - cf1MemstoreSizePhaseIV.getHeapSize());
+        cf1MemstoreSizePhaseIII.getHeapSize() - cf1MemstoreSizePhaseIV.getHeapSize()
+            - CellArrayImmutableSegment.DEEP_OVERHEAD_CAM);
     assertEquals(3, // active, one in pipeline, snapshot
         ((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore).getSegments().size());
     // CF2 should have been cleared


Mime
View raw message