Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5640C200C41 for ; Fri, 10 Mar 2017 06:43:20 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 54D20160B87; Fri, 10 Mar 2017 05:43:20 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 78DF6160B75 for ; Fri, 10 Mar 2017 06:43:19 +0100 (CET) Received: (qmail 27922 invoked by uid 500); 10 Mar 2017 05:43:18 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 27909 invoked by uid 99); 10 Mar 2017 05:43:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Mar 2017 05:43:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 70869DFF47; Fri, 10 Mar 2017 05:43:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: anoopsamjohn@apache.org To: commits@hbase.apache.org Date: Fri, 10 Mar 2017 05:43:18 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] hbase git commit: HBASE-17338 Treat Cell data size under global memstore heap size only when that Cell can not be copied to MSLAB. archived-at: Fri, 10 Mar 2017 05:43:20 -0000 Repository: hbase Updated Branches: refs/heads/master c63a87117 -> ab5970773 http://git-wip-us.apache.org/repos/asf/hbase/blob/ab597077/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java index de01d4a..6e5cbf8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java @@ -193,7 +193,8 @@ public class TestPerColumnFamilyFlush { // We should have cleared out only CF1, since we chose the flush thresholds // and number of puts accordingly. - assertEquals(MemstoreSize.EMPTY_SIZE, cf1MemstoreSize); + assertEquals(0, cf1MemstoreSize.getDataSize()); + assertEquals(0, cf1MemstoreSize.getHeapSize()); // Nothing should have happened to CF2, ... assertEquals(cf2MemstoreSize, oldCF2MemstoreSize); // ... or CF3 @@ -229,8 +230,10 @@ public class TestPerColumnFamilyFlush { .getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes()); // CF1 and CF2, both should be absent. - assertEquals(MemstoreSize.EMPTY_SIZE, cf1MemstoreSize); - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSize); + assertEquals(0, cf1MemstoreSize.getDataSize()); + assertEquals(0, cf1MemstoreSize.getHeapSize()); + assertEquals(0, cf2MemstoreSize.getDataSize()); + assertEquals(0, cf2MemstoreSize.getHeapSize()); // CF3 shouldn't have been touched. assertEquals(cf3MemstoreSize, oldCF3MemstoreSize); assertEquals(totalMemstoreSize, cf3MemstoreSize.getDataSize()); @@ -310,9 +313,12 @@ public class TestPerColumnFamilyFlush { region.getWAL().getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes()); // Everything should have been cleared - assertEquals(MemstoreSize.EMPTY_SIZE, cf1MemstoreSize); - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSize); - assertEquals(MemstoreSize.EMPTY_SIZE, cf3MemstoreSize); + assertEquals(0, cf1MemstoreSize.getDataSize()); + assertEquals(0, cf1MemstoreSize.getHeapSize()); + assertEquals(0, cf2MemstoreSize.getDataSize()); + assertEquals(0, cf2MemstoreSize.getHeapSize()); + assertEquals(0, cf3MemstoreSize.getDataSize()); + assertEquals(0, cf3MemstoreSize.getHeapSize()); assertEquals(0, totalMemstoreSize); assertEquals(HConstants.NO_SEQNUM, smallestSeqInRegionCurrentMemstore); HBaseTestingUtility.closeRegionAndWAL(region); http://git-wip-us.apache.org/repos/asf/hbase/blob/ab597077/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java index 4c09810..0d339b1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java @@ -245,7 +245,8 @@ public class TestStore { Assert.assertEquals(kvSize2, size); flushStore(store, id++); size = store.memstore.getFlushableSize(); - Assert.assertEquals(MemstoreSize.EMPTY_SIZE, size); + assertEquals(0, size.getDataSize()); + assertEquals(0, size.getHeapSize()); return null; } }); http://git-wip-us.apache.org/repos/asf/hbase/blob/ab597077/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java index f2da776..57eee30 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java @@ -232,15 +232,16 @@ public class TestWalAndCompactingMemStoreFlush { // CF1 was flushed to memory, but there is nothing to compact, and CF1 was flattened assertTrue(cf1MemstoreSizePhaseII.getDataSize() == cf1MemstoreSizePhaseI.getDataSize()); - assertTrue(cf1MemstoreSizePhaseII.getHeapOverhead() < cf1MemstoreSizePhaseI.getHeapOverhead()); + assertTrue(cf1MemstoreSizePhaseII.getHeapSize() < cf1MemstoreSizePhaseI.getHeapSize()); // CF2 should become empty - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); + assertEquals(0, cf2MemstoreSizePhaseII.getDataSize()); + assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize()); // verify that CF3 was flushed to memory and was compacted (this is approximation check) assertTrue(cf3MemstoreSizePhaseI.getDataSize() > cf3MemstoreSizePhaseII.getDataSize()); assertTrue( - cf3MemstoreSizePhaseI.getHeapOverhead() / 2 > cf3MemstoreSizePhaseII.getHeapOverhead()); + cf3MemstoreSizePhaseI.getHeapSize() / 2 > cf3MemstoreSizePhaseII.getHeapSize()); // Now the smallest LSN in the region should be the same as the smallest // LSN in the memstore of CF1. @@ -295,7 +296,8 @@ public class TestWalAndCompactingMemStoreFlush { // CF1's pipeline component (inserted before first flush) should be flushed to disk // CF2 should be flushed to disk assertTrue(cf1MemstoreSizePhaseIII.getDataSize() > cf1MemstoreSizePhaseIV.getDataSize()); - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseIV); + assertEquals(0, cf2MemstoreSizePhaseIV.getDataSize()); + assertEquals(0, cf2MemstoreSizePhaseIV.getHeapSize()); // CF3 shouldn't have been touched. assertEquals(cf3MemstoreSizePhaseIV, cf3MemstoreSizePhaseII); @@ -318,9 +320,12 @@ public class TestWalAndCompactingMemStoreFlush { long smallestSeqInRegionCurrentMemstorePhaseV = getWAL(region) .getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes()); - assertEquals(MemstoreSize.EMPTY_SIZE , cf1MemstoreSizePhaseV); - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseV); - assertEquals(MemstoreSize.EMPTY_SIZE, cf3MemstoreSizePhaseV); + assertEquals(0, cf1MemstoreSizePhaseV.getDataSize()); + assertEquals(0, cf1MemstoreSizePhaseV.getHeapSize()); + assertEquals(0, cf2MemstoreSizePhaseV.getDataSize()); + assertEquals(0, cf2MemstoreSizePhaseV.getHeapSize()); + assertEquals(0, cf3MemstoreSizePhaseV.getDataSize()); + assertEquals(0, cf3MemstoreSizePhaseV.getHeapSize()); // What happens when we hit the memstore limit, but we are not able to find // any Column Family above the threshold? @@ -463,14 +468,14 @@ public class TestWalAndCompactingMemStoreFlush { /* PHASE II - validation */ // CF1 was flushed to memory, should be flattened and take less space assertEquals(cf1MemstoreSizePhaseII.getDataSize() , cf1MemstoreSizePhaseI.getDataSize()); - assertTrue(cf1MemstoreSizePhaseII.getHeapOverhead() < cf1MemstoreSizePhaseI.getHeapOverhead()); + assertTrue(cf1MemstoreSizePhaseII.getHeapSize() < cf1MemstoreSizePhaseI.getHeapSize()); // CF2 should become empty - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); + assertEquals(0, cf2MemstoreSizePhaseII.getDataSize()); + assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize()); // verify that CF3 was flushed to memory and was not compacted (this is an approximation check) // if compacted CF# should be at least twice less because its every key was duplicated assertEquals(cf3MemstoreSizePhaseII.getDataSize() , cf3MemstoreSizePhaseI.getDataSize()); - assertTrue( - cf3MemstoreSizePhaseI.getHeapOverhead() / 2 < cf3MemstoreSizePhaseII.getHeapOverhead()); + assertTrue(cf3MemstoreSizePhaseI.getHeapSize() / 2 < cf3MemstoreSizePhaseII.getHeapSize()); // Now the smallest LSN in the region should be the same as the smallest // LSN in the memstore of CF1. @@ -533,7 +538,8 @@ public class TestWalAndCompactingMemStoreFlush { // CF1's biggest pipeline component (inserted before first flush) should be flushed to disk // CF2 should remain empty assertTrue(cf1MemstoreSizePhaseIII.getDataSize() > cf1MemstoreSizePhaseIV.getDataSize()); - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseIV); + assertEquals(0, cf2MemstoreSizePhaseIV.getDataSize()); + assertEquals(0, cf2MemstoreSizePhaseIV.getHeapSize()); // CF3 shouldn't have been touched. assertEquals(cf3MemstoreSizePhaseIV, cf3MemstoreSizePhaseII); // the smallest LSN of CF3 shouldn't change @@ -561,9 +567,12 @@ public class TestWalAndCompactingMemStoreFlush { /*------------------------------------------------------------------------------*/ /* PHASE V - validation */ - assertEquals(MemstoreSize.EMPTY_SIZE, cf1MemstoreSizePhaseV); - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseV); - assertEquals(MemstoreSize.EMPTY_SIZE, cf3MemstoreSizePhaseV); + assertEquals(0, cf1MemstoreSizePhaseV.getDataSize()); + assertEquals(0, cf1MemstoreSizePhaseV.getHeapSize()); + assertEquals(0, cf2MemstoreSizePhaseV.getDataSize()); + assertEquals(0, cf2MemstoreSizePhaseV.getHeapSize()); + assertEquals(0, cf3MemstoreSizePhaseV.getDataSize()); + assertEquals(0, cf3MemstoreSizePhaseV.getHeapSize()); // The total memstores size should be empty assertEquals(0, totalMemstoreSizePhaseV); // Because there is nothing in any memstore the WAL's LSN should be -1 @@ -683,7 +692,8 @@ public class TestWalAndCompactingMemStoreFlush { long smallestSeqCF3PhaseII = region.getOldestSeqIdOfStore(FAMILY3); // CF2 should have been cleared - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); + assertEquals(0, cf2MemstoreSizePhaseII.getDataSize()); + assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize()); String s = "\n\n----------------------------------\n" + "Upon initial insert and flush, LSN of CF1 is:" @@ -816,7 +826,8 @@ public class TestWalAndCompactingMemStoreFlush { long smallestSeqCF3PhaseII = region.getOldestSeqIdOfStore(FAMILY3); // CF2 should have been cleared - assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); + assertEquals(0, cf2MemstoreSizePhaseII.getDataSize()); + assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize()); // Add same entries to compact them later for (int i = 1; i <= 1200; i++) {