Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 2609E200D2E for ; Tue, 31 Oct 2017 16:16:54 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 24928160BFE; Tue, 31 Oct 2017 15:16:54 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E777D160BFA for ; Tue, 31 Oct 2017 16:16:52 +0100 (CET) Received: (qmail 36017 invoked by uid 500); 31 Oct 2017 15:16:51 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 35794 invoked by uid 99); 31 Oct 2017 15:16:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Oct 2017 15:16:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D170FDFD45; Tue, 31 Oct 2017 15:16:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Tue, 31 Oct 2017 15:16:53 -0000 Message-Id: <2f6d34a9b2874f659ff8c0542776bc85@git.apache.org> In-Reply-To: <1125b91ba4af4b08b647bebab99dc5b6@git.apache.org> References: <1125b91ba4af4b08b647bebab99dc5b6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [05/51] [partial] hbase-site git commit: Published site at . archived-at: Tue, 31 Oct 2017 15:16:54 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/35decbe4/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.html index d28cf30..c97c8f0 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.html @@ -102,7 +102,7 @@ 094 } 095 096 // test 1 bucket -097 long totalCellsLen = addRowsByKeys(memstore, keys1); +097 long totalCellsLen = addRowsByKeysDataSize(memstore, keys1); 098 long cellBeforeFlushSize = cellBeforeFlushSize(); 099 long cellAfterFlushSize = cellAfterFlushSize(); 100 long totalHeapSize = MutableSegment.DEEP_OVERHEAD + 4 * cellBeforeFlushSize; @@ -148,7 +148,7 @@ 140 String[] keys1 = { "A", "A", "B", "C" }; 141 String[] keys2 = { "A", "B", "D" }; 142 -143 long totalCellsLen1 = addRowsByKeys(memstore, keys1); // INSERT 4 +143 long totalCellsLen1 = addRowsByKeysDataSize(memstore, keys1); // INSERT 4 144 long cellBeforeFlushSize = cellBeforeFlushSize(); 145 long cellAfterFlushSize = cellAfterFlushSize(); 146 long totalHeapSize1 = MutableSegment.DEEP_OVERHEAD + 4 * cellBeforeFlushSize; @@ -172,7 +172,7 @@ 164 assertEquals(totalCellsLen1, regionServicesForStores.getMemStoreSize()); 165 assertEquals(totalHeapSize1, ((CompactingMemStore) memstore).heapSize()); 166 -167 long totalCellsLen2 = addRowsByKeys(memstore, keys2); // INSERT 3 (3+3=6) +167 long totalCellsLen2 = addRowsByKeysDataSize(memstore, keys2); // INSERT 3 (3+3=6) 168 long totalHeapSize2 = 3 * cellBeforeFlushSize; 169 assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemStoreSize()); 170 assertEquals(totalHeapSize1 + totalHeapSize2, ((CompactingMemStore) memstore).heapSize()); @@ -210,7 +210,7 @@ 202 String[] keys2 = { "A", "B", "D" }; 203 String[] keys3 = { "D", "B", "B" }; 204 -205 long totalCellsLen1 = addRowsByKeys(memstore, keys1); +205 long totalCellsLen1 = addRowsByKeysDataSize(memstore, keys1); 206 long cellBeforeFlushSize = cellBeforeFlushSize(); 207 long cellAfterFlushSize = cellAfterFlushSize(); 208 long totalHeapSize1 = MutableSegment.DEEP_OVERHEAD + 4 * cellBeforeFlushSize; @@ -231,7 +231,7 @@ 223 assertEquals(totalCellsLen1, regionServicesForStores.getMemStoreSize()); 224 assertEquals(totalHeapSize1, ((CompactingMemStore) memstore).heapSize()); 225 -226 long totalCellsLen2 = addRowsByKeys(memstore, keys2); +226 long totalCellsLen2 = addRowsByKeysDataSize(memstore, keys2); 227 long totalHeapSize2 = 3 * cellBeforeFlushSize; 228 229 assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemStoreSize()); @@ -245,7 +245,7 @@ 237 assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemStoreSize()); 238 assertEquals(totalHeapSize1 + totalHeapSize2, ((CompactingMemStore) memstore).heapSize()); 239 -240 long totalCellsLen3 = addRowsByKeys(memstore, keys3); +240 long totalCellsLen3 = addRowsByKeysDataSize(memstore, keys3); 241 long totalHeapSize3 = 3 * cellBeforeFlushSize; 242 assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3, 243 regionServicesForStores.getMemStoreSize()); @@ -302,7 +302,7 @@ 294 memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, 295 String.valueOf(compactionType)); 296 ((CompactingMemStore)memstore).initiateType(compactionType); -297 addRowsByKeys(memstore, keys1); +297 addRowsByKeysDataSize(memstore, keys1); 298 299 ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline should not compact 300 @@ -311,7 +311,7 @@ 303 } 304 assertEquals(0, memstore.getSnapshot().getCellsCount()); 305 -306 addRowsByKeys(memstore, keys2); // also should only flatten +306 addRowsByKeysDataSize(memstore, keys2); // also should only flatten 307 308 int counter2 = 0; 309 for ( Segment s : memstore.getSegments()) { @@ -330,7 +330,7 @@ 322 } 323 assertEquals(12, counter3); 324 -325 addRowsByKeys(memstore, keys3); +325 addRowsByKeysDataSize(memstore, keys3); 326 327 int counter4 = 0; 328 for ( Segment s : memstore.getSegments()) { @@ -612,49 +612,104 @@ 604 } 605 606 -607 private long addRowsByKeys(final AbstractMemStore hmc, String[] keys) { -608 byte[] fam = Bytes.toBytes("testfamily"); -609 byte[] qf = Bytes.toBytes("testqualifier"); -610 MemStoreSizing memstoreSizing = new MemStoreSizing(); -611 for (int i = 0; i < keys.length; i++) { -612 long timestamp = System.currentTimeMillis(); -613 Threads.sleep(1); // to make sure each kv gets a different ts -614 byte[] row = Bytes.toBytes(keys[i]); -615 byte[] val = Bytes.toBytes(keys[i] + i); -616 KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); -617 hmc.add(kv, memstoreSizing); -618 LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); -619 } -620 regionServicesForStores.addMemStoreSize(memstoreSizing); -621 return memstoreSizing.getDataSize(); -622 } -623 -624 private long cellBeforeFlushSize() { -625 // make one cell -626 byte[] row = Bytes.toBytes("A"); -627 byte[] val = Bytes.toBytes("A" + 0); -628 KeyValue kv = -629 new KeyValue(row, Bytes.toBytes("testfamily"), Bytes.toBytes("testqualifier"), -630 System.currentTimeMillis(), val); -631 return ClassSize.align( -632 ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + KeyValue.FIXED_OVERHEAD + KeyValueUtil.length(kv)); -633 } +607 @Test +608 public void testFlatteningToCellChunkMap() throws IOException { +609 +610 // set memstore to flat into CellChunkMap +611 MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC; +612 memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, +613 String.valueOf(compactionType)); +614 ((CompactingMemStore)memstore).initiateType(compactionType); +615 memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, +616 String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); +617 ((CompactingMemStore)memstore).setIndexType(); +618 int numOfCells = 8; +619 String[] keys1 = { "A", "A", "B", "C", "D", "D", "E", "F" }; //A1, A2, B3, C4, D5, D6, E7, F8 +620 +621 // make one cell +622 byte[] row = Bytes.toBytes(keys1[0]); +623 byte[] val = Bytes.toBytes(keys1[0] + 0); +624 KeyValue kv = +625 new KeyValue(row, Bytes.toBytes("testfamily"), Bytes.toBytes("testqualifier"), +626 System.currentTimeMillis(), val); +627 +628 // test 1 bucket +629 int totalCellsLen = addRowsByKeys(memstore, keys1); +630 long oneCellOnCSLMHeapSize = +631 ClassSize.align( +632 ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + KeyValue.FIXED_OVERHEAD + KeyValueUtil +633 .length(kv)); 634 -635 private long cellAfterFlushSize() { -636 // make one cell -637 byte[] row = Bytes.toBytes("A"); -638 byte[] val = Bytes.toBytes("A" + 0); -639 KeyValue kv = -640 new KeyValue(row, Bytes.toBytes("testfamily"), Bytes.toBytes("testqualifier"), -641 System.currentTimeMillis(), val); -642 -643 return toCellChunkMap ? -644 ClassSize.align( -645 ClassSize.CELL_CHUNK_MAP_ENTRY + KeyValueUtil.length(kv)) : -646 ClassSize.align( -647 ClassSize.CELL_ARRAY_MAP_ENTRY + KeyValue.FIXED_OVERHEAD + KeyValueUtil.length(kv)); -648 } -649} +635 long totalHeapSize = numOfCells * oneCellOnCSLMHeapSize + MutableSegment.DEEP_OVERHEAD; +636 assertEquals(totalCellsLen, regionServicesForStores.getMemStoreSize()); +637 assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); +638 +639 ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and flatten +640 assertEquals(0, memstore.getSnapshot().getCellsCount()); +641 // One cell is duplicated, but it shouldn't be compacted because we are in BASIC mode. +642 // totalCellsLen should remain the same +643 long oneCellOnCCMHeapSize = +644 ClassSize.CELL_CHUNK_MAP_ENTRY + ClassSize.align(KeyValueUtil.length(kv)); +645 totalHeapSize = MutableSegment.DEEP_OVERHEAD + CellChunkImmutableSegment.DEEP_OVERHEAD_CCM +646 + numOfCells * oneCellOnCCMHeapSize; +647 +648 assertEquals(totalCellsLen, regionServicesForStores.getMemStoreSize()); +649 assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); +650 +651 MemStoreSize size = memstore.getFlushableSize(); +652 MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot +653 region.decrMemStoreSize(size); // simulate flusher +654 ImmutableSegment s = memstore.getSnapshot(); +655 assertEquals(numOfCells, s.getCellsCount()); +656 assertEquals(0, regionServicesForStores.getMemStoreSize()); +657 +658 memstore.clearSnapshot(snapshot.getId()); +659 } +660 +661 +662 private long addRowsByKeysDataSize(final AbstractMemStore hmc, String[] keys) { +663 byte[] fam = Bytes.toBytes("testfamily"); +664 byte[] qf = Bytes.toBytes("testqualifier"); +665 MemStoreSizing memstoreSizing = new MemStoreSizing(); +666 for (int i = 0; i < keys.length; i++) { +667 long timestamp = System.currentTimeMillis(); +668 Threads.sleep(1); // to make sure each kv gets a different ts +669 byte[] row = Bytes.toBytes(keys[i]); +670 byte[] val = Bytes.toBytes(keys[i] + i); +671 KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); +672 hmc.add(kv, memstoreSizing); +673 LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); +674 } +675 regionServicesForStores.addMemStoreSize(memstoreSizing); +676 return memstoreSizing.getDataSize(); +677 } +678 +679 private long cellBeforeFlushSize() { +680 // make one cell +681 byte[] row = Bytes.toBytes("A"); +682 byte[] val = Bytes.toBytes("A" + 0); +683 KeyValue kv = +684 new KeyValue(row, Bytes.toBytes("testfamily"), Bytes.toBytes("testqualifier"), +685 System.currentTimeMillis(), val); +686 return ClassSize.align( +687 ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + KeyValue.FIXED_OVERHEAD + KeyValueUtil.length(kv)); +688 } +689 +690 private long cellAfterFlushSize() { +691 // make one cell +692 byte[] row = Bytes.toBytes("A"); +693 byte[] val = Bytes.toBytes("A" + 0); +694 KeyValue kv = +695 new KeyValue(row, Bytes.toBytes("testfamily"), Bytes.toBytes("testqualifier"), +696 System.currentTimeMillis(), val); +697 +698 return toCellChunkMap ? +699 ClassSize.align( +700 ClassSize.CELL_CHUNK_MAP_ENTRY + KeyValueUtil.length(kv)) : +701 ClassSize.align( +702 ClassSize.CELL_ARRAY_MAP_ENTRY + KeyValue.FIXED_OVERHEAD + KeyValueUtil.length(kv)); +703 } +704}