From commits-return-65020-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Wed Jan 10 16:31:21 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id D261E18076D for ; Wed, 10 Jan 2018 16:31:21 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id C1BF9160C46; Wed, 10 Jan 2018 15:31:21 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D8D52160C40 for ; Wed, 10 Jan 2018 16:31:19 +0100 (CET) Received: (qmail 54447 invoked by uid 500); 10 Jan 2018 15:31:18 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 54238 invoked by uid 99); 10 Jan 2018 15:31:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 10 Jan 2018 15:31:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 61662DFF26; Wed, 10 Jan 2018 15:31:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Wed, 10 Jan 2018 15:31:20 -0000 Message-Id: <356dd49108da4f55822976f96c9d0c64@git.apache.org> In-Reply-To: <00262d5ef07742b591cd4d1311321bdc@git.apache.org> References: <00262d5ef07742b591cd4d1311321bdc@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [04/16] hbase-site git commit: Published site at . http://git-wip-us.apache.org/repos/asf/hbase-site/blob/828b306f/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.html index a5b3b10..40819c7 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.html @@ -45,136 +45,139 @@ 037import org.apache.hadoop.hbase.testclassification.SmallTests; 038import org.apache.hadoop.hbase.util.Bytes; 039import org.junit.BeforeClass; -040import org.junit.Test; -041import org.junit.experimental.categories.Category; -042 -043@Category({RegionServerTests.class, SmallTests.class}) -044public class TestMemstoreLABWithoutPool { -045 private final static Configuration conf = new Configuration(); -046 -047 private static final byte[] rk = Bytes.toBytes("r1"); -048 private static final byte[] cf = Bytes.toBytes("f"); -049 private static final byte[] q = Bytes.toBytes("q"); -050 -051 @BeforeClass -052 public static void setUpBeforeClass() throws Exception { -053 long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage() -054 .getMax() * 0.8); -055 // disable pool -056 ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT + Bytes.SIZEOF_LONG, false, globalMemStoreLimit, -057 0.0f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); -058 } -059 -060 /** -061 * Test a bunch of random allocations -062 */ -063 @Test -064 public void testLABRandomAllocation() { -065 Random rand = new Random(); -066 MemStoreLAB mslab = new MemStoreLABImpl(); -067 int expectedOff = 0; -068 ByteBuffer lastBuffer = null; -069 int lastChunkId = -1; -070 // 100K iterations by 0-1K alloc -> 50MB expected -071 // should be reasonable for unit test and also cover wraparound -072 // behavior -073 for (int i = 0; i < 100000; i++) { -074 int valSize = rand.nextInt(1000); -075 KeyValue kv = new KeyValue(rk, cf, q, new byte[valSize]); -076 int size = KeyValueUtil.length(kv); -077 ByteBufferKeyValue newKv = (ByteBufferKeyValue) mslab.copyCellInto(kv); -078 if (newKv.getBuffer() != lastBuffer) { -079 // since we add the chunkID at the 0th offset of the chunk and the -080 // chunkid is an int we need to account for those 4 bytes -081 expectedOff = Bytes.SIZEOF_INT; -082 lastBuffer = newKv.getBuffer(); -083 int chunkId = newKv.getBuffer().getInt(0); -084 assertTrue("chunkid should be different", chunkId != lastChunkId); -085 lastChunkId = chunkId; -086 } -087 assertEquals(expectedOff, newKv.getOffset()); -088 assertTrue("Allocation overruns buffer", -089 newKv.getOffset() + size <= newKv.getBuffer().capacity()); -090 expectedOff += size; -091 } -092 } -093 -094 /** -095 * Test frequent chunk retirement with chunk pool triggered by lots of threads, making sure -096 * there's no memory leak (HBASE-16195) -097 * @throws Exception if any error occurred -098 */ -099 @Test -100 public void testLABChunkQueueWithMultipleMSLABs() throws Exception { -101 Configuration conf = HBaseConfiguration.create(); -102 MemStoreLABImpl[] mslab = new MemStoreLABImpl[10]; -103 for (int i = 0; i < 10; i++) { -104 mslab[i] = new MemStoreLABImpl(conf); -105 } -106 // launch multiple threads to trigger frequent chunk retirement -107 List<Thread> threads = new ArrayList<>(); -108 // create smaller sized kvs -109 final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), -110 new byte[0]); -111 for (int i = 0; i < 10; i++) { -112 for (int j = 0; j < 10; j++) { -113 threads.add(getChunkQueueTestThread(mslab[i], "testLABChunkQueue-" + j, kv)); -114 } -115 } -116 for (Thread thread : threads) { -117 thread.start(); +040import org.junit.Ignore; +041import org.junit.Test; +042import org.junit.experimental.categories.Category; +043 +044 +045@Ignore // See HBASE-19742 for issue on reenabling. +046@Category({RegionServerTests.class, SmallTests.class}) +047public class TestMemstoreLABWithoutPool { +048 private final static Configuration conf = new Configuration(); +049 +050 private static final byte[] rk = Bytes.toBytes("r1"); +051 private static final byte[] cf = Bytes.toBytes("f"); +052 private static final byte[] q = Bytes.toBytes("q"); +053 +054 @BeforeClass +055 public static void setUpBeforeClass() throws Exception { +056 long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage() +057 .getMax() * 0.8); +058 // disable pool +059 ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT + Bytes.SIZEOF_LONG, false, globalMemStoreLimit, +060 0.0f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); +061 } +062 +063 /** +064 * Test a bunch of random allocations +065 */ +066 @Test +067 public void testLABRandomAllocation() { +068 Random rand = new Random(); +069 MemStoreLAB mslab = new MemStoreLABImpl(); +070 int expectedOff = 0; +071 ByteBuffer lastBuffer = null; +072 int lastChunkId = -1; +073 // 100K iterations by 0-1K alloc -> 50MB expected +074 // should be reasonable for unit test and also cover wraparound +075 // behavior +076 for (int i = 0; i < 100000; i++) { +077 int valSize = rand.nextInt(1000); +078 KeyValue kv = new KeyValue(rk, cf, q, new byte[valSize]); +079 int size = KeyValueUtil.length(kv); +080 ByteBufferKeyValue newKv = (ByteBufferKeyValue) mslab.copyCellInto(kv); +081 if (newKv.getBuffer() != lastBuffer) { +082 // since we add the chunkID at the 0th offset of the chunk and the +083 // chunkid is an int we need to account for those 4 bytes +084 expectedOff = Bytes.SIZEOF_INT; +085 lastBuffer = newKv.getBuffer(); +086 int chunkId = newKv.getBuffer().getInt(0); +087 assertTrue("chunkid should be different", chunkId != lastChunkId); +088 lastChunkId = chunkId; +089 } +090 assertEquals(expectedOff, newKv.getOffset()); +091 assertTrue("Allocation overruns buffer", +092 newKv.getOffset() + size <= newKv.getBuffer().capacity()); +093 expectedOff += size; +094 } +095 } +096 +097 /** +098 * Test frequent chunk retirement with chunk pool triggered by lots of threads, making sure +099 * there's no memory leak (HBASE-16195) +100 * @throws Exception if any error occurred +101 */ +102 @Test +103 public void testLABChunkQueueWithMultipleMSLABs() throws Exception { +104 Configuration conf = HBaseConfiguration.create(); +105 MemStoreLABImpl[] mslab = new MemStoreLABImpl[10]; +106 for (int i = 0; i < 10; i++) { +107 mslab[i] = new MemStoreLABImpl(conf); +108 } +109 // launch multiple threads to trigger frequent chunk retirement +110 List<Thread> threads = new ArrayList<>(); +111 // create smaller sized kvs +112 final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), +113 new byte[0]); +114 for (int i = 0; i < 10; i++) { +115 for (int j = 0; j < 10; j++) { +116 threads.add(getChunkQueueTestThread(mslab[i], "testLABChunkQueue-" + j, kv)); +117 } 118 } -119 // let it run for some time -120 Thread.sleep(3000); -121 for (Thread thread : threads) { -122 thread.interrupt(); -123 } -124 boolean threadsRunning = true; -125 boolean alive = false; -126 while (threadsRunning) { -127 alive = false; -128 for (Thread thread : threads) { -129 if (thread.isAlive()) { -130 alive = true; -131 break; -132 } -133 } -134 if (!alive) { -135 threadsRunning = false; +119 for (Thread thread : threads) { +120 thread.start(); +121 } +122 // let it run for some time +123 Thread.sleep(3000); +124 for (Thread thread : threads) { +125 thread.interrupt(); +126 } +127 boolean threadsRunning = true; +128 boolean alive = false; +129 while (threadsRunning) { +130 alive = false; +131 for (Thread thread : threads) { +132 if (thread.isAlive()) { +133 alive = true; +134 break; +135 } 136 } -137 } -138 // close the mslab -139 for (int i = 0; i < 10; i++) { -140 mslab[i].close(); -141 } -142 // all of the chunkIds would have been returned back -143 assertTrue("All the chunks must have been cleared", -144 ChunkCreator.INSTANCE.numberOfMappedChunks() == 0); -145 } -146 -147 private Thread getChunkQueueTestThread(final MemStoreLABImpl mslab, String threadName, -148 Cell cellToCopyInto) { -149 Thread thread = new Thread() { -150 volatile boolean stopped = false; -151 -152 @Override -153 public void run() { -154 while (!stopped) { -155 // keep triggering chunk retirement -156 mslab.copyCellInto(cellToCopyInto); -157 } -158 } -159 -160 @Override -161 public void interrupt() { -162 this.stopped = true; -163 } -164 }; -165 thread.setName(threadName); -166 thread.setDaemon(true); -167 return thread; -168 } -169} +137 if (!alive) { +138 threadsRunning = false; +139 } +140 } +141 // close the mslab +142 for (int i = 0; i < 10; i++) { +143 mslab[i].close(); +144 } +145 // all of the chunkIds would have been returned back +146 assertTrue("All the chunks must have been cleared", +147 ChunkCreator.INSTANCE.numberOfMappedChunks() == 0); +148 } +149 +150 private Thread getChunkQueueTestThread(final MemStoreLABImpl mslab, String threadName, +151 Cell cellToCopyInto) { +152 Thread thread = new Thread() { +153 volatile boolean stopped = false; +154 +155 @Override +156 public void run() { +157 while (!stopped) { +158 // keep triggering chunk retirement +159 mslab.copyCellInto(cellToCopyInto); +160 } +161 } +162 +163 @Override +164 public void interrupt() { +165 this.stopped = true; +166 } +167 }; +168 thread.setName(threadName); +169 thread.setDaemon(true); +170 return thread; +171 } +172}