hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1546428 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/regionserver/HRegion.java test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
Date Thu, 28 Nov 2013 18:13:26 GMT
Author: liyin
Date: Thu Nov 28 18:13:26 2013
New Revision: 1546428

URL: http://svn.apache.org/r1546428
Log:
[HBASE-8805] Add online configeration supporting and corresponding testcases

Author: daviddeng

Summary: HRegion implements ConfigurationObserver, register/deregister from HRegionServer.configurationManager,
testcases

Test Plan: Run testcase

Reviewers: gauravm, zelaine, liyintang

Reviewed By: liyintang

CC: mahesh, hbase-eng@

Differential Revision: https://phabricator.fb.com/D1067616

Task ID: 2977299

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1546428&r1=1546427&r2=1546428&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Thu Nov 28 18:13:26 2013
@@ -26,11 +26,13 @@ import java.io.UnsupportedEncodingExcept
 import java.lang.reflect.Constructor;
 import java.net.InetSocketAddress;
 import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -53,8 +55,6 @@ import java.util.concurrent.atomic.Atomi
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.Date;
-import java.text.SimpleDateFormat;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -72,14 +72,15 @@ import org.apache.hadoop.hbase.HRegionIn
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.NotServingRegionException;
-import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowLock;
+import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.Reference.Range;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
@@ -143,7 +144,7 @@ import com.google.common.collect.Lists;
  * regionName is a unique identifier for this HRegion. (startKey, endKey]
  * defines the keyspace for this HRegion.
  */
-public class HRegion implements HeapSize {
+public class HRegion implements HeapSize, ConfigurationObserver {
   public static final Log LOG = LogFactory.getLog(HRegion.class);
   static final String SPLITDIR = "splits";
   static final String MERGEDIR = "merges";
@@ -274,7 +275,7 @@ public class HRegion implements HeapSize
   final long memstoreFlushSize;
   // The maximum size a column family's memstore can grow up to,
   // before being flushed.
-  final long columnfamilyMemstoreFlushSize;
+  volatile long columnfamilyMemstoreFlushSize;
   // Last flush time for each Store. Useful when we are flushing for each column
   private Map<Store, Long> lastStoreFlushTimeMap
     = new ConcurrentHashMap<Store, Long>();
@@ -534,15 +535,30 @@ public class HRegion implements HeapSize
     }
     this.disableWAL = regionInfo.getTableDesc().isWALDisabled();
     this.memstoreFlushSize = flushSize;
-    this.columnfamilyMemstoreFlushSize = conf.getLong(
-            HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE,
-            HTableDescriptor.DEFAULT_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE);
     this.blockingMemStoreSize = (long)(this.memstoreFlushSize *
       conf.getFloat(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 2));
     this.waitOnMemstoreBlock =
         conf.getBoolean(HConstants.HREGION_MEMSTORE_WAIT_ON_BLOCK, true);
     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
+    // initialize dynamic parameters with current configuration
+    this.loadDynamicConf(conf);
+  }
 
+  @Override
+  public void notifyOnChange(Configuration conf) {
+    LOG.info("Online configuration changed!");
+    this.loadDynamicConf(conf);
+  }
+  /**
+   * Load online configurable parameters from a specified Configuration
+   */
+  private void loadDynamicConf(Configuration conf) {
+    this.columnfamilyMemstoreFlushSize = conf.getLong(
+        HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE,
+        HTableDescriptor.DEFAULT_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE);
+
+    LOG.info(String.format("columnfamilyMemstoreFlushSize is set to %d",
+        this.columnfamilyMemstoreFlushSize));
   }
 
   /**
@@ -563,6 +579,8 @@ public class HRegion implements HeapSize
    */
   public long initialize(final Progressable reporter)
   throws IOException {
+    HRegionServer.configurationManager.registerObserver(this);
+
     MonitoredTask status = TaskMonitor.get().createStatus(
         "Initializing region " + this);
     try {
@@ -813,6 +831,8 @@ public class HRegion implements HeapSize
    * @throws IOException e
    */
   public List<StoreFile> close(final boolean abort) throws IOException {
+    HRegionServer.configurationManager.deregisterObserver(this);
+
     MonitoredTask status = TaskMonitor.get().createStatus(
         "Closing region " + this + (abort ? " due to abort" : ""));
     if (isClosed()) {
@@ -4094,5 +4114,4 @@ public class HRegion implements HeapSize
       s.updateConfiguration();
     }
   }
-
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java?rev=1546428&r1=1546427&r2=1546428&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
Thu Nov 28 18:13:26 2013
@@ -20,8 +20,13 @@
 
 package org.apache.hadoop.hbase;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
 import junit.framework.Assert;
 import junit.framework.TestCase;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -36,10 +41,6 @@ import org.apache.hadoop.hbase.util.Byte
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
 /**
  * This test verifies the correctness of the Per Column Family flushing strategy
  */
@@ -60,9 +61,8 @@ public class TestPerColumnFamilyFlush ex
   public static final byte[] FAMILY2 = families[1];
   public static final byte[] FAMILY3 = families[2];
 
-  private void initHRegion (String callingMethod,
-                            HBaseConfiguration conf)
-          throws IOException {
+  private void initHRegion (String callingMethod, Configuration conf)
+      throws IOException {
     HTableDescriptor htd = new HTableDescriptor(TABLENAME);
     for(byte [] family : families) {
       htd.addFamily(new HColumnDescriptor(family));
@@ -105,12 +105,12 @@ public class TestPerColumnFamilyFlush ex
   @Test
   public void testSelectiveFlushWhenEnabled() throws IOException {
     // Set up the configuration
-    HBaseConfiguration conf = new HBaseConfiguration();
+    Configuration conf = HBaseConfiguration.create();
     conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200*1024);
     conf.setBoolean(HConstants.HREGION_MEMSTORE_PER_COLUMN_FAMILY_FLUSH, true);
     conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, 100*1024);
 
-    // Intialize the HRegion
+    // Initialize the HRegion
     initHRegion(getName(), conf);
     // Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
     for (int i = 1; i <= 1200; i++) {
@@ -247,15 +247,123 @@ public class TestPerColumnFamilyFlush ex
     Assert.assertEquals(0, region.getMemstoreSize().get());
   }
 
+  public void testSelectiveFlushWithThreshold(long t) throws IOException {
+    /*         t->|
+     * Phase 1    |     |     |
+     *            +-----+-----+
+     *           FM1   FM2   FM3
+     */
+    // flush all
+    region.flushcache(false);
+    region.put(createPut(2, 1));
+    region.put(createPut(3, 1));
+
+
+    int cnt = 0;
+    while (region.getStore(FAMILY1).getMemStoreSize() <= t) {
+      region.put(createPut(1, cnt));
+      cnt ++;
+    }
+
+    final long sizeFamily1_1 = region.getStore(FAMILY1).getMemStoreSize();
+    final long sizeFamily2_1 = region.getStore(FAMILY2).getMemStoreSize();
+    final long sizeFamily3_1 = region.getStore(FAMILY3).getMemStoreSize();
+
+    Assert.assertTrue(sizeFamily1_1 > t);
+    Assert.assertTrue(sizeFamily2_1 > 0);
+    Assert.assertTrue(sizeFamily3_1 > 0);
+
+    /*         t->
+     * Phase 2          |     |
+     *            +-----+-----+
+     *           FM1   FM2   FM3
+     */
+    // flush, should only flush family 1
+    region.flushcache(true);
+
+    final long sizeFamily1_2 = region.getStore(FAMILY1).getMemStoreSize();
+    final long sizeFamily2_2 = region.getStore(FAMILY2).getMemStoreSize();
+    final long sizeFamily3_2 = region.getStore(FAMILY3).getMemStoreSize();
+
+    // should clear FAMILY1 only
+    Assert.assertEquals("sizeFamily1", 0, sizeFamily1_2);
+    Assert.assertEquals("sizeFamily2", sizeFamily2_1, sizeFamily2_2);
+    Assert.assertEquals("sizeFamily3", sizeFamily3_1, sizeFamily3_2);
+    /*         t->,
+     * Phase 3    |     |     |
+     *            +-----+-----+
+     *           FM1   FM2   FM3
+     */
+    // flush all
+    region.flushcache(false);
+
+    region.put(createPut(2, 1));
+    region.put(createPut(3, 1));
+    // reduce cnt to 1 to make the size less than t
+    cnt--;
+    for (int i = 0; i < cnt; i ++) {
+      region.put(createPut(1, i));
+    }
+
+    final long sizeFamily1_3 = region.getStore(FAMILY1).getMemStoreSize();
+    final long sizeFamily2_3 = region.getStore(FAMILY2).getMemStoreSize();
+    final long sizeFamily3_3 = region.getStore(FAMILY3).getMemStoreSize();
+
+    Assert.assertTrue("sizeFamily1 < t", sizeFamily1_3 < t);
+    Assert.assertEquals("sizeFamily2", sizeFamily2_1, sizeFamily2_3);
+    Assert.assertEquals("sizeFamily3", sizeFamily3_1, sizeFamily3_3);
+    /*         t->
+     * Phase 3
+     *            +-----+-----+
+     *           FM1   FM2   FM3
+     */
+    // flush, should flush all
+    region.flushcache(true);
+
+    final long sizeFamily1_4 = region.getStore(FAMILY1).getMemStoreSize();
+    final long sizeFamily2_4 = region.getStore(FAMILY2).getMemStoreSize();
+    final long sizeFamily3_4 = region.getStore(FAMILY3).getMemStoreSize();
+    Assert.assertEquals("sizeFamily1", 0, sizeFamily1_4);
+    Assert.assertEquals("sizeFamily2", 0, sizeFamily2_4);
+    Assert.assertEquals("sizeFamily3", 0, sizeFamily3_4);
+  }
+
+  @Test
+  public void testConfigueChange() throws IOException {
+    final int T_init = 100*1024;
+    final int T_1 = 200*1024;
+    final int T_2 = 50*1024;
+
+    // Set up the configuration
+    Configuration conf = HBaseConfiguration.create();
+    conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200*1024);
+    conf.setBoolean(HConstants.HREGION_MEMSTORE_PER_COLUMN_FAMILY_FLUSH, true);
+    conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, T_init);
+
+    // Initialize the HRegion
+    initHRegion(getName(), conf);
+
+    // test for threshold of T_init
+    this.testSelectiveFlushWithThreshold(T_init);
+    // test for threshold of T_1
+    conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, T_1);
+    HRegionServer.configurationManager.notifyAllObservers(conf);
+    this.testSelectiveFlushWithThreshold(T_1);
+    // test for threshold of T_2
+    conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, T_2);
+    HRegionServer.configurationManager.notifyAllObservers(conf);
+    this.testSelectiveFlushWithThreshold(T_2);
+  }
+
   @Test
   public void testSelectiveFlushWhenNotEnabled() throws IOException {
     // Set up the configuration
-    HBaseConfiguration conf = new HBaseConfiguration();
+    Configuration conf = HBaseConfiguration.create();
     conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200 * 1024);
     conf.setBoolean(HConstants.HREGION_MEMSTORE_PER_COLUMN_FAMILY_FLUSH, false);
     conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, 100 * 1024);
 
-    // Intialize the HRegion
+    // Initialize the HRegion
     initHRegion(getName(), conf);
     // Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
     for (int i = 1; i <= 1200; i++) {



Mime
View raw message