hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1548981 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/regionserver/HRegion.java test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
Date Sun, 08 Dec 2013 03:01:25 GMT
Author: liyin
Date: Sun Dec  8 03:01:24 2013
New Revision: 1548981

URL: http://svn.apache.org/r1548981
Log:
[HBASE-8805] Add online configeration supporting and corresponding testcases

Summary: HRegion implements ConfigurationObserver, register/deregister from HRegionServer.configurationManager,
testcases

Test Plan: Run testcase

Reviewers: gauravm, zelaine, liyintang

Reviewed By: liyintang

CC: mahesh, hbase-eng@

Differential Revision: https://phabricator.fb.com/D1067616

Task ID: 2977299

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1548981&r1=1548980&r2=1548981&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Sun Dec  8 03:01:24 2013
@@ -26,13 +26,11 @@ import java.io.UnsupportedEncodingExcept
 import java.lang.reflect.Constructor;
 import java.net.InetSocketAddress;
 import java.text.ParseException;
-import java.text.SimpleDateFormat;
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -55,6 +53,8 @@ import java.util.concurrent.atomic.Atomi
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.Date;
+import java.text.SimpleDateFormat;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -72,15 +72,14 @@ import org.apache.hadoop.hbase.HRegionIn
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowLock;
-import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.Reference.Range;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
@@ -144,7 +143,7 @@ import com.google.common.collect.Lists;
  * regionName is a unique identifier for this HRegion. (startKey, endKey]
  * defines the keyspace for this HRegion.
  */
-public class HRegion implements HeapSize, ConfigurationObserver {
+public class HRegion implements HeapSize {
   public static final Log LOG = LogFactory.getLog(HRegion.class);
   static final String SPLITDIR = "splits";
   static final String MERGEDIR = "merges";
@@ -586,8 +585,6 @@ public class HRegion implements HeapSize
    */
   public long initialize(final Progressable reporter)
   throws IOException {
-    HRegionServer.configurationManager.registerObserver(this);
-
     MonitoredTask status = TaskMonitor.get().createStatus(
         "Initializing region " + this);
     try {
@@ -838,8 +835,6 @@ public class HRegion implements HeapSize
    * @throws IOException e
    */
   public List<StoreFile> close(final boolean abort) throws IOException {
-    HRegionServer.configurationManager.deregisterObserver(this);
-
     MonitoredTask status = TaskMonitor.get().createStatus(
         "Closing region " + this + (abort ? " due to abort" : ""));
     if (isClosed()) {
@@ -4129,4 +4124,5 @@ public class HRegion implements HeapSize
       s.updateConfiguration();
     }
   }
+
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java?rev=1548981&r1=1548980&r2=1548981&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
Sun Dec  8 03:01:24 2013
@@ -20,13 +20,8 @@
 
 package org.apache.hadoop.hbase;
 
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
 import junit.framework.Assert;
 import junit.framework.TestCase;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -41,6 +36,10 @@ import org.apache.hadoop.hbase.util.Byte
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * This test verifies the correctness of the Per Column Family flushing strategy
  */
@@ -61,8 +60,9 @@ public class TestPerColumnFamilyFlush ex
   public static final byte[] FAMILY2 = families[1];
   public static final byte[] FAMILY3 = families[2];
 
-  private void initHRegion (String callingMethod, Configuration conf)
-      throws IOException {
+  private void initHRegion (String callingMethod,
+                            HBaseConfiguration conf)
+          throws IOException {
     HTableDescriptor htd = new HTableDescriptor(TABLENAME);
     for(byte [] family : families) {
       htd.addFamily(new HColumnDescriptor(family));
@@ -105,12 +105,12 @@ public class TestPerColumnFamilyFlush ex
   @Test
   public void testSelectiveFlushWhenEnabled() throws IOException {
     // Set up the configuration
-    Configuration conf = HBaseConfiguration.create();
+    HBaseConfiguration conf = new HBaseConfiguration();
     conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200*1024);
     conf.setBoolean(HConstants.HREGION_MEMSTORE_PER_COLUMN_FAMILY_FLUSH, true);
     conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, 100*1024);
 
-    // Initialize the HRegion
+    // Intialize the HRegion
     initHRegion(getName(), conf);
     // Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
     for (int i = 1; i <= 1200; i++) {
@@ -247,123 +247,15 @@ public class TestPerColumnFamilyFlush ex
     Assert.assertEquals(0, region.getMemstoreSize().get());
   }
 
-  public void testSelectiveFlushWithThreshold(long t) throws IOException {
-    /*         t->|
-     * Phase 1    |     |     |
-     *            +-----+-----+
-     *           FM1   FM2   FM3
-     */
-    // flush all
-    region.flushcache(false);
-    region.put(createPut(2, 1));
-    region.put(createPut(3, 1));
-
-
-    int cnt = 0;
-    while (region.getStore(FAMILY1).getMemStoreSize() <= t) {
-      region.put(createPut(1, cnt));
-      cnt ++;
-    }
-
-    final long sizeFamily1_1 = region.getStore(FAMILY1).getMemStoreSize();
-    final long sizeFamily2_1 = region.getStore(FAMILY2).getMemStoreSize();
-    final long sizeFamily3_1 = region.getStore(FAMILY3).getMemStoreSize();
-
-    Assert.assertTrue(sizeFamily1_1 > t);
-    Assert.assertTrue(sizeFamily2_1 > 0);
-    Assert.assertTrue(sizeFamily3_1 > 0);
-
-    /*         t->
-     * Phase 2          |     |
-     *            +-----+-----+
-     *           FM1   FM2   FM3
-     */
-    // flush, should only flush family 1
-    region.flushcache(true);
-
-    final long sizeFamily1_2 = region.getStore(FAMILY1).getMemStoreSize();
-    final long sizeFamily2_2 = region.getStore(FAMILY2).getMemStoreSize();
-    final long sizeFamily3_2 = region.getStore(FAMILY3).getMemStoreSize();
-
-    // should clear FAMILY1 only
-    Assert.assertEquals("sizeFamily1", 0, sizeFamily1_2);
-    Assert.assertEquals("sizeFamily2", sizeFamily2_1, sizeFamily2_2);
-    Assert.assertEquals("sizeFamily3", sizeFamily3_1, sizeFamily3_2);
-    /*         t->,
-     * Phase 3    |     |     |
-     *            +-----+-----+
-     *           FM1   FM2   FM3
-     */
-    // flush all
-    region.flushcache(false);
-
-    region.put(createPut(2, 1));
-    region.put(createPut(3, 1));
-    // reduce cnt to 1 to make the size less than t
-    cnt--;
-    for (int i = 0; i < cnt; i ++) {
-      region.put(createPut(1, i));
-    }
-
-    final long sizeFamily1_3 = region.getStore(FAMILY1).getMemStoreSize();
-    final long sizeFamily2_3 = region.getStore(FAMILY2).getMemStoreSize();
-    final long sizeFamily3_3 = region.getStore(FAMILY3).getMemStoreSize();
-
-    Assert.assertTrue("sizeFamily1 < t", sizeFamily1_3 < t);
-    Assert.assertEquals("sizeFamily2", sizeFamily2_1, sizeFamily2_3);
-    Assert.assertEquals("sizeFamily3", sizeFamily3_1, sizeFamily3_3);
-    /*         t->
-     * Phase 3
-     *            +-----+-----+
-     *           FM1   FM2   FM3
-     */
-    // flush, should flush all
-    region.flushcache(true);
-
-    final long sizeFamily1_4 = region.getStore(FAMILY1).getMemStoreSize();
-    final long sizeFamily2_4 = region.getStore(FAMILY2).getMemStoreSize();
-    final long sizeFamily3_4 = region.getStore(FAMILY3).getMemStoreSize();
-    Assert.assertEquals("sizeFamily1", 0, sizeFamily1_4);
-    Assert.assertEquals("sizeFamily2", 0, sizeFamily2_4);
-    Assert.assertEquals("sizeFamily3", 0, sizeFamily3_4);
-  }
-
-  @Test
-  public void testConfigueChange() throws IOException {
-    final int T_init = 100*1024;
-    final int T_1 = 200*1024;
-    final int T_2 = 50*1024;
-
-    // Set up the configuration
-    Configuration conf = HBaseConfiguration.create();
-    conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200*1024);
-    conf.setBoolean(HConstants.HREGION_MEMSTORE_PER_COLUMN_FAMILY_FLUSH, true);
-    conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, T_init);
-
-    // Initialize the HRegion
-    initHRegion(getName(), conf);
-
-    // test for threshold of T_init
-    this.testSelectiveFlushWithThreshold(T_init);
-    // test for threshold of T_1
-    conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, T_1);
-    HRegionServer.configurationManager.notifyAllObservers(conf);
-    this.testSelectiveFlushWithThreshold(T_1);
-    // test for threshold of T_2
-    conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, T_2);
-    HRegionServer.configurationManager.notifyAllObservers(conf);
-    this.testSelectiveFlushWithThreshold(T_2);
-  }
-
   @Test
   public void testSelectiveFlushWhenNotEnabled() throws IOException {
     // Set up the configuration
-    Configuration conf = HBaseConfiguration.create();
+    HBaseConfiguration conf = new HBaseConfiguration();
     conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200 * 1024);
     conf.setBoolean(HConstants.HREGION_MEMSTORE_PER_COLUMN_FAMILY_FLUSH, false);
     conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, 100 * 1024);
 
-    // Initialize the HRegion
+    // Intialize the HRegion
     initHRegion(getName(), conf);
     // Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
     for (int i = 1; i <= 1200; i++) {



Mime
View raw message