hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-12651 Backport HBASE-12559 'Provide LoadBalancer with online configuration capability' to branch-1
Date Mon, 15 Dec 2014 20:04:02 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 8b85feceb -> 677153ad6


HBASE-12651 Backport HBASE-12559 'Provide LoadBalancer with online configuration capability'
to branch-1


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/677153ad
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/677153ad
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/677153ad

Branch: refs/heads/branch-1
Commit: 677153ad61a260f14cbaa6ec6cb899e5428c94d8
Parents: 8b85fec
Author: tedyu <yuzhihong@gmail.com>
Authored: Mon Dec 15 12:03:55 2014 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Mon Dec 15 12:03:55 2014 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/master/HMaster.java |   1 +
 .../hadoop/hbase/master/LoadBalancer.java       |  10 +-
 .../hbase/master/balancer/BaseLoadBalancer.java |   5 +
 .../master/balancer/StochasticLoadBalancer.java |  35 +++--
 .../hbase/regionserver/HRegionServer.java       |   2 +-
 .../hbase/client/TestUpdateConfiguration.java   |  28 ++++
 hbase-server/src/test/resources/hbase-site2.xml | 154 +++++++++++++++++++
 7 files changed, 221 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f966b12..738be01 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -729,6 +729,7 @@ public class HMaster extends HRegionServer implements MasterServices,
Server {
 
     status.markComplete("Initialization successful");
     LOG.info("Master has completed initialization");
+    configurationManager.registerObserver(this.balancer);
     initialized = true;
     // clear the dead servers with same host name and port of online server because we are
not
     // removing dead server with same hostname and port of rs which is trying to check in
before

http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
index e24d745..f979403 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
@@ -22,7 +22,9 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -46,7 +48,7 @@ import org.apache.hadoop.hbase.Stoppable;
  * <p>This classes produces plans for the {@link AssignmentManager} to execute.
  */
 @InterfaceAudience.Private
-public interface LoadBalancer extends Configurable, Stoppable {
+public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObserver {
 
   /**
    * Set the current cluster status.  This allows a LoadBalancer to map host name to a server
@@ -130,4 +132,10 @@ public interface LoadBalancer extends Configurable, Stoppable {
    * @param regionInfo
    */
   void regionOffline(HRegionInfo regionInfo);
+
+  /*
+   * Notification that config has changed
+   * @param conf
+   */
+  void onConfigurationChange(Configuration conf);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index 21f25ee..7aca88e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.RegionLoad;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.hbase.master.LoadBalancer;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RackManager;
@@ -1373,4 +1374,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
       return new HashMap<ServerName, List<HRegionInfo>>();
     }
   }
+
+  @Override
+  public void onConfigurationChange(Configuration conf) {
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index 8187630..9d8c7cb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -125,8 +125,14 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   private RegionReplicaRackCostFunction regionReplicaRackCostFunction;
 
   @Override
-  public void setConf(Configuration conf) {
+  public void onConfigurationChange(Configuration conf) {
+    setConf(conf);
+  }
+
+  @Override
+  public synchronized void setConf(Configuration conf) {
     super.setConf(conf);
+    LOG.info("loading config");
 
     maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
 
@@ -135,15 +141,19 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
 
     numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
 
-    localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
+    if (localityCandidateGenerator == null) {
+      localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
+    }
     localityCost = new LocalityCostFunction(conf, services);
 
-    candidateGenerators = new CandidateGenerator[] {
-      new RandomCandidateGenerator(),
-      new LoadCandidateGenerator(),
-      localityCandidateGenerator,
-      new RegionReplicaRackCandidateGenerator(),
-    };
+    if (candidateGenerators == null) {
+      candidateGenerators = new CandidateGenerator[] {
+          new RandomCandidateGenerator(),
+          new LoadCandidateGenerator(),
+          localityCandidateGenerator,
+          new RegionReplicaRackCandidateGenerator(),
+      };
+    }
 
     regionLoadFunctions = new CostFromRegionLoadFunction[] {
       new ReadRequestCostFunction(conf),
@@ -175,7 +185,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   }
 
   @Override
-  public void setClusterStatus(ClusterStatus st) {
+  public synchronized void setClusterStatus(ClusterStatus st) {
     super.setClusterStatus(st);
     updateRegionLoad();
     for(CostFromRegionLoadFunction cost : regionLoadFunctions) {
@@ -184,7 +194,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   }
 
   @Override
-  public void setMasterServices(MasterServices masterServices) {
+  public synchronized void setMasterServices(MasterServices masterServices) {
     super.setMasterServices(masterServices);
     this.localityCost.setServices(masterServices);
     this.localityCandidateGenerator.setServices(masterServices);
@@ -192,7 +202,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   }
 
   @Override
-  protected boolean areSomeRegionReplicasColocated(Cluster c) {
+  protected synchronized boolean areSomeRegionReplicasColocated(Cluster c) {
     regionReplicaHostCostFunction.init(c);
     if (regionReplicaHostCostFunction.cost() > 0) return true;
     regionReplicaRackCostFunction.init(c);
@@ -205,7 +215,8 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
    * should always approach the optimal state given enough steps.
    */
   @Override
-  public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>>
clusterState) {
+  public synchronized List<RegionPlan> balanceCluster(Map<ServerName,
+    List<HRegionInfo>> clusterState) {
     List<RegionPlan> plans = balanceMasterRegions(clusterState);
     if (plans != null || clusterState == null || clusterState.size() <= 1) {
       return plans;

http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b52065d..ddeacd3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -440,7 +440,7 @@ public class HRegionServer extends HasThread implements
    * Configuration manager is used to register/deregister and notify the configuration observers
    * when the regionserver is notified that there was a change in the on disk configs.
    */
-  private final ConfigurationManager configurationManager;
+  protected final ConfigurationManager configurationManager;
 
   /**
    * Starts a HRegionServer at the default location.

http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
index 74590e0..e2af1ab 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
@@ -18,10 +18,17 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import static org.junit.Assert.assertEquals;
+
 import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -46,4 +53,25 @@ public class TestUpdateConfiguration {
     ServerName server = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName();
     admin.updateConfiguration(server);
   }
+
+  @Test
+  public void testMasterOnlineConfigChange() throws IOException {
+    LOG.debug("Starting the test");
+    Path cnfPath = FileSystems.getDefault().getPath("target/test-classes/hbase-site.xml");
+    Path cnf2Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site2.xml");
+    Path cnf3Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site3.xml");
+    // make a backup of hbase-site.xml
+    Files.copy(cnfPath, cnf3Path, StandardCopyOption.REPLACE_EXISTING);
+    // update hbase-site.xml by overwriting it
+    Files.copy(cnf2Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
+
+    Admin admin = TEST_UTIL.getHBaseAdmin();
+    ServerName server = TEST_UTIL.getHBaseCluster().getMaster().getServerName();
+    admin.updateConfiguration(server);
+    Configuration conf = TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
+    int custom = conf.getInt("hbase.custom.config", 0);
+    assertEquals(custom, 1000);
+    // restore hbase-site.xml
+    Files.copy(cnf3Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/test/resources/hbase-site2.xml
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/resources/hbase-site2.xml b/hbase-server/src/test/resources/hbase-site2.xml
new file mode 100644
index 0000000..87d340e
--- /dev/null
+++ b/hbase-server/src/test/resources/hbase-site2.xml
@@ -0,0 +1,154 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>hbase.custom.config</name>
+    <value>1000</value>
+  </property>
+  <property>
+    <name>hbase.regionserver.msginterval</name>
+    <value>1000</value>
+    <description>Interval between messages from the RegionServer to HMaster
+    in milliseconds.  Default is 15. Set this value low if you want unit
+    tests to be responsive.
+    </description>
+  </property>
+  <property>
+    <name>hbase.defaults.for.version.skip</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hbase.server.thread.wakefrequency</name>
+    <value>1000</value>
+    <description>Time to sleep in between searches for work (in milliseconds).
+    Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.event.waiting.time</name>
+    <value>50</value>
+    <description>Time to sleep between checks to see if a table event took place.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.handler.count</name>
+    <value>5</value>
+  </property>
+  <property>
+    <name>hbase.master.info.port</name>
+    <value>-1</value>
+    <description>The port for the hbase master web UI
+    Set to -1 if you do not want the info server to run.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.port</name>
+    <value>0</value>
+    <description>Always have masters and regionservers come up on port '0' so we don't
clash over
+      default ports.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.port</name>
+    <value>0</value>
+    <description>Always have masters and regionservers come up on port '0' so we don't
clash over
+      default ports.
+    </description>
+  </property>
+  <property>
+    <name>hbase.ipc.client.fallback-to-simple-auth-allowed</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>hbase.regionserver.info.port</name>
+    <value>-1</value>
+    <description>The port for the hbase regionserver web UI
+    Set to -1 if you do not want the info server to run.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.info.port.auto</name>
+    <value>true</value>
+    <description>Info server auto port bind. Enables automatic port
+    search if hbase.regionserver.info.port is already in use.
+    Enabled for testing to run multiple tests on one machine.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.lease.thread.wakefrequency</name>
+    <value>3000</value>
+    <description>The interval between checks for expired region server leases.
+    This value has been reduced due to the other reduced values above so that
+    the master will notice a dead region server sooner. The default is 15 seconds.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.safemode</name>
+    <value>false</value>
+    <description>
+    Turn on/off safe mode in region server. Always on for production, always off
+    for tests.
+    </description>
+  </property>
+  <property>
+    <name>hbase.hregion.max.filesize</name>
+    <value>67108864</value>
+    <description>
+    Maximum desired file size for an HRegion.  If filesize exceeds
+    value + (value / 2), the HRegion is split in two.  Default: 256M.
+
+    Keep the maximum filesize small so we split more often in tests.
+    </description>
+  </property>
+  <property>
+    <name>hadoop.log.dir</name>
+    <value>${user.dir}/../logs</value>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.clientPort</name>
+    <value>21818</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The port at which the clients will connect.
+    </description>
+  </property>
+  <property>
+    <name>hbase.defaults.for.version.skip</name>
+    <value>true</value>
+    <description>
+    Set to true to skip the 'hbase.defaults.for.version'.
+    Setting this to true can be useful in contexts other than
+    the other side of a maven generation; i.e. running in an
+    ide.  You'll want to set this boolean to true to avoid
+    seeing the RuntimException complaint: "hbase-default.xml file
+    seems to be for and old version of HBase (@@@VERSION@@@), this
+    version is X.X.X-SNAPSHOT"
+    </description>
+  </property>
+  <property>
+    <name>hbase.table.sanity.checks</name>
+    <value>false</value>
+    <description>Skip sanity checks in tests
+    </description>
+  </property>
+</configuration>


Mime
View raw message