hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject svn commit: r792636 - in /hadoop/hbase/trunk: ./ bin/ conf/ src/java/org/apache/hadoop/hbase/zookeeper/ src/test/org/apache/hadoop/hbase/zookeeper/
Date Thu, 09 Jul 2009 19:04:43 GMT
Author: nitay
Date: Thu Jul  9 19:04:43 2009
New Revision: 792636

URL: http://svn.apache.org/viewvc?rev=792636&view=rev
Log:
HBASE-1551 HBase should manage multiple node ZooKeeper quorum

Added:
    hadoop/hbase/trunk/bin/zookeepers.sh
      - copied, changed from r792619, hadoop/hbase/trunk/bin/zookeeper.sh
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
Removed:
    hadoop/hbase/trunk/bin/hbase-zookeeper.sh
    hadoop/hbase/trunk/bin/zookeeper.sh
Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/bin/hbase-daemons.sh
    hadoop/hbase/trunk/bin/start-hbase.sh
    hadoop/hbase/trunk/bin/stop-hbase.sh
    hadoop/hbase/trunk/conf/hbase-default.xml
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=792636&r1=792635&r2=792636&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Thu Jul  9 19:04:43 2009
@@ -452,6 +452,7 @@
    HBASE-1624  Don't sort Puts if only one in list in HCM#processBatchOfRows
    HBASE-1626  Allow emitting Deletes out of new TableReducer
                (Lars George via Stack)
+   HBASE-1551  HBase should manage multiple node ZooKeeper quorum
 
   OPTIMIZATIONS
    HBASE-1412  Change values for delete column and column family in KeyValue

Modified: hadoop/hbase/trunk/bin/hbase-daemons.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/bin/hbase-daemons.sh?rev=792636&r1=792635&r2=792636&view=diff
==============================================================================
--- hadoop/hbase/trunk/bin/hbase-daemons.sh (original)
+++ hadoop/hbase/trunk/bin/hbase-daemons.sh Thu Jul  9 19:04:43 2009
@@ -37,6 +37,20 @@
 
 . $bin/hbase-config.sh
 
-exec "$bin/regionservers.sh" --config "${HBASE_CONF_DIR}" \
- cd "${HBASE_HOME}" \; \
- "$bin/hbase-daemon.sh" --config "${HBASE_CONF_DIR}" "$@"
+remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@"
+args="--config ${HBASE_CONF_DIR} $remote_cmd"
+
+command=$2
+case $command in
+  (regionserver)
+    exec "$bin/regionservers.sh" $args
+    ;;
+  (zookeeper)
+    exec "$bin/zookeepers.sh" $args
+    ;;
+  (*)
+    echo $usage
+    exit 1
+    ;;
+esac
+

Modified: hadoop/hbase/trunk/bin/start-hbase.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/bin/start-hbase.sh?rev=792636&r1=792635&r2=792636&view=diff
==============================================================================
--- hadoop/hbase/trunk/bin/start-hbase.sh (original)
+++ hadoop/hbase/trunk/bin/start-hbase.sh Thu Jul  9 19:04:43 2009
@@ -38,8 +38,8 @@
 then
   exit $errCode
 fi
-"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \
-  start zookeeper
+
+"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper
 "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master 
 "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
   --hosts "${HBASE_REGIONSERVERS}" start regionserver

Modified: hadoop/hbase/trunk/bin/stop-hbase.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/bin/stop-hbase.sh?rev=792636&r1=792635&r2=792636&view=diff
==============================================================================
--- hadoop/hbase/trunk/bin/stop-hbase.sh (original)
+++ hadoop/hbase/trunk/bin/stop-hbase.sh Thu Jul  9 19:04:43 2009
@@ -30,5 +30,4 @@
 . "$bin"/hbase-config.sh
 
 "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master
-"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \
-  stop zookeeper
+"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper

Copied: hadoop/hbase/trunk/bin/zookeepers.sh (from r792619, hadoop/hbase/trunk/bin/zookeeper.sh)
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/bin/zookeepers.sh?p2=hadoop/hbase/trunk/bin/zookeepers.sh&p1=hadoop/hbase/trunk/bin/zookeeper.sh&r1=792619&r2=792636&rev=792636&view=diff
==============================================================================
--- hadoop/hbase/trunk/bin/zookeeper.sh (original)
+++ hadoop/hbase/trunk/bin/zookeepers.sh Thu Jul  9 19:04:43 2009
@@ -20,18 +20,17 @@
 # * limitations under the License.
 # */
 # 
-# Run a shell command on all regionserver hosts.
+# Run a shell command on all zookeeper hosts.
 #
 # Environment Variables
 #
-#   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
 #   HBASE_CONF_DIR  Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
-#   HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
-#   HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
+#   HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
+#   HBASE_SSH_OPTS Options passed to ssh when running remote commands.
 #
 # Modelled after $HADOOP_HOME/bin/slaves.sh.
 
-usage="Usage: zookeeper [--config <hbase-confdir>] command..."
+usage="Usage: zookeepers [--config <hbase-confdir>] command..."
 
 # if no args specified, show usage
 if [ $# -le 0 ]; then
@@ -53,10 +52,14 @@
 fi
 
 if [ "$HBASE_MANAGES_ZK" = "true" ]; then
- ssh $HBASE_SSH_OPTS localhost $"${@// /\\ }" 2>&1 | sed "s/^/localhost: /" &
- if [ "$HBASE_SLAVE_SLEEP" != "" ]; then
-   sleep $HBASE_SLAVE_SLEEP
- fi
+  hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool`
+  cmd=$"${@// /\\ }"
+  for zookeeper in $hosts; do
+   ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" &
+   if [ "$HBASE_SLAVE_SLEEP" != "" ]; then
+     sleep $HBASE_SLAVE_SLEEP
+   fi
+  done
 fi
 
 wait

Modified: hadoop/hbase/trunk/conf/hbase-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/conf/hbase-default.xml?rev=792636&r1=792635&r2=792636&view=diff
==============================================================================
--- hadoop/hbase/trunk/conf/hbase-default.xml (original)
+++ hadoop/hbase/trunk/conf/hbase-default.xml Thu Jul  9 19:04:43 2009
@@ -200,6 +200,21 @@
     </description>
   </property>
   <property>
+    <name>hbase.zookeeper.dns.interface</name>
+    <value>default</value>
+    <description>The name of the Network Interface from which a ZooKeeper server
+      should report its IP address.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.dns.nameserver</name>
+    <value>default</value>
+    <description>The host name or IP address of the name server (DNS)
+      which a ZooKeeper server should use to determine the host name used by the
+      master for communication and display purposes.
+    </description>
+  </property>
+  <property>
     <name>hbase.regionserver.dns.interface</name>
     <value>default</value>
     <description>The name of the Network Interface from which a region server 

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java?rev=792636&r1=792635&r2=792636&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java Thu Jul
 9 19:04:43 2009
@@ -19,8 +19,11 @@
  */
 package org.apache.hadoop.hbase.zookeeper;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintWriter;
+import java.net.UnknownHostException;
 import java.util.Properties;
 import java.util.Map.Entry;
 
@@ -28,11 +31,11 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.net.DNS;
 import org.apache.zookeeper.server.ServerConfig;
 import org.apache.zookeeper.server.ZooKeeperServerMain;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 import org.apache.zookeeper.server.quorum.QuorumPeerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
 
 /**
  * HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage
@@ -53,28 +56,24 @@
   private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
 
   /**
-   * Parse ZooKeeper configuration and run a QuorumPeer.
-   * While parsing the zoo.cfg, we substitute variables with values from
-   * hbase-site.xml.
+   * Parse ZooKeeper configuration from HBase XML config and run a QuorumPeer.
    * @param args String[] of command line arguments. Not used.
-   * @throws IOException
    */
-  public static void main(String[] args) throws IOException {
+  public static void main(String[] args) {
     HBaseConfiguration conf = new HBaseConfiguration();
-    Properties zkProperties = makeZKProps(conf);
-
-    QuorumPeerConfig zkConfig = new QuorumPeerConfig();
     try {
+      Properties zkProperties = makeZKProps(conf);
+      writeMyID(zkProperties);
+      QuorumPeerConfig zkConfig = new QuorumPeerConfig();
       zkConfig.parseProperties(zkProperties);
-    } catch (ConfigException e) {
+      runZKServer(zkConfig);
+    } catch (Exception e) {
       e.printStackTrace();
       System.exit(-1);
     }
-
-    startZKServer(zkConfig);
   }
 
-  private static void startZKServer(QuorumPeerConfig zkConfig) throws IOException {
+  private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException,
IOException {
     if (zkConfig.isDistributed()) {
       QuorumPeerMain qp = new QuorumPeerMain();
       qp.runFromConfig(zkConfig);
@@ -86,6 +85,48 @@
     }
   }
 
+  private static void writeMyID(Properties properties) throws UnknownHostException, IOException
{
+    HBaseConfiguration conf = new HBaseConfiguration();
+    String myAddress = DNS.getDefaultHost(
+        conf.get("hbase.zookeeper.dns.interface","default"),
+        conf.get("hbase.zookeeper.dns.nameserver","default"));
+
+    long myId = -1;
+
+    for (Entry<Object, Object> entry : properties.entrySet()) {
+      String key = entry.getKey().toString().trim();
+      String value = entry.getValue().toString().trim();
+      if (key.startsWith("server.")) {
+        int dot = key.indexOf('.');
+        long id = Long.parseLong(key.substring(dot + 1));
+        String[] parts = value.split(":");
+        String address = parts[0];
+        if (myAddress.equals(address)) {
+          myId = id;
+          break;
+        }
+      }
+    }
+
+    if (myId == -1) {
+      throw new IOException("Could not find my address: " + myAddress +
+                            " in list of ZooKeeper quorum servers");
+    }
+
+    String dataDirStr = properties.get("dataDir").toString().trim();
+    File dataDir = new File(dataDirStr);
+    if (!dataDir.isDirectory()) {
+      if (!dataDir.mkdirs()) {
+        throw new IOException("Unable to create data dir " + dataDir);
+      }
+    }
+
+    File myIdFile = new File(dataDir, "myid");
+    PrintWriter w = new PrintWriter(myIdFile);
+    w.println(myId);
+    w.close();
+  }
+
   /**
    * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
    * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
@@ -193,11 +234,11 @@
       if (key.startsWith("server.")) {
         if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) &&
             value.startsWith("localhost")) {
-           String msg = "The server in zoo.cfg cannot be set to localhost " +
+          String msg = "The server in zoo.cfg cannot be set to localhost " +
               "in a fully-distributed setup because it won't be reachable. " +
               "See \"Getting Started\" for more information.";
-           LOG.fatal(msg);
-           throw new IOException(msg);
+          LOG.fatal(msg);
+          throw new IOException(msg);
         }
       }
       newValue.append(value.substring(varEnd));

Added: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java?rev=792636&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java (added)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java Thu Jul
 9 19:04:43 2009
@@ -0,0 +1,33 @@
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.util.Properties;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+
+/**
+ * Tool for reading ZooKeeper servers from HBase XML configuation and producing
+ * a line-by-line list for use by bash scripts.
+ */
+public class ZKServerTool implements HConstants {
+  /**
+   * Run the tool.
+   * @param args Command line arguments. First arg is path to zookeepers file.
+   */
+  public static void main(String args[]) {
+    HBaseConfiguration conf = new HBaseConfiguration();
+    // Note that we do not simply grab the property ZOOKEEPER_QUORUM from
+    // the HBaseConfiguration because the user may be using a zoo.cfg file.
+    Properties zkProps = HQuorumPeer.makeZKProps(conf);
+    for (Entry<Object, Object> entry : zkProps.entrySet()) {
+      String key = entry.getKey().toString().trim();
+      String value = entry.getValue().toString().trim();
+      if (key.startsWith("server.")) {
+        String[] parts = value.split(":");
+        String host = parts[0];
+        System.out.println(host);
+      }
+    }
+  }
+}

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java?rev=792636&r1=792635&r2=792636&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java Thu
Jul  9 19:04:43 2009
@@ -25,7 +25,6 @@
 import java.util.Properties;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;



Mime
View raw message