hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject svn commit: r791037 - in /hadoop/hbase/trunk: ./ conf/ src/contrib/stargate/ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/zookeeper/ src/test/ src/test/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/zookeeper/
Date Fri, 03 Jul 2009 23:33:29 GMT
Author: nitay
Date: Fri Jul  3 23:33:28 2009
New Revision: 791037

URL: http://svn.apache.org/viewvc?rev=791037&view=rev
Log:
HBASE-1606 Remove zoo.cfg, put config options into hbase-site.xml

Removed:
    hadoop/hbase/trunk/conf/zoo.cfg
    hadoop/hbase/trunk/src/test/zoo.cfg
Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/build.xml
    hadoop/hbase/trunk/conf/hbase-default.xml
    hadoop/hbase/trunk/src/contrib/stargate/build.xml
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
    hadoop/hbase/trunk/src/test/hbase-site.xml
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Fri Jul  3 23:33:28 2009
@@ -438,6 +438,7 @@
    HBASE-1607  Redo MemStore heap sizing to be accurate, testable, and more
                like new LruBlockCache (Jon Gray via Stack)
    HBASE-1218  Implement in-memory column (Jon Gray via Stack)
+   HBASE-1606  Remove zoo.cfg, put config options into hbase-site.xml
 
   OPTIMIZATIONS
    HBASE-1412  Change values for delete column and column family in KeyValue

Modified: hadoop/hbase/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/build.xml?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/build.xml (original)
+++ hadoop/hbase/trunk/build.xml Fri Jul  3 23:33:28 2009
@@ -185,7 +185,7 @@
       <fileset dir="${conf.dir}" >
         <include name="hbase-default.xml" />
       </fileset>
-      <zipfileset dir="conf" prefix="conf" includes="zoo.cfg,hbase-default.xml" />
+      <zipfileset dir="conf" prefix="conf" includes="hbase-default.xml" />
       <zipfileset dir="${build.webapps}" prefix="webapps"/>
    		<manifest>
             <attribute name="Main-Class" value="org/apache/hadoop/hbase/mapreduce/Driver"
/>

Modified: hadoop/hbase/trunk/conf/hbase-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/conf/hbase-default.xml?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/conf/hbase-default.xml (original)
+++ hadoop/hbase/trunk/conf/hbase-default.xml Fri Jul  3 23:33:28 2009
@@ -396,7 +396,7 @@
     <value>/hbase</value>
     <description>Root ZNode for HBase in ZooKeeper. All of HBase's ZooKeeper
       files that are configured with a relative path will go under this node.
-      By default, all of HBase's ZooKeeper file patsh are configured with a
+      By default, all of HBase's ZooKeeper file path are configured with a
       relative path, so they will all go under this directory unless changed.
     </description>
   </property>
@@ -419,4 +419,95 @@
       mode flag is stored at /hbase/safe-mode.
     </description>
   </property>
+
+  <!--
+  The following three properties are used together to create the list of
+  host:peer_port:leader_port quorum servers for ZooKeeper.
+  -->
+  <property>
+    <name>hbase.zookeeper.quorum</name>
+    <value>localhost</value>
+    <description>Comma separated list of servers in the ZooKeeper Quorum.
+    For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+    By default this is set to localhost for local and pseudo-distributed modes
+    of operation. For a fully-distributed setup, this should be set to a full
+    list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
+    this is the list of servers which we will start/stop ZooKeeper on.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.peerport</name>
+    <value>2888</value>
+    <description>Port used by ZooKeeper peers to talk to each other.
+    See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper
+    for more information.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.leaderport</name>
+    <value>3888</value>
+    <description>Port used by ZooKeeper for leader election.
+    See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper
+    for more information.
+    </description>
+  </property>
+  <!-- End of properties used to generate ZooKeeper host:port quorum list. -->
+
+  <!--
+  Beginning of properties that are directly mapped from ZooKeeper's zoo.cfg.
+  All properties with an "hbase.zookeeper.property." prefix are converted for
+  ZooKeeper's configuration. Hence, if you want to add an option from zoo.cfg,
+  e.g.  "initLimit=10" you would append the following to your configuration:
+    <property>
+      <name>hbase.zookeeper.property.initLimit</name>
+      <value>10</value>
+    </property>
+  -->
+  <property>
+    <name>hbase.zookeeper.property.tickTime</name>
+    <value>2000</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The number of milliseconds of each tick.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.initLimit</name>
+    <value>10</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The number of ticks that the initial synchronization phase can take.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.syncLimit</name>
+    <value>5</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The number of ticks that can pass between sending a request and getting an
+    acknowledgment.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.dataDir</name>
+    <value>${hbase.tmp.dir}/zookeeper</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The directory where the snapshot is stored.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.clientPort</name>
+    <value>2181</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The port at which the clients will connect.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.maxClientCnxns</name>
+    <value>30</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    Limit on number of concurrent connections (at the socket level) that a
+    single client, identified by IP address, may make to a single member of
+    the ZooKeeper ensemble. Set high to avoid zk connection issues running
+    standalone and pseudo-distributed.
+    </description>
+  </property>
+  <!-- End of properties that are directly mapped from ZooKeeper's zoo.cfg -->
 </configuration>

Modified: hadoop/hbase/trunk/src/contrib/stargate/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/contrib/stargate/build.xml?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/contrib/stargate/build.xml (original)
+++ hadoop/hbase/trunk/src/contrib/stargate/build.xml Fri Jul  3 23:33:28 2009
@@ -73,11 +73,6 @@
       <fileset dir="${lib.dir}"/>
       <mapper type="flatten"/>
     </copy>
-    <copy todir="${build.war.classes}" overwrite="true">
-      <fileset dir="${conf.dir}">
-        <include name="zoo.cfg"/>
-      </fileset>
-    </copy>
     
     <war destfile="${build.dir}/${war.file}" webxml="${web.xml.file}">
       <lib dir="${build.dir}/lib"/>

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java Fri Jul  3 23:33:28
2009
@@ -75,6 +75,9 @@
   /** default port for master web api */
   static final int DEFAULT_MASTER_INFOPORT = 60010;
 
+  /** Name of ZooKeeper quorum configuration parameter. */
+  static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
+
   /** Name of ZooKeeper config file in conf/ directory. */
   static final String ZOOKEEPER_CONFIG_NAME = "zoo.cfg";
 

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java Fri Jul
 3 23:33:28 2009
@@ -32,6 +32,7 @@
 import org.apache.zookeeper.server.ZooKeeperServerMain;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
 
 /**
  * HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage
@@ -42,61 +43,113 @@
  */
 public class HQuorumPeer implements HConstants {
   private static final Log LOG = LogFactory.getLog(HQuorumPeer.class);
+
   private static final String VARIABLE_START = "${";
   private static final int VARIABLE_START_LENGTH = VARIABLE_START.length();
   private static final String VARIABLE_END = "}";
   private static final int VARIABLE_END_LENGTH = VARIABLE_END.length();
 
+  private static final String ZK_CFG_PROPERTY = "hbase.zookeeper.property.";
+  private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
+
   /**
    * Parse ZooKeeper configuration and run a QuorumPeer.
    * While parsing the zoo.cfg, we substitute variables with values from
    * hbase-site.xml.
    * @param args String[] of command line arguments. Not used.
-   * @throws IOException 
+   * @throws IOException
    */
   public static void main(String[] args) throws IOException {
-    QuorumPeerConfig config = new QuorumPeerConfig();
+    HBaseConfiguration conf = new HBaseConfiguration();
+    Properties zkProperties = makeZKProps(conf);
+
+    QuorumPeerConfig zkConfig = new QuorumPeerConfig();
     try {
-      Properties properties = parseZooKeeperConfig();
-      config.parseProperties(properties);
-    } catch (Exception e) {
+      zkConfig.parseProperties(zkProperties);
+    } catch (ConfigException e) {
       e.printStackTrace();
       System.exit(-1);
     }
-    if (config.isDistributed()) {
+
+    startZKServer(zkConfig);
+  }
+
+  private static void startZKServer(QuorumPeerConfig zkConfig) throws IOException {
+    if (zkConfig.isDistributed()) {
       QuorumPeerMain qp = new QuorumPeerMain();
-      qp.runFromConfig(config);
+      qp.runFromConfig(zkConfig);
     } else {
       ZooKeeperServerMain zk = new ZooKeeperServerMain();
       ServerConfig serverConfig = new ServerConfig();
-      serverConfig.readFrom(config);
+      serverConfig.readFrom(zkConfig);
       zk.runFromConfig(serverConfig);
     }
   }
 
   /**
-   * Parse ZooKeeper's zoo.cfg, injecting HBase Configuration variables in.
-   * @return Properties parsed from config stream with variables substituted.
-   * @throws IOException if anything goes wrong parsing config
+   * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
+   * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
+   * the corresponding config options from the HBase XML configs and generate
+   * the appropriate ZooKeeper properties.
+   * @param conf HBaseConfiguration to read from.
+   * @return Properties holding mappings representing ZooKeeper zoo.cfg file.
    */
-  public static Properties parseZooKeeperConfig() throws IOException {
+  public static Properties makeZKProps(HBaseConfiguration conf) {
+    // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
+    // it and grab its configuration properties.
     ClassLoader cl = HQuorumPeer.class.getClassLoader();
     InputStream inputStream = cl.getResourceAsStream(ZOOKEEPER_CONFIG_NAME);
-    if (inputStream == null) {
-      throw new IOException(ZOOKEEPER_CONFIG_NAME + " not found");
+    if (inputStream != null) {
+      try {
+        return parseZooCfg(conf, inputStream);
+      } catch (IOException e) {
+        LOG.warn("Cannot read " + ZOOKEEPER_CONFIG_NAME +
+                 ", loading from XML files", e);
+      }
     }
-    return parseConfig(inputStream);
+
+    // Otherwise, use the configuration options from HBase's XML files.
+    Properties zkProperties = new Properties();
+
+    // Directly map all of the hbase.zookeeper.property.KEY properties.
+    for (Entry<String, String> entry : conf) {
+      String key = entry.getKey();
+      if (key.startsWith(ZK_CFG_PROPERTY)) {
+        String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE);
+        String value = entry.getValue();
+        // If the value has variables substitutions, need to do a get.
+        if (value.contains(VARIABLE_START)) {
+          value = conf.get(key);
+        }
+        zkProperties.put(zkKey, value);
+      }
+    }
+
+    // Create the server.X properties.
+    int peerPort = conf.getInt("hbase.zookeeper.peerport", 2888);
+    int leaderPort = conf.getInt("hbase.zookeeper.leaderport", 3888);
+
+    String[] serverHosts = conf.getStrings(ZOOKEEPER_QUORUM, "localhost");
+    for (int i = 0; i < serverHosts.length; ++i) {
+      String serverHost = serverHosts[i];
+      String address = serverHost + ":" + peerPort + ":" + leaderPort;
+      String key = "server." + i;
+      zkProperties.put(key, address);
+    }
+
+    return zkProperties;
   }
 
   /**
    * Parse ZooKeeper's zoo.cfg, injecting HBase Configuration variables in.
    * This method is used for testing so we can pass our own InputStream.
+   * @param conf HBaseConfiguration to use for injecting variables.
    * @param inputStream InputStream to read from.
    * @return Properties parsed from config stream with variables substituted.
    * @throws IOException if anything goes wrong parsing config
    */
-  public static Properties parseConfig(InputStream inputStream) throws IOException {
-    HBaseConfiguration conf = new HBaseConfiguration();
+  public static Properties parseZooCfg(HBaseConfiguration conf,
+      InputStream inputStream) throws IOException {
     Properties properties = new Properties();
     try {
       properties.load(inputStream);
@@ -130,7 +183,7 @@
           LOG.fatal(msg);
           throw new IOException(msg);
         }
-        
+
         newValue.append(substituteValue);
 
         varEnd += VARIABLE_END_LENGTH;
@@ -138,7 +191,7 @@
       }
       // Special case for 'hbase.cluster.distributed' property being 'true'
       if (key.startsWith("server.")) {
-        if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) && 
+        if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) &&
             value.startsWith("localhost")) {
            String msg = "The server in zoo.cfg cannot be set to localhost " +
               "in a fully-distributed setup because it won't be reachable. " +

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java Fri
Jul  3 23:33:28 2009
@@ -56,18 +56,15 @@
   // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
   private static final char ZNODE_PATH_SEPARATOR = '/';
 
-  private static String quorumServers = null;
-  static {
-    loadZooKeeperConfig();
-  }
+  private String quorumServers = null;
 
   private final ZooKeeper zooKeeper;
 
   private final String parentZNode;
-  public final String rootRegionZNode;
-  public final String outOfSafeModeZNode;
-  public final String rsZNode;
-  public final String masterElectionZNode;
+  private final String rootRegionZNode;
+  private final String outOfSafeModeZNode;
+  private final String rsZNode;
+  private final String masterElectionZNode;
   public final String clusterStateZNode;
 
   /**
@@ -78,6 +75,8 @@
    */
   public ZooKeeperWrapper(HBaseConfiguration conf, Watcher watcher)
   throws IOException {
+    Properties properties = HQuorumPeer.makeZKProps(conf);
+    setQuorumServers(properties);
     if (quorumServers == null) {
       throw new IOException("Could not read quorum servers from " +
                             ZOOKEEPER_CONFIG_NAME);
@@ -110,76 +109,7 @@
     clusterStateZNode = getZNode(parentZNode, stateZNodeName);
   }
 
-  /** @return String dump of everything in ZooKeeper. */
-  public String dump() {
-    StringBuilder sb = new StringBuilder();
-    sb.append("\nHBase tree in ZooKeeper is rooted at ").append(parentZNode);
-    sb.append("\n  Cluster up? ").append(exists(clusterStateZNode));
-    sb.append("\n  In safe mode? ").append(!checkOutOfSafeMode());
-    sb.append("\n  Master address: ").append(readMasterAddress(null));
-    sb.append("\n  Region server holding ROOT: ").append(readRootRegionLocation());
-    sb.append("\n  Region servers:");
-    for (HServerAddress address : scanRSDirectory()) {
-      sb.append("\n    - ").append(address);
-    }
-    return sb.toString();
-  }
-
-  private boolean exists(String znode) {
-    try {
-      return zooKeeper.exists(znode, null) != null;
-    } catch (KeeperException e) {
-      return false;
-    } catch (InterruptedException e) {
-      return false;
-    }
-  }
-
-  /** @return ZooKeeper used by this wrapper. */
-  public ZooKeeper getZooKeeper() {
-    return zooKeeper;
-  }
-
-  /**
-   * This is for testing KeeperException.SessionExpiredExcseption.
-   * See HBASE-1232.
-   * @return long session ID of this ZooKeeper session.
-   */
-  public long getSessionID() {
-    return zooKeeper.getSessionId();
-  }
-
-  /**
-   * This is for testing KeeperException.SessionExpiredExcseption.
-   * See HBASE-1232.
-   * @return byte[] password of this ZooKeeper session.
-   */
-  public byte[] getSessionPassword() {
-    return zooKeeper.getSessionPasswd();
-  }
-
-  /**
-   * This is for tests to directly set the ZooKeeper quorum servers.
-   * @param servers comma separated host:port ZooKeeper quorum servers.
-   */
-  public static void setQuorumServers(String servers) {
-    quorumServers = servers;
-  }
-
-  /** @return comma separated host:port list of ZooKeeper quorum servers. */
-  public static String getQuorumServers() {
-    return quorumServers;
-  }
-
-  private static void loadZooKeeperConfig() {
-    Properties properties = null;
-    try {
-      properties = HQuorumPeer.parseZooKeeperConfig();
-    } catch (IOException e) {
-      LOG.fatal("Fail to read properties from " + ZOOKEEPER_CONFIG_NAME, e);
-      System.exit(-1);
-    }
-
+  private void setQuorumServers(Properties properties) {
     String clientPort = null;
     List<String> servers = new ArrayList<String>();
 
@@ -217,7 +147,7 @@
     if (servers.isEmpty()) {
       LOG.fatal("No server.X lines found in conf/zoo.cfg. HBase must have a " +
                 "ZooKeeper cluster configured for its operation.");
-      System.exit(-1);
+      return;
     }
 
     StringBuilder hostPortBuilder = new StringBuilder();
@@ -232,7 +162,59 @@
     }
 
     quorumServers = hostPortBuilder.toString();
-    LOG.info("Quorum servers: " + quorumServers);
+  }
+
+  /** @return String dump of everything in ZooKeeper. */
+  public String dump() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("\nHBase tree in ZooKeeper is rooted at ").append(parentZNode);
+    sb.append("\n  Cluster up? ").append(exists(clusterStateZNode));
+    sb.append("\n  In safe mode? ").append(!checkOutOfSafeMode());
+    sb.append("\n  Master address: ").append(readMasterAddress(null));
+    sb.append("\n  Region server holding ROOT: ").append(readRootRegionLocation());
+    sb.append("\n  Region servers:");
+    for (HServerAddress address : scanRSDirectory()) {
+      sb.append("\n    - ").append(address);
+    }
+    return sb.toString();
+  }
+
+  private boolean exists(String znode) {
+    try {
+      return zooKeeper.exists(znode, null) != null;
+    } catch (KeeperException e) {
+      return false;
+    } catch (InterruptedException e) {
+      return false;
+    }
+  }
+
+  /** @return ZooKeeper used by this wrapper. */
+  public ZooKeeper getZooKeeper() {
+    return zooKeeper;
+  }
+
+  /**
+   * This is for testing KeeperException.SessionExpiredException.
+   * See HBASE-1232.
+   * @return long session ID of this ZooKeeper session.
+   */
+  public long getSessionID() {
+    return zooKeeper.getSessionId();
+  }
+
+  /**
+   * This is for testing KeeperException.SessionExpiredException.
+   * See HBASE-1232.
+   * @return byte[] password of this ZooKeeper session.
+   */
+  public byte[] getSessionPassword() {
+    return zooKeeper.getSessionPasswd();
+  }
+
+  /** @return host:port list of quorum servers. */
+  public String getQuorumServers() {
+    return quorumServers;
   }
 
   /** @return true if currently connected to ZooKeeper, false otherwise. */

Modified: hadoop/hbase/trunk/src/test/hbase-site.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/hbase-site.xml?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/hbase-site.xml (original)
+++ hadoop/hbase/trunk/src/test/hbase-site.xml Fri Jul  3 23:33:28 2009
@@ -127,4 +127,11 @@
     <name>hadoop.log.dir</name>
     <value>${user.dir}/../logs</value>
   </property>
+  <property>
+    <name>hbase.zookeeper.property.clientPort</name>
+    <value>21810</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The port at which the clients will connect.
+    </description>
+  </property>
 </configuration>

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java Fri Jul
 3 23:33:28 2009
@@ -96,7 +96,8 @@
     // Note that this is done before we create the MiniHBaseCluster because we
     // need to edit the config to add the ZooKeeper servers.
     this.zooKeeperCluster = new MiniZooKeeperCluster();
-    this.zooKeeperCluster.startup(testDir);
+    int clientPort = this.zooKeeperCluster.startup(testDir);
+    conf.set("hbase.zookeeper.property.clientPort", Integer.toString(clientPort));
 
     // start the mini cluster
     this.cluster = new MiniHBaseCluster(conf, regionServers);

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java Fri Jul
 3 23:33:28 2009
@@ -70,10 +70,11 @@
 
   /**
    * @param baseDir
+   * @return ClientPort server bound to.
    * @throws IOException
    * @throws InterruptedException
    */
-  public void startup(File baseDir) throws IOException,
+  public int startup(File baseDir) throws IOException,
       InterruptedException {
     setupTestEnv();
 
@@ -96,14 +97,13 @@
     }
     standaloneServerFactory.startup(server);
 
-    String quorumServers = "localhost:" + clientPort;
-    ZooKeeperWrapper.setQuorumServers(quorumServers);
-
     if (!waitForServerUp(clientPort, CONNECTION_TIMEOUT)) {
       throw new IOException("Waiting for startup of standalone server");
     }
 
     started = true;
+
+    return clientPort;
   }
 
   private void recreateDir(File dir) throws IOException {

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java Fri Jul  3 23:33:28
2009
@@ -26,7 +26,6 @@
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -40,7 +39,8 @@
  */
 public class TestZooKeeper extends HBaseClusterTestCase {
   private static class EmptyWatcher implements Watcher {
-    public EmptyWatcher() {}
+    public static EmptyWatcher instance = new EmptyWatcher();
+    private EmptyWatcher() {}
     public void process(WatchedEvent event) {}
   }
 
@@ -54,7 +54,7 @@
    * @throws IOException
    */
   public void testWritesRootRegionLocation() throws IOException {
-    ZooKeeperWrapper zooKeeper = new ZooKeeperWrapper(conf, new EmptyWatcher());
+    ZooKeeperWrapper zooKeeper = new ZooKeeperWrapper(conf, EmptyWatcher.instance);
 
     boolean outOfSafeMode = zooKeeper.checkOutOfSafeMode();
     assertFalse(outOfSafeMode);
@@ -82,9 +82,11 @@
    * @throws IOException
    */
   public void testParentExists() throws IOException {
+    String oldValue = conf.get("zookeeper.znode.safemode");
     conf.set("zookeeper.znode.safemode", "/a/b/c/d/e");
-    ZooKeeperWrapper zooKeeper = new ZooKeeperWrapper(conf, new EmptyWatcher());
+    ZooKeeperWrapper zooKeeper = new ZooKeeperWrapper(conf, EmptyWatcher.instance);
     assertTrue(zooKeeper.writeOutOfSafeMode());
+    conf.set("zookeeper.znode.safemode", oldValue);
   }
 
   /**
@@ -95,15 +97,15 @@
   public void testClientSessionExpired() throws IOException, InterruptedException {
     new HTable(conf, HConstants.META_TABLE_NAME);
 
-    String quorumServers = ZooKeeperWrapper.getQuorumServers();
+    ZooKeeperWrapper zkw = new ZooKeeperWrapper(conf, EmptyWatcher.instance);
+    String quorumServers = zkw.getQuorumServers();
     int sessionTimeout = conf.getInt("zookeeper.session.timeout", 2 * 1000);
-    Watcher watcher = new EmptyWatcher();
     HConnection connection = HConnectionManager.getConnection(conf);
     ZooKeeperWrapper connectionZK = connection.getZooKeeperWrapper();
     long sessionID = connectionZK.getSessionID();
     byte[] password = connectionZK.getSessionPassword();
 
-    ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, watcher, sessionID, password);
+    ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance, sessionID,
password);
     zk.close();
 
     Thread.sleep(sessionTimeout * 3);
@@ -119,16 +121,16 @@
     try {
       new HTable(conf, HConstants.META_TABLE_NAME);
   
-      String quorumServers = ZooKeeperWrapper.getQuorumServers();
+      ZooKeeperWrapper zkw = new ZooKeeperWrapper(conf, EmptyWatcher.instance);
+      String quorumServers = zkw.getQuorumServers();
       int sessionTimeout = conf.getInt("zookeeper.session.timeout", 2 * 1000);
-  
-      Watcher watcher = new EmptyWatcher();
+
       HRegionServer rs = cluster.getRegionServer(0);
       ZooKeeperWrapper rsZK = rs.getZooKeeperWrapper();
       long sessionID = rsZK.getSessionID();
       byte[] password = rsZK.getSessionPassword();
   
-      ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, watcher, sessionID, password);
+      ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance,
sessionID, password);
       zk.close();
 
       Thread.sleep(sessionTimeout * 3);

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java?rev=791037&r1=791036&r2=791037&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java Fri
Jul  3 23:33:28 2009
@@ -25,7 +25,9 @@
 import java.util.Properties;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
 
@@ -59,10 +61,34 @@
     super.tearDown();
   }
 
-  /** @throws Exception */
-  public void testConfigInjection() throws Exception {
+  /** */
+  public void testMakeZKProps() {
+    Properties properties = HQuorumPeer.makeZKProps(conf);
+    assertEquals(2000, Integer.parseInt(properties.getProperty("tickTime")));
+    assertEquals(Integer.valueOf(10), Integer.valueOf(properties.getProperty("initLimit")));
+    assertEquals(Integer.valueOf(5), Integer.valueOf(properties.getProperty("syncLimit")));
+    assertEquals(dataDir.toString(), properties.get("dataDir"));
+    assertEquals(Integer.valueOf(21810), Integer.valueOf(properties.getProperty("clientPort")));
+    assertEquals("localhost:2888:3888", properties.get("server.0"));
+    assertEquals(null, properties.get("server.1"));
 
+    String oldValue = conf.get(HConstants.ZOOKEEPER_QUORUM);
+    conf.set(HConstants.ZOOKEEPER_QUORUM, "a.foo.bar,b.foo.bar,c.foo.bar");
+    properties = HQuorumPeer.makeZKProps(conf);
+    assertEquals(2000, Integer.parseInt(properties.getProperty("tickTime")));
+    assertEquals(Integer.valueOf(10), Integer.valueOf(properties.getProperty("initLimit")));
+    assertEquals(Integer.valueOf(5), Integer.valueOf(properties.getProperty("syncLimit")));
+    assertEquals(dataDir.toString(), properties.get("dataDir"));
+    assertEquals(Integer.valueOf(21810), Integer.valueOf(properties.getProperty("clientPort")));
+    assertEquals("a.foo.bar:2888:3888", properties.get("server.0"));
+    assertEquals("b.foo.bar:2888:3888", properties.get("server.1"));
+    assertEquals("c.foo.bar:2888:3888", properties.get("server.2"));
+    assertEquals(null, properties.get("server.3"));
+    conf.set(HConstants.ZOOKEEPER_QUORUM, oldValue);
+  }
 
+  /** @throws Exception */
+  public void testConfigInjection() throws Exception {
     String s =
       "tickTime=2000\n" +
       "initLimit=10\n" +
@@ -71,8 +97,9 @@
       "clientPort=2181\n" +
       "server.0=${hbase.master.hostname}:2888:3888\n";
 
+    System.setProperty("hbase.master.hostname", "localhost");
     InputStream is = new ByteArrayInputStream(s.getBytes());
-    Properties properties = HQuorumPeer.parseConfig(is);
+    Properties properties = HQuorumPeer.parseZooCfg(conf, is);
 
     assertEquals(Integer.valueOf(2000), Integer.valueOf(properties.getProperty("tickTime")));
     assertEquals(Integer.valueOf(10), Integer.valueOf(properties.getProperty("initLimit")));
@@ -101,7 +128,7 @@
     // Override with system property.
     System.setProperty("hbase.master.hostname", "foo.bar");
     is = new ByteArrayInputStream(s.getBytes());
-    properties = HQuorumPeer.parseConfig(is);
+    properties = HQuorumPeer.parseZooCfg(conf, is);
     assertEquals("foo.bar:2888:3888", properties.get("server.0"));
 
     config.parseProperties(properties);
@@ -109,17 +136,5 @@
     servers = config.getServers();
     server = servers.get(Long.valueOf(0));
     assertEquals("foo.bar", server.addr.getHostName());
-
-    // Special case for property 'hbase.master.hostname' being 'local'
-    System.setProperty("hbase.master.hostname", "local");
-    is = new ByteArrayInputStream(s.getBytes());
-    properties = HQuorumPeer.parseConfig(is);
-    assertEquals("localhost:2888:3888", properties.get("server.0"));
-
-    config.parseProperties(properties);
-
-    servers = config.getServers();
-    server = servers.get(Long.valueOf(0));
-    assertEquals("localhost", server.addr.getHostName());
   }
 }



Mime
View raw message