Return-Path: Delivered-To: apmail-hadoop-hbase-commits-archive@minotaur.apache.org Received: (qmail 58587 invoked from network); 3 Jul 2009 23:33:50 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 3 Jul 2009 23:33:50 -0000 Received: (qmail 32173 invoked by uid 500); 3 Jul 2009 23:34:01 -0000 Delivered-To: apmail-hadoop-hbase-commits-archive@hadoop.apache.org Received: (qmail 32128 invoked by uid 500); 3 Jul 2009 23:34:00 -0000 Mailing-List: contact hbase-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hbase-dev@hadoop.apache.org Delivered-To: mailing list hbase-commits@hadoop.apache.org Received: (qmail 32119 invoked by uid 99); 3 Jul 2009 23:34:00 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 03 Jul 2009 23:34:00 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 03 Jul 2009 23:33:50 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 018CC23888E5; Fri, 3 Jul 2009 23:33:30 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r791037 - in /hadoop/hbase/trunk: ./ conf/ src/contrib/stargate/ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/zookeeper/ src/test/ src/test/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/zookeeper/ Date: Fri, 03 Jul 2009 23:33:29 -0000 To: hbase-commits@hadoop.apache.org From: nitay@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090703233330.018CC23888E5@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: nitay Date: Fri Jul 3 23:33:28 2009 New Revision: 791037 URL: http://svn.apache.org/viewvc?rev=791037&view=rev Log: HBASE-1606 Remove zoo.cfg, put config options into hbase-site.xml Removed: hadoop/hbase/trunk/conf/zoo.cfg hadoop/hbase/trunk/src/test/zoo.cfg Modified: hadoop/hbase/trunk/CHANGES.txt hadoop/hbase/trunk/build.xml hadoop/hbase/trunk/conf/hbase-default.xml hadoop/hbase/trunk/src/contrib/stargate/build.xml hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java hadoop/hbase/trunk/src/test/hbase-site.xml hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java Modified: hadoop/hbase/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/CHANGES.txt (original) +++ hadoop/hbase/trunk/CHANGES.txt Fri Jul 3 23:33:28 2009 @@ -438,6 +438,7 @@ HBASE-1607 Redo MemStore heap sizing to be accurate, testable, and more like new LruBlockCache (Jon Gray via Stack) HBASE-1218 Implement in-memory column (Jon Gray via Stack) + HBASE-1606 Remove zoo.cfg, put config options into hbase-site.xml OPTIMIZATIONS HBASE-1412 Change values for delete column and column family in KeyValue Modified: hadoop/hbase/trunk/build.xml URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/build.xml?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/build.xml (original) +++ hadoop/hbase/trunk/build.xml Fri Jul 3 23:33:28 2009 @@ -185,7 +185,7 @@ - + Modified: hadoop/hbase/trunk/conf/hbase-default.xml URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/conf/hbase-default.xml?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/conf/hbase-default.xml (original) +++ hadoop/hbase/trunk/conf/hbase-default.xml Fri Jul 3 23:33:28 2009 @@ -396,7 +396,7 @@ /hbase Root ZNode for HBase in ZooKeeper. All of HBase's ZooKeeper files that are configured with a relative path will go under this node. - By default, all of HBase's ZooKeeper file patsh are configured with a + By default, all of HBase's ZooKeeper file path are configured with a relative path, so they will all go under this directory unless changed. @@ -419,4 +419,95 @@ mode flag is stored at /hbase/safe-mode. + + + + hbase.zookeeper.quorum + localhost + Comma separated list of servers in the ZooKeeper Quorum. + For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". + By default this is set to localhost for local and pseudo-distributed modes + of operation. For a fully-distributed setup, this should be set to a full + list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh + this is the list of servers which we will start/stop ZooKeeper on. + + + + hbase.zookeeper.peerport + 2888 + Port used by ZooKeeper peers to talk to each other. + See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper + for more information. + + + + hbase.zookeeper.leaderport + 3888 + Port used by ZooKeeper for leader election. + See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper + for more information. + + + + + + + hbase.zookeeper.property.tickTime + 2000 + Property from ZooKeeper's config zoo.cfg. + The number of milliseconds of each tick. + + + + hbase.zookeeper.property.initLimit + 10 + Property from ZooKeeper's config zoo.cfg. + The number of ticks that the initial synchronization phase can take. + + + + hbase.zookeeper.property.syncLimit + 5 + Property from ZooKeeper's config zoo.cfg. + The number of ticks that can pass between sending a request and getting an + acknowledgment. + + + + hbase.zookeeper.property.dataDir + ${hbase.tmp.dir}/zookeeper + Property from ZooKeeper's config zoo.cfg. + The directory where the snapshot is stored. + + + + hbase.zookeeper.property.clientPort + 2181 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + + + hbase.zookeeper.property.maxClientCnxns + 30 + Property from ZooKeeper's config zoo.cfg. + Limit on number of concurrent connections (at the socket level) that a + single client, identified by IP address, may make to a single member of + the ZooKeeper ensemble. Set high to avoid zk connection issues running + standalone and pseudo-distributed. + + + Modified: hadoop/hbase/trunk/src/contrib/stargate/build.xml URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/contrib/stargate/build.xml?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/src/contrib/stargate/build.xml (original) +++ hadoop/hbase/trunk/src/contrib/stargate/build.xml Fri Jul 3 23:33:28 2009 @@ -73,11 +73,6 @@ - - - - - Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java Fri Jul 3 23:33:28 2009 @@ -75,6 +75,9 @@ /** default port for master web api */ static final int DEFAULT_MASTER_INFOPORT = 60010; + /** Name of ZooKeeper quorum configuration parameter. */ + static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + /** Name of ZooKeeper config file in conf/ directory. */ static final String ZOOKEEPER_CONFIG_NAME = "zoo.cfg"; Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java Fri Jul 3 23:33:28 2009 @@ -32,6 +32,7 @@ import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerMain; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; /** * HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage @@ -42,61 +43,113 @@ */ public class HQuorumPeer implements HConstants { private static final Log LOG = LogFactory.getLog(HQuorumPeer.class); + private static final String VARIABLE_START = "${"; private static final int VARIABLE_START_LENGTH = VARIABLE_START.length(); private static final String VARIABLE_END = "}"; private static final int VARIABLE_END_LENGTH = VARIABLE_END.length(); + private static final String ZK_CFG_PROPERTY = "hbase.zookeeper.property."; + private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length(); + /** * Parse ZooKeeper configuration and run a QuorumPeer. * While parsing the zoo.cfg, we substitute variables with values from * hbase-site.xml. * @param args String[] of command line arguments. Not used. - * @throws IOException + * @throws IOException */ public static void main(String[] args) throws IOException { - QuorumPeerConfig config = new QuorumPeerConfig(); + HBaseConfiguration conf = new HBaseConfiguration(); + Properties zkProperties = makeZKProps(conf); + + QuorumPeerConfig zkConfig = new QuorumPeerConfig(); try { - Properties properties = parseZooKeeperConfig(); - config.parseProperties(properties); - } catch (Exception e) { + zkConfig.parseProperties(zkProperties); + } catch (ConfigException e) { e.printStackTrace(); System.exit(-1); } - if (config.isDistributed()) { + + startZKServer(zkConfig); + } + + private static void startZKServer(QuorumPeerConfig zkConfig) throws IOException { + if (zkConfig.isDistributed()) { QuorumPeerMain qp = new QuorumPeerMain(); - qp.runFromConfig(config); + qp.runFromConfig(zkConfig); } else { ZooKeeperServerMain zk = new ZooKeeperServerMain(); ServerConfig serverConfig = new ServerConfig(); - serverConfig.readFrom(config); + serverConfig.readFrom(zkConfig); zk.runFromConfig(serverConfig); } } /** - * Parse ZooKeeper's zoo.cfg, injecting HBase Configuration variables in. - * @return Properties parsed from config stream with variables substituted. - * @throws IOException if anything goes wrong parsing config + * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg. + * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse + * the corresponding config options from the HBase XML configs and generate + * the appropriate ZooKeeper properties. + * @param conf HBaseConfiguration to read from. + * @return Properties holding mappings representing ZooKeeper zoo.cfg file. */ - public static Properties parseZooKeeperConfig() throws IOException { + public static Properties makeZKProps(HBaseConfiguration conf) { + // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read + // it and grab its configuration properties. ClassLoader cl = HQuorumPeer.class.getClassLoader(); InputStream inputStream = cl.getResourceAsStream(ZOOKEEPER_CONFIG_NAME); - if (inputStream == null) { - throw new IOException(ZOOKEEPER_CONFIG_NAME + " not found"); + if (inputStream != null) { + try { + return parseZooCfg(conf, inputStream); + } catch (IOException e) { + LOG.warn("Cannot read " + ZOOKEEPER_CONFIG_NAME + + ", loading from XML files", e); + } } - return parseConfig(inputStream); + + // Otherwise, use the configuration options from HBase's XML files. + Properties zkProperties = new Properties(); + + // Directly map all of the hbase.zookeeper.property.KEY properties. + for (Entry entry : conf) { + String key = entry.getKey(); + if (key.startsWith(ZK_CFG_PROPERTY)) { + String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE); + String value = entry.getValue(); + // If the value has variables substitutions, need to do a get. + if (value.contains(VARIABLE_START)) { + value = conf.get(key); + } + zkProperties.put(zkKey, value); + } + } + + // Create the server.X properties. + int peerPort = conf.getInt("hbase.zookeeper.peerport", 2888); + int leaderPort = conf.getInt("hbase.zookeeper.leaderport", 3888); + + String[] serverHosts = conf.getStrings(ZOOKEEPER_QUORUM, "localhost"); + for (int i = 0; i < serverHosts.length; ++i) { + String serverHost = serverHosts[i]; + String address = serverHost + ":" + peerPort + ":" + leaderPort; + String key = "server." + i; + zkProperties.put(key, address); + } + + return zkProperties; } /** * Parse ZooKeeper's zoo.cfg, injecting HBase Configuration variables in. * This method is used for testing so we can pass our own InputStream. + * @param conf HBaseConfiguration to use for injecting variables. * @param inputStream InputStream to read from. * @return Properties parsed from config stream with variables substituted. * @throws IOException if anything goes wrong parsing config */ - public static Properties parseConfig(InputStream inputStream) throws IOException { - HBaseConfiguration conf = new HBaseConfiguration(); + public static Properties parseZooCfg(HBaseConfiguration conf, + InputStream inputStream) throws IOException { Properties properties = new Properties(); try { properties.load(inputStream); @@ -130,7 +183,7 @@ LOG.fatal(msg); throw new IOException(msg); } - + newValue.append(substituteValue); varEnd += VARIABLE_END_LENGTH; @@ -138,7 +191,7 @@ } // Special case for 'hbase.cluster.distributed' property being 'true' if (key.startsWith("server.")) { - if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) && + if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) && value.startsWith("localhost")) { String msg = "The server in zoo.cfg cannot be set to localhost " + "in a fully-distributed setup because it won't be reachable. " + Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java Fri Jul 3 23:33:28 2009 @@ -56,18 +56,15 @@ // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved. private static final char ZNODE_PATH_SEPARATOR = '/'; - private static String quorumServers = null; - static { - loadZooKeeperConfig(); - } + private String quorumServers = null; private final ZooKeeper zooKeeper; private final String parentZNode; - public final String rootRegionZNode; - public final String outOfSafeModeZNode; - public final String rsZNode; - public final String masterElectionZNode; + private final String rootRegionZNode; + private final String outOfSafeModeZNode; + private final String rsZNode; + private final String masterElectionZNode; public final String clusterStateZNode; /** @@ -78,6 +75,8 @@ */ public ZooKeeperWrapper(HBaseConfiguration conf, Watcher watcher) throws IOException { + Properties properties = HQuorumPeer.makeZKProps(conf); + setQuorumServers(properties); if (quorumServers == null) { throw new IOException("Could not read quorum servers from " + ZOOKEEPER_CONFIG_NAME); @@ -110,76 +109,7 @@ clusterStateZNode = getZNode(parentZNode, stateZNodeName); } - /** @return String dump of everything in ZooKeeper. */ - public String dump() { - StringBuilder sb = new StringBuilder(); - sb.append("\nHBase tree in ZooKeeper is rooted at ").append(parentZNode); - sb.append("\n Cluster up? ").append(exists(clusterStateZNode)); - sb.append("\n In safe mode? ").append(!checkOutOfSafeMode()); - sb.append("\n Master address: ").append(readMasterAddress(null)); - sb.append("\n Region server holding ROOT: ").append(readRootRegionLocation()); - sb.append("\n Region servers:"); - for (HServerAddress address : scanRSDirectory()) { - sb.append("\n - ").append(address); - } - return sb.toString(); - } - - private boolean exists(String znode) { - try { - return zooKeeper.exists(znode, null) != null; - } catch (KeeperException e) { - return false; - } catch (InterruptedException e) { - return false; - } - } - - /** @return ZooKeeper used by this wrapper. */ - public ZooKeeper getZooKeeper() { - return zooKeeper; - } - - /** - * This is for testing KeeperException.SessionExpiredExcseption. - * See HBASE-1232. - * @return long session ID of this ZooKeeper session. - */ - public long getSessionID() { - return zooKeeper.getSessionId(); - } - - /** - * This is for testing KeeperException.SessionExpiredExcseption. - * See HBASE-1232. - * @return byte[] password of this ZooKeeper session. - */ - public byte[] getSessionPassword() { - return zooKeeper.getSessionPasswd(); - } - - /** - * This is for tests to directly set the ZooKeeper quorum servers. - * @param servers comma separated host:port ZooKeeper quorum servers. - */ - public static void setQuorumServers(String servers) { - quorumServers = servers; - } - - /** @return comma separated host:port list of ZooKeeper quorum servers. */ - public static String getQuorumServers() { - return quorumServers; - } - - private static void loadZooKeeperConfig() { - Properties properties = null; - try { - properties = HQuorumPeer.parseZooKeeperConfig(); - } catch (IOException e) { - LOG.fatal("Fail to read properties from " + ZOOKEEPER_CONFIG_NAME, e); - System.exit(-1); - } - + private void setQuorumServers(Properties properties) { String clientPort = null; List servers = new ArrayList(); @@ -217,7 +147,7 @@ if (servers.isEmpty()) { LOG.fatal("No server.X lines found in conf/zoo.cfg. HBase must have a " + "ZooKeeper cluster configured for its operation."); - System.exit(-1); + return; } StringBuilder hostPortBuilder = new StringBuilder(); @@ -232,7 +162,59 @@ } quorumServers = hostPortBuilder.toString(); - LOG.info("Quorum servers: " + quorumServers); + } + + /** @return String dump of everything in ZooKeeper. */ + public String dump() { + StringBuilder sb = new StringBuilder(); + sb.append("\nHBase tree in ZooKeeper is rooted at ").append(parentZNode); + sb.append("\n Cluster up? ").append(exists(clusterStateZNode)); + sb.append("\n In safe mode? ").append(!checkOutOfSafeMode()); + sb.append("\n Master address: ").append(readMasterAddress(null)); + sb.append("\n Region server holding ROOT: ").append(readRootRegionLocation()); + sb.append("\n Region servers:"); + for (HServerAddress address : scanRSDirectory()) { + sb.append("\n - ").append(address); + } + return sb.toString(); + } + + private boolean exists(String znode) { + try { + return zooKeeper.exists(znode, null) != null; + } catch (KeeperException e) { + return false; + } catch (InterruptedException e) { + return false; + } + } + + /** @return ZooKeeper used by this wrapper. */ + public ZooKeeper getZooKeeper() { + return zooKeeper; + } + + /** + * This is for testing KeeperException.SessionExpiredException. + * See HBASE-1232. + * @return long session ID of this ZooKeeper session. + */ + public long getSessionID() { + return zooKeeper.getSessionId(); + } + + /** + * This is for testing KeeperException.SessionExpiredException. + * See HBASE-1232. + * @return byte[] password of this ZooKeeper session. + */ + public byte[] getSessionPassword() { + return zooKeeper.getSessionPasswd(); + } + + /** @return host:port list of quorum servers. */ + public String getQuorumServers() { + return quorumServers; } /** @return true if currently connected to ZooKeeper, false otherwise. */ Modified: hadoop/hbase/trunk/src/test/hbase-site.xml URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/hbase-site.xml?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/hbase-site.xml (original) +++ hadoop/hbase/trunk/src/test/hbase-site.xml Fri Jul 3 23:33:28 2009 @@ -127,4 +127,11 @@ hadoop.log.dir ${user.dir}/../logs + + hbase.zookeeper.property.clientPort + 21810 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseClusterTestCase.java Fri Jul 3 23:33:28 2009 @@ -96,7 +96,8 @@ // Note that this is done before we create the MiniHBaseCluster because we // need to edit the config to add the ZooKeeper servers. this.zooKeeperCluster = new MiniZooKeeperCluster(); - this.zooKeeperCluster.startup(testDir); + int clientPort = this.zooKeeperCluster.startup(testDir); + conf.set("hbase.zookeeper.property.clientPort", Integer.toString(clientPort)); // start the mini cluster this.cluster = new MiniHBaseCluster(conf, regionServers); Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java Fri Jul 3 23:33:28 2009 @@ -70,10 +70,11 @@ /** * @param baseDir + * @return ClientPort server bound to. * @throws IOException * @throws InterruptedException */ - public void startup(File baseDir) throws IOException, + public int startup(File baseDir) throws IOException, InterruptedException { setupTestEnv(); @@ -96,14 +97,13 @@ } standaloneServerFactory.startup(server); - String quorumServers = "localhost:" + clientPort; - ZooKeeperWrapper.setQuorumServers(quorumServers); - if (!waitForServerUp(clientPort, CONNECTION_TIMEOUT)) { throw new IOException("Waiting for startup of standalone server"); } started = true; + + return clientPort; } private void recreateDir(File dir) throws IOException { Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestZooKeeper.java Fri Jul 3 23:33:28 2009 @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.BatchUpdate; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.util.Bytes; @@ -40,7 +39,8 @@ */ public class TestZooKeeper extends HBaseClusterTestCase { private static class EmptyWatcher implements Watcher { - public EmptyWatcher() {} + public static EmptyWatcher instance = new EmptyWatcher(); + private EmptyWatcher() {} public void process(WatchedEvent event) {} } @@ -54,7 +54,7 @@ * @throws IOException */ public void testWritesRootRegionLocation() throws IOException { - ZooKeeperWrapper zooKeeper = new ZooKeeperWrapper(conf, new EmptyWatcher()); + ZooKeeperWrapper zooKeeper = new ZooKeeperWrapper(conf, EmptyWatcher.instance); boolean outOfSafeMode = zooKeeper.checkOutOfSafeMode(); assertFalse(outOfSafeMode); @@ -82,9 +82,11 @@ * @throws IOException */ public void testParentExists() throws IOException { + String oldValue = conf.get("zookeeper.znode.safemode"); conf.set("zookeeper.znode.safemode", "/a/b/c/d/e"); - ZooKeeperWrapper zooKeeper = new ZooKeeperWrapper(conf, new EmptyWatcher()); + ZooKeeperWrapper zooKeeper = new ZooKeeperWrapper(conf, EmptyWatcher.instance); assertTrue(zooKeeper.writeOutOfSafeMode()); + conf.set("zookeeper.znode.safemode", oldValue); } /** @@ -95,15 +97,15 @@ public void testClientSessionExpired() throws IOException, InterruptedException { new HTable(conf, HConstants.META_TABLE_NAME); - String quorumServers = ZooKeeperWrapper.getQuorumServers(); + ZooKeeperWrapper zkw = new ZooKeeperWrapper(conf, EmptyWatcher.instance); + String quorumServers = zkw.getQuorumServers(); int sessionTimeout = conf.getInt("zookeeper.session.timeout", 2 * 1000); - Watcher watcher = new EmptyWatcher(); HConnection connection = HConnectionManager.getConnection(conf); ZooKeeperWrapper connectionZK = connection.getZooKeeperWrapper(); long sessionID = connectionZK.getSessionID(); byte[] password = connectionZK.getSessionPassword(); - ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, watcher, sessionID, password); + ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance, sessionID, password); zk.close(); Thread.sleep(sessionTimeout * 3); @@ -119,16 +121,16 @@ try { new HTable(conf, HConstants.META_TABLE_NAME); - String quorumServers = ZooKeeperWrapper.getQuorumServers(); + ZooKeeperWrapper zkw = new ZooKeeperWrapper(conf, EmptyWatcher.instance); + String quorumServers = zkw.getQuorumServers(); int sessionTimeout = conf.getInt("zookeeper.session.timeout", 2 * 1000); - - Watcher watcher = new EmptyWatcher(); + HRegionServer rs = cluster.getRegionServer(0); ZooKeeperWrapper rsZK = rs.getZooKeeperWrapper(); long sessionID = rsZK.getSessionID(); byte[] password = rsZK.getSessionPassword(); - ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, watcher, sessionID, password); + ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance, sessionID, password); zk.close(); Thread.sleep(sessionTimeout * 3); Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java?rev=791037&r1=791036&r2=791037&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java Fri Jul 3 23:33:28 2009 @@ -25,7 +25,9 @@ import java.util.Properties; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HConstants; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer; @@ -59,10 +61,34 @@ super.tearDown(); } - /** @throws Exception */ - public void testConfigInjection() throws Exception { + /** */ + public void testMakeZKProps() { + Properties properties = HQuorumPeer.makeZKProps(conf); + assertEquals(2000, Integer.parseInt(properties.getProperty("tickTime"))); + assertEquals(Integer.valueOf(10), Integer.valueOf(properties.getProperty("initLimit"))); + assertEquals(Integer.valueOf(5), Integer.valueOf(properties.getProperty("syncLimit"))); + assertEquals(dataDir.toString(), properties.get("dataDir")); + assertEquals(Integer.valueOf(21810), Integer.valueOf(properties.getProperty("clientPort"))); + assertEquals("localhost:2888:3888", properties.get("server.0")); + assertEquals(null, properties.get("server.1")); + String oldValue = conf.get(HConstants.ZOOKEEPER_QUORUM); + conf.set(HConstants.ZOOKEEPER_QUORUM, "a.foo.bar,b.foo.bar,c.foo.bar"); + properties = HQuorumPeer.makeZKProps(conf); + assertEquals(2000, Integer.parseInt(properties.getProperty("tickTime"))); + assertEquals(Integer.valueOf(10), Integer.valueOf(properties.getProperty("initLimit"))); + assertEquals(Integer.valueOf(5), Integer.valueOf(properties.getProperty("syncLimit"))); + assertEquals(dataDir.toString(), properties.get("dataDir")); + assertEquals(Integer.valueOf(21810), Integer.valueOf(properties.getProperty("clientPort"))); + assertEquals("a.foo.bar:2888:3888", properties.get("server.0")); + assertEquals("b.foo.bar:2888:3888", properties.get("server.1")); + assertEquals("c.foo.bar:2888:3888", properties.get("server.2")); + assertEquals(null, properties.get("server.3")); + conf.set(HConstants.ZOOKEEPER_QUORUM, oldValue); + } + /** @throws Exception */ + public void testConfigInjection() throws Exception { String s = "tickTime=2000\n" + "initLimit=10\n" + @@ -71,8 +97,9 @@ "clientPort=2181\n" + "server.0=${hbase.master.hostname}:2888:3888\n"; + System.setProperty("hbase.master.hostname", "localhost"); InputStream is = new ByteArrayInputStream(s.getBytes()); - Properties properties = HQuorumPeer.parseConfig(is); + Properties properties = HQuorumPeer.parseZooCfg(conf, is); assertEquals(Integer.valueOf(2000), Integer.valueOf(properties.getProperty("tickTime"))); assertEquals(Integer.valueOf(10), Integer.valueOf(properties.getProperty("initLimit"))); @@ -101,7 +128,7 @@ // Override with system property. System.setProperty("hbase.master.hostname", "foo.bar"); is = new ByteArrayInputStream(s.getBytes()); - properties = HQuorumPeer.parseConfig(is); + properties = HQuorumPeer.parseZooCfg(conf, is); assertEquals("foo.bar:2888:3888", properties.get("server.0")); config.parseProperties(properties); @@ -109,17 +136,5 @@ servers = config.getServers(); server = servers.get(Long.valueOf(0)); assertEquals("foo.bar", server.addr.getHostName()); - - // Special case for property 'hbase.master.hostname' being 'local' - System.setProperty("hbase.master.hostname", "local"); - is = new ByteArrayInputStream(s.getBytes()); - properties = HQuorumPeer.parseConfig(is); - assertEquals("localhost:2888:3888", properties.get("server.0")); - - config.parseProperties(properties); - - servers = config.getServers(); - server = servers.get(Long.valueOf(0)); - assertEquals("localhost", server.addr.getHostName()); } }