incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r958386 - in /incubator/hama/trunk: ./ bin/ conf/ src/java/org/apache/hama/ src/java/org/apache/hama/zookeeper/ src/test/org/apache/hama/
Date Sun, 27 Jun 2010 13:29:59 GMT
Author: edwardyoon
Date: Sun Jun 27 13:29:59 2010
New Revision: 958386

URL: http://svn.apache.org/viewvc?rev=958386&view=rev
Log:
Hama/Zookeeper Integration

Added:
    incubator/hama/trunk/bin/zookeepers.sh   (with props)
    incubator/hama/trunk/src/java/org/apache/hama/zookeeper/
    incubator/hama/trunk/src/java/org/apache/hama/zookeeper/QuorumPeer.java
    incubator/hama/trunk/src/java/org/apache/hama/zookeeper/ZKServerTool.java
Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/bin/hama
    incubator/hama/trunk/bin/hama-daemons.sh
    incubator/hama/trunk/bin/start-bspd.sh
    incubator/hama/trunk/bin/stop-bspd.sh
    incubator/hama/trunk/conf/hama-default.xml
    incubator/hama/trunk/conf/hama-env.sh
    incubator/hama/trunk/src/java/org/apache/hama/Constants.java
    incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java
    incubator/hama/trunk/src/test/org/apache/hama/HamaCluster.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Sun Jun 27 13:29:59 2010
@@ -4,6 +4,7 @@ Trunk (unreleased changes)
 
   NEW FEATURES
 
+    HAMA-272: Hama/Zookeeper Integration (edwardyoon)
     HAMA-265: Add example Pi estimatior based on BSP (edwardyoon)
     HAMA-266: Add Ant target to generate the UML class diagrams (edwardyoon)
     HAMA-209: Add example for Cosine Similarity Matrix (edwardyoon)

Modified: incubator/hama/trunk/bin/hama
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/bin/hama?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/bin/hama (original)
+++ incubator/hama/trunk/bin/hama Sun Jun 27 13:29:59 2010
@@ -57,6 +57,7 @@ if [ $# = 0 ]; then
   echo "where COMMAND is one of:"
   echo "  bspmaster            run the BSP Master node"
   echo "  groom                run the Groom node"
+  echo "  zookeeper            run a Zookeeper server"
   echo "  jar <jar>            run a jar file"
   echo " or"
   echo "  CLASSNAME            run the class named CLASSNAME"
@@ -159,6 +160,8 @@ if [ "$COMMAND" = "bspmaster" ] ; then
 elif [ "$COMMAND" = "groom" ] ; then
   CLASS='org.apache.hama.bsp.GroomServer'
   BSP_OPTS="$BSP_OPTS $BSP_GROOMSERVER_OPTS"
+elif [ "$COMMAND" = "zookeeper" ] ; then
+  CLASS='org.apache.hama.zookeeper.QuorumPeer'
 elif [ "$COMMAND" = "test" ] ; then
   CLASS=org.apache.hama.bsp.BSPTestDriver
   BSP_OPTS="$BSP_OPTS"

Modified: incubator/hama/trunk/bin/hama-daemons.sh
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/bin/hama-daemons.sh?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/bin/hama-daemons.sh (original)
+++ incubator/hama/trunk/bin/hama-daemons.sh Sun Jun 27 13:29:59 2010
@@ -31,4 +31,15 @@ bin=`cd "$bin"; pwd`
 
 . $bin/hama-config.sh
 
-exec "$bin/grooms.sh" --config $HAMA_CONF_DIR cd "$HAMA_HOME" \; "$bin/hama-daemon.sh" --config
$HAMA_CONF_DIR "$@"
+remote_cmd="cd ${HAMA_HOME}; $bin/hama-daemon.sh --config ${HAMA_CONF_DIR} $@"
+args="--config ${HAMA_CONF_DIR} $remote_cmd"
+command=$2
+
+case $command in
+  (zookeeper)
+    exec "$bin/zookeepers.sh" $args
+    ;;
+  (*)
+    exec "$bin/grooms.sh" $args
+    ;;
+esac
\ No newline at end of file

Modified: incubator/hama/trunk/bin/start-bspd.sh
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/bin/start-bspd.sh?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/bin/start-bspd.sh (original)
+++ incubator/hama/trunk/bin/start-bspd.sh Sun Jun 27 13:29:59 2010
@@ -25,5 +25,6 @@ bin=`cd "$bin"; pwd`
 
 # start mapred daemons
 # start jobtracker first to minimize connection errors at startup
+"$bin"/hama-daemons.sh --config "${HAMA_CONF_DIR}" start zookeeper
 "$bin"/hama-daemon.sh --config $HAMA_CONF_DIR start bspmaster
 "$bin"/hama-daemons.sh --config $HAMA_CONF_DIR start groom

Modified: incubator/hama/trunk/bin/stop-bspd.sh
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/bin/stop-bspd.sh?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/bin/stop-bspd.sh (original)
+++ incubator/hama/trunk/bin/stop-bspd.sh Sun Jun 27 13:29:59 2010
@@ -25,4 +25,4 @@ bin=`cd "$bin"; pwd`
 
 "$bin"/hama-daemon.sh --config $HAMA_CONF_DIR stop bspmaster
 "$bin"/hama-daemons.sh --config $HAMA_CONF_DIR stop groom
-
+"$bin"/hama-daemons.sh --config "${HAMA_CONF_DIR}" stop zookeeper

Added: incubator/hama/trunk/bin/zookeepers.sh
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/bin/zookeepers.sh?rev=958386&view=auto
==============================================================================
--- incubator/hama/trunk/bin/zookeepers.sh (added)
+++ incubator/hama/trunk/bin/zookeepers.sh Sun Jun 27 13:29:59 2010
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Copyright 2009 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+# 
+# Run a shell command on all zookeeper hosts.
+#
+# Environment Variables
+#
+#   HAMA_CONF_DIR  Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
+#   HAMA_SSH_OPTS Options passed to ssh when running remote commands.
+#
+# Modelled after $HADOOP_HOME/bin/slaves.sh.
+
+usage="Usage: zookeepers [--config <hbase-confdir>] command..."
+
+# if no args specified, show usage
+if [ $# -le 0 ]; then
+  echo $usage
+  exit 1
+fi
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/hama-config.sh
+
+if [ "$HAMA_MANAGES_ZK" = "" ]; then
+  HAMA_MANAGES_ZK=true
+fi
+
+if [ "$HAMA_MANAGES_ZK" = "true" ]; then
+  hosts=`"$bin"/hama org.apache.hama.zookeeper.ZKServerTool`
+  cmd=$"${@// /\\ }"
+  for zookeeper in $hosts; do
+   ssh $HAMA_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" &
+  done
+fi
+
+wait

Propchange: incubator/hama/trunk/bin/zookeepers.sh
------------------------------------------------------------------------------
    svn:executable = *

Modified: incubator/hama/trunk/conf/hama-default.xml
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/conf/hama-default.xml?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/conf/hama-default.xml (original)
+++ incubator/hama/trunk/conf/hama-default.xml Sun Jun 27 13:29:59 2010
@@ -23,27 +23,90 @@
 -->
 <configuration>
   <property>
+    <name>bsp.master.address</name>
+    <value>local</value>
+    <description></description>
+  </property>
+  <property>
     <name>bsp.master.port</name>
     <value>40000</value>
     <description>The port master should bind to.</description>
   </property>
-
   <property>
     <name>bsp.groom.port</name>
     <value>40020</value>
     <description>The port an groom server binds to.</description>
   </property>
-  
   <property>
     <name>bsp.local.dir</name>
     <value>${hadoop.tmp.dir}/bsp/local</value>
     <description>local directory for temporal store</description> 
   </property>
-  
   <property>
     <name>bsp.system.dir</name>
     <value>${hadoop.tmp.dir}/bsp/system</value>
     <description>The shared directory where BSP stores control files.
     </description>
   </property>
+  <property>
+    <name>hama.tmp.dir</name>
+    <value>/tmp/hama-${user.name}</value>
+    <description>Temporary directory on the local filesystem.</description>
+  </property>
+  
+  <!--
+  Beginning of properties that are directly mapped from ZooKeeper's zoo.cfg.
+  All properties with an "hama.zookeeper.property." prefix are converted for
+  ZooKeeper's configuration. Hence, if you want to add an option from zoo.cfg,
+  e.g.  "initLimit=10" you would append the following to your configuration:
+    <property>
+      <name>hama.zookeeper.property.initLimit</name>
+      <value>10</value>
+    </property>
+  -->
+  <property>
+    <name>hama.zookeeper.property.initLimit</name>
+    <value>10</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The number of ticks that the initial synchronization phase can take.
+    </description>
+  </property>
+  <property>
+    <name>hama.zookeeper.property.syncLimit</name>
+    <value>5</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The number of ticks that can pass between sending a request and getting an
+    acknowledgment.
+    </description>
+  </property>
+  <property>
+    <name>hama.zookeeper.property.dataDir</name>
+    <value>${hama.tmp.dir}/zookeeper</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The directory where the snapshot is stored.
+    </description>
+  </property>
+  <property>
+    <name>hama.zookeeper.property.clientPort</name>
+    <value>21810</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The port at which the clients will connect.
+    </description>
+  </property>
+  <property>
+    <name>hama.zookeeper.property.tickTime</name>
+    <value>3000</value>
+    <description></description>
+  </property>
+  <property>
+    <name>hama.zookeeper.property.maxClientCnxns</name>
+    <value>30</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    Limit on number of concurrent connections (at the socket level) that a
+    single client, identified by IP address, may make to a single member of
+    the ZooKeeper ensemble. Set high to avoid zk connection issues running
+    standalone and pseudo-distributed.
+    </description>
+  </property>
+  <!-- End of properties that are directly mapped from ZooKeeper's zoo.cfg -->
 </configuration>
\ No newline at end of file

Modified: incubator/hama/trunk/conf/hama-env.sh
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/conf/hama-env.sh?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/conf/hama-env.sh (original)
+++ incubator/hama/trunk/conf/hama-env.sh Sun Jun 27 13:29:59 2010
@@ -22,7 +22,7 @@
 # Set environment variables here.
 
 # The java implementation to use.  Required.
-# export JAVA_HOME=/usr/java/jdk1.6.0_18
+export JAVA_HOME=/usr/lib/jvm/java-6-sun
 
 # hadoop conf dir. to find the mapreduce cluster.
 # export HADOOP_CONF_DIR=/usr/local/src/hadoop-0.20.1/conf
@@ -38,3 +38,9 @@
 
 # Where log files are stored.  $HAMA_HOME/logs by default.
 # export HAMA_LOG_DIR=${HAMA_HOME}/logs
+
+# Extra ssh options.  Empty by default.
+# export HAMA_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HAMA_CONF_DIR"
+
+# Tell Hama whether it should manage it's own instance of Zookeeper or not.
+# export HAMA_MANAGES_ZK=true
\ No newline at end of file

Modified: incubator/hama/trunk/src/java/org/apache/hama/Constants.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Constants.java?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Constants.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Constants.java Sun Jun 27 13:29:59 2010
@@ -60,6 +60,16 @@ public interface Constants {
   public static final int DEFAULT_ZOOKEEPER_PAUSE = 2 * 1000;
 
   ///////////////////////////////////////
+  // Constants for ZooKeeper
+  ///////////////////////////////////////  
+  static final String ZOOKEEPER_CONFIG_NAME = "zoo.cfg";
+  static final int DEFAULT_ZOOKEPER_CLIENT_PORT = 21810;
+  static final String ZOOKEEPER_QUORUM = "hama.zookeeper.quorum";
+  /** Cluster is in distributed mode or not */
+  static final String CLUSTER_DISTRIBUTED = "hama.cluster.distributed";
+  /** Cluster is fully-distributed */
+  static final String CLUSTER_IS_DISTRIBUTED = "true";
+  ///////////////////////////////////////
   // Constants for Matrix Package
   ///////////////////////////////////////
   /**

Modified: incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java Sun Jun 27 13:29:59
2010
@@ -82,7 +82,7 @@ public class HamaConfiguration extends H
    * Adds Hama configuration files to a Configuration
    */
   private void addHamaResources() {
-    addResource("hama-default.xml");
-    addResource("hama-site.xml");
+    Configuration.addDefaultResource("hama-default.xml");
+    Configuration.addDefaultResource("hama-site.xml");
   }
 }

Added: incubator/hama/trunk/src/java/org/apache/hama/zookeeper/QuorumPeer.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/zookeeper/QuorumPeer.java?rev=958386&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/zookeeper/QuorumPeer.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/zookeeper/QuorumPeer.java Sun Jun 27 13:29:59
2010
@@ -0,0 +1,260 @@
+package org.apache.hama.zookeeper;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Properties;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.DNS;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hama.Constants;
+import org.apache.hama.HamaConfiguration;
+import org.apache.zookeeper.server.ServerConfig;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+
+public class QuorumPeer implements Constants {
+  private static final Log LOG = LogFactory.getLog(QuorumPeer.class);
+
+  private static final String VARIABLE_START = "${";
+  private static final int VARIABLE_START_LENGTH = VARIABLE_START.length();
+  private static final String VARIABLE_END = "}";
+  private static final int VARIABLE_END_LENGTH = VARIABLE_END.length();
+
+  private static final String ZK_CFG_PROPERTY = "hama.zookeeper.property.";
+  private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
+  private static final String ZK_CLIENT_PORT_KEY = ZK_CFG_PROPERTY
+      + "clientPort";
+
+  /**
+   * Parse ZooKeeper configuration from Hama XML config and run a QuorumPeer.
+   * @param args String[] of command line arguments. Not used.
+   */
+  public static void main(String[] args) {
+    Configuration conf = new HamaConfiguration();
+    try {
+      Properties zkProperties = makeZKProps(conf);
+      writeMyID(zkProperties);
+      QuorumPeerConfig zkConfig = new QuorumPeerConfig();
+      zkConfig.parseProperties(zkProperties);
+      runZKServer(zkConfig);
+    } catch (Exception e) {
+      e.printStackTrace();
+      System.exit(-1);
+    }
+  }
+
+  private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException,
IOException {
+    if (zkConfig.isDistributed()) {
+      QuorumPeerMain qp = new QuorumPeerMain();
+      qp.runFromConfig(zkConfig);
+    } else {
+      ZooKeeperServerMain zk = new ZooKeeperServerMain();
+      ServerConfig serverConfig = new ServerConfig();
+      serverConfig.readFrom(zkConfig);
+      zk.runFromConfig(serverConfig);
+    }
+  }
+
+  private static boolean addressIsLocalHost(String address) {
+    return address.equals("localhost") || address.equals("127.0.0.1");
+  }
+
+  private static void writeMyID(Properties properties) throws IOException {
+    long myId = -1;
+
+    Configuration conf = new HamaConfiguration();
+    String myAddress = DNS.getDefaultHost(
+        conf.get("hama.zookeeper.dns.interface","default"),
+        conf.get("hama.zookeeper.dns.nameserver","default"));
+
+    List<String> ips = new ArrayList<String>();
+
+    // Add what could be the best (configured) match
+    ips.add(myAddress.contains(".") ?
+        myAddress :
+        StringUtils.simpleHostname(myAddress));
+
+    // For all nics get all hostnames and IPs
+    Enumeration<?> nics = NetworkInterface.getNetworkInterfaces();
+    while(nics.hasMoreElements()) {
+      Enumeration<?> rawAdrs =
+          ((NetworkInterface)nics.nextElement()).getInetAddresses();
+      while(rawAdrs.hasMoreElements()) {
+        InetAddress inet = (InetAddress) rawAdrs.nextElement();
+        ips.add(StringUtils.simpleHostname(inet.getHostName()));
+        ips.add(inet.getHostAddress());
+      }
+    }
+
+    for (Entry<Object, Object> entry : properties.entrySet()) {
+      String key = entry.getKey().toString().trim();
+      String value = entry.getValue().toString().trim();
+      if (key.startsWith("server.")) {
+        int dot = key.indexOf('.');
+        long id = Long.parseLong(key.substring(dot + 1));
+        String[] parts = value.split(":");
+        String address = parts[0];
+        if (addressIsLocalHost(address) || ips.contains(address)) {
+          myId = id;
+          break;
+        }
+      }
+    }
+
+    if (myId == -1) {
+      throw new IOException("Could not find my address: " + myAddress +
+                            " in list of ZooKeeper quorum servers");
+    }
+
+    String dataDirStr = properties.get("dataDir").toString().trim();
+    File dataDir = new File(dataDirStr);
+    if (!dataDir.isDirectory()) {
+      if (!dataDir.mkdirs()) {
+        throw new IOException("Unable to create data dir " + dataDir);
+      }
+    }
+
+    File myIdFile = new File(dataDir, "myid");
+    PrintWriter w = new PrintWriter(myIdFile);
+    w.println(myId);
+    w.close();
+  }
+
+  /**
+   * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
+   * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
+   * the corresponding config options from the HBase XML configs and generate
+   * the appropriate ZooKeeper properties.
+   * @param conf Configuration to read from.
+   * @return Properties holding mappings representing ZooKeeper zoo.cfg file.
+   */
+  public static Properties makeZKProps(Configuration conf) {
+    // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
+    // it and grab its configuration properties.
+    ClassLoader cl = QuorumPeer.class.getClassLoader();
+    InputStream inputStream = cl.getResourceAsStream(ZOOKEEPER_CONFIG_NAME);
+    if (inputStream != null) {
+      try {
+        return parseZooCfg(conf, inputStream);
+      } catch (IOException e) {
+        LOG.warn("Cannot read " + ZOOKEEPER_CONFIG_NAME +
+                 ", loading from XML files", e);
+      }
+    }
+
+    // Otherwise, use the configuration options from HBase's XML files.
+    Properties zkProperties = new Properties();
+
+    // Directly map all of the hbase.zookeeper.property.KEY properties.
+    for (Entry<String, String> entry : conf) {
+      String key = entry.getKey();
+      if (key.startsWith(ZK_CFG_PROPERTY)) {
+        String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE);
+        String value = entry.getValue();
+        // If the value has variables substitutions, need to do a get.
+        if (value.contains(VARIABLE_START)) {
+          value = conf.get(key);
+        }
+        zkProperties.put(zkKey, value);
+      }
+    }
+
+    // If clientPort is not set, assign the default
+    if (zkProperties.getProperty(ZK_CLIENT_PORT_KEY) == null) {
+      zkProperties.put(ZK_CLIENT_PORT_KEY, DEFAULT_ZOOKEPER_CLIENT_PORT);
+    }
+
+    // Create the server.X properties.
+    int peerPort = conf.getInt("hama.zookeeper.peerport", 2888);
+    int leaderPort = conf.getInt("hama.zookeeper.leaderport", 3888);
+
+    String[] serverHosts = conf.getStrings(ZOOKEEPER_QUORUM, "localhost");
+    for (int i = 0; i < serverHosts.length; ++i) {
+      String serverHost = serverHosts[i];
+      String address = serverHost + ":" + peerPort + ":" + leaderPort;
+      String key = "server." + i;
+      zkProperties.put(key, address);
+    }
+
+    return zkProperties;
+  }
+
+  /**
+   * Parse ZooKeeper's zoo.cfg, injecting HBase Configuration variables in.
+   * This method is used for testing so we can pass our own InputStream.
+   * @param conf HBaseConfiguration to use for injecting variables.
+   * @param inputStream InputStream to read from.
+   * @return Properties parsed from config stream with variables substituted.
+   * @throws IOException if anything goes wrong parsing config
+   */
+  public static Properties parseZooCfg(Configuration conf,
+      InputStream inputStream) throws IOException {
+    Properties properties = new Properties();
+    try {
+      properties.load(inputStream);
+    } catch (IOException e) {
+      String msg = "fail to read properties from " + ZOOKEEPER_CONFIG_NAME;
+      LOG.fatal(msg);
+      throw new IOException(msg, e);
+    }
+    for (Entry<Object, Object> entry : properties.entrySet()) {
+      String value = entry.getValue().toString().trim();
+      String key = entry.getKey().toString().trim();
+      StringBuilder newValue = new StringBuilder();
+      int varStart = value.indexOf(VARIABLE_START);
+      int varEnd = 0;
+      while (varStart != -1) {
+        varEnd = value.indexOf(VARIABLE_END, varStart);
+        if (varEnd == -1) {
+          String msg = "variable at " + varStart + " has no end marker";
+          LOG.fatal(msg);
+          throw new IOException(msg);
+        }
+        String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd);
+
+        String substituteValue = System.getProperty(variable);
+        if (substituteValue == null) {
+          substituteValue = conf.get(variable);
+        }
+        if (substituteValue == null) {
+          String msg = "variable " + variable + " not set in system property "
+                     + "or hbase configs";
+          LOG.fatal(msg);
+          throw new IOException(msg);
+        }
+
+        newValue.append(substituteValue);
+
+        varEnd += VARIABLE_END_LENGTH;
+        varStart = value.indexOf(VARIABLE_START, varEnd);
+      }
+      // Special case for 'hbase.cluster.distributed' property being 'true'
+      if (key.startsWith("server.")) {
+        if (conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) &&
+            value.startsWith("localhost")) {
+          String msg = "The server in zoo.cfg cannot be set to localhost " +
+              "in a fully-distributed setup because it won't be reachable. " +
+              "See \"Getting Started\" for more information.";
+          LOG.fatal(msg);
+          throw new IOException(msg);
+        }
+      }
+      newValue.append(value.substring(varEnd));
+      properties.setProperty(key, newValue.toString());
+    }
+    return properties;
+  }
+}

Added: incubator/hama/trunk/src/java/org/apache/hama/zookeeper/ZKServerTool.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/zookeeper/ZKServerTool.java?rev=958386&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/zookeeper/ZKServerTool.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/zookeeper/ZKServerTool.java Sun Jun 27 13:29:59
2010
@@ -0,0 +1,30 @@
+package org.apache.hama.zookeeper;
+
+import java.util.Properties;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hama.HamaConfiguration;
+
+public class ZKServerTool {
+  /**
+   * Run the tool.
+   * 
+   * @param args Command line arguments. First arg is path to zookeepers file.
+   */
+  public static void main(String args[]) {
+    Configuration conf = new HamaConfiguration();
+    // Note that we do not simply grab the property ZOOKEEPER_QUORUM from
+    // the HamaConfiguration because the user may be using a zoo.cfg file.
+    Properties zkProps = QuorumPeer.makeZKProps(conf);
+    for (Entry<Object, Object> entry : zkProps.entrySet()) {
+      String key = entry.getKey().toString().trim();
+      String value = entry.getValue().toString().trim();
+      if (key.startsWith("server.")) {
+        String[] parts = value.split(":");
+        String host = parts[0];
+        System.out.println(host);
+      }
+    }
+  }
+}

Modified: incubator/hama/trunk/src/test/org/apache/hama/HamaCluster.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/HamaCluster.java?rev=958386&r1=958385&r2=958386&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/HamaCluster.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/HamaCluster.java Sun Jun 27 13:29:59 2010
@@ -38,8 +38,8 @@ public abstract class HamaCluster extend
     String[] args = new String[0];
     StringUtils.startupShutdownMessage(LocalBSPCluster.class, args, LOG);
     HamaConfiguration conf = new HamaConfiguration();
-    LocalBSPCluster cluster = new LocalBSPCluster(conf);
-    cluster.startup();
+    //LocalBSPCluster cluster = new LocalBSPCluster(conf);
+    //cluster.startup();
   }
 
   protected static HamaConfiguration getConf() {



Mime
View raw message