hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From raw...@apache.org
Subject svn commit: r901030 - in /hadoop/hbase/trunk: ./ bin/ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/master/ src/test/org/apache/hadoop/hbase/
Date Wed, 20 Jan 2010 01:42:12 GMT
Author: rawson
Date: Wed Jan 20 01:42:12 2010
New Revision: 901030

URL: http://svn.apache.org/viewvc?rev=901030&view=rev
Log:
HBASE-2147 - run zookeeper in the same jvm as master during non-distributed mode

Added:
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HBaseConfTool.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java
Removed:
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniZooKeeperCluster.java
Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/bin/start-hbase.sh
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=901030&r1=901029&r2=901030&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Wed Jan 20 01:42:12 2010
@@ -178,6 +178,7 @@
    HBASE-2140  findbugs issues - 2 performance warnings as suggested by findbugs
                (Kay Kay via Stack)
    HBASE-2139  findbugs task in build.xml (Kay Kay via Stack)
+   HBASE-2147  run zookeeper in the same jvm as master during non-distributed mode
 
   IMPROVEMENTS
    HBASE-1760  Cleanup TODOs in HTable

Modified: hadoop/hbase/trunk/bin/start-hbase.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/bin/start-hbase.sh?rev=901030&r1=901029&r2=901030&view=diff
==============================================================================
--- hadoop/hbase/trunk/bin/start-hbase.sh (original)
+++ hadoop/hbase/trunk/bin/start-hbase.sh Wed Jan 20 01:42:12 2010
@@ -38,7 +38,16 @@
   exit $errCode
 fi
 
+distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
+
+
+if [ $distMode == 'false' ] 
+then
+  echo "Non distributed mode startup"
+  "$bin"/hbase-daemon.sh start master
+else
 "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper
 "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master 
 "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
   --hosts "${HBASE_REGIONSERVERS}" start regionserver
+fi

Added: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HBaseConfTool.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HBaseConfTool.java?rev=901030&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HBaseConfTool.java (added)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HBaseConfTool.java Wed Jan 20 01:42:12
2010
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class HBaseConfTool {
+
+  public static void main(String args[]) {
+    if (args.length < 1)
+      return;
+
+    Configuration conf = HBaseConfiguration.create();
+    System.out.println(conf.get(args[0]));
+  }
+}

Added: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java?rev=901030&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java (added)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java Wed Jan
20 01:42:12 2010
@@ -0,0 +1,219 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.net.BindException;
+import java.net.Socket;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.zookeeper.server.NIOServerCnxn;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.persistence.FileTxnLog;
+
+/**
+ * TODO: Most of the code in this class is ripped from ZooKeeper tests. Instead
+ * of redoing it, we should contribute updates to their code which let us more
+ * easily access testing helper objects.
+ */
+public class MiniZooKeeperCluster {
+  private static final Log LOG = LogFactory.getLog(MiniZooKeeperCluster.class);
+
+  private static final int TICK_TIME = 2000;
+  private static final int CONNECTION_TIMEOUT = 30000;
+
+  private boolean started;
+  private int clientPort = 21810; // use non-standard port
+
+  private NIOServerCnxn.Factory standaloneServerFactory;
+  private int tickTime = 0;
+
+  /** Create mini ZooKeeper cluster. */
+  public MiniZooKeeperCluster() {
+    this.started = false;
+  }
+
+  public void setClientPort(int clientPort) {
+    this.clientPort = clientPort;
+  }
+
+  public void setTickTime(int tickTime) {
+    this.tickTime = tickTime;
+  }
+
+  // / XXX: From o.a.zk.t.ClientBase
+  private static void setupTestEnv() {
+    // during the tests we run with 100K prealloc in the logs.
+    // on windows systems prealloc of 64M was seen to take ~15seconds
+    // resulting in test failure (client timeout on first session).
+    // set env and directly in order to handle static init/gc issues
+    System.setProperty("zookeeper.preAllocSize", "100");
+    FileTxnLog.setPreallocSize(100);
+  }
+
+  /**
+   * @param baseDir
+   * @return ClientPort server bound to.
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public int startup(File baseDir) throws IOException,
+      InterruptedException {
+
+    setupTestEnv();
+
+    shutdown();
+
+    File dir = new File(baseDir, "zookeeper").getAbsoluteFile();
+    recreateDir(dir);
+
+    int tickTimeToUse;
+    if (this.tickTime > 0) {
+      tickTimeToUse = this.tickTime;
+    } else {
+      tickTimeToUse = TICK_TIME;
+    }
+    ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
+    while (true) {
+      try {
+        standaloneServerFactory = new NIOServerCnxn.Factory(clientPort);
+      } catch (BindException e) {
+        LOG.info("Faild binding ZK Server to client port: " + clientPort);
+        //this port is already in use. try to use another
+        clientPort++;
+        continue;
+      }
+      break;
+    }
+    standaloneServerFactory.startup(server);
+
+    if (!waitForServerUp(clientPort, CONNECTION_TIMEOUT)) {
+      throw new IOException("Waiting for startup of standalone server");
+    }
+
+    started = true;
+
+    return clientPort;
+  }
+
+  private void recreateDir(File dir) throws IOException {
+    if (dir.exists()) {
+      FileUtil.fullyDelete(dir);
+    }
+    try {
+      dir.mkdirs();
+    } catch (SecurityException e) {
+      throw new IOException("creating dir: " + dir, e);
+    }
+  }
+
+  /**
+   * @throws IOException
+   */
+  public void shutdown() throws IOException {
+    if (!started) {
+      return;
+    }
+
+    standaloneServerFactory.shutdown();
+    if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
+      throw new IOException("Waiting for shutdown of standalone server");
+    }
+
+    started = false;
+  }
+
+  // XXX: From o.a.zk.t.ClientBase
+  private static boolean waitForServerDown(int port, long timeout) {
+    long start = System.currentTimeMillis();
+    while (true) {
+      try {
+        Socket sock = new Socket("localhost", port);
+        try {
+          OutputStream outstream = sock.getOutputStream();
+          outstream.write("stat".getBytes());
+          outstream.flush();
+        } finally {
+          sock.close();
+        }
+      } catch (IOException e) {
+        return true;
+      }
+
+      if (System.currentTimeMillis() > start + timeout) {
+        break;
+      }
+      try {
+        Thread.sleep(250);
+      } catch (InterruptedException e) {
+        // ignore
+      }
+    }
+    return false;
+  }
+
+  // XXX: From o.a.zk.t.ClientBase
+  private static boolean waitForServerUp(int port, long timeout) {
+    long start = System.currentTimeMillis();
+    while (true) {
+      try {
+        Socket sock = new Socket("localhost", port);
+        BufferedReader reader = null;
+        try {
+          OutputStream outstream = sock.getOutputStream();
+          outstream.write("stat".getBytes());
+          outstream.flush();
+
+          Reader isr = new InputStreamReader(sock.getInputStream());
+          reader = new BufferedReader(isr);
+          String line = reader.readLine();
+          if (line != null && line.startsWith("Zookeeper version:")) {
+            return true;
+          }
+        } finally {
+          sock.close();
+          if (reader != null) {
+            reader.close();
+          }
+        }
+      } catch (IOException e) {
+        // ignore as this is expected
+        LOG.info("server localhost:" + port + " not up " + e);
+      }
+
+      if (System.currentTimeMillis() > start + timeout) {
+        break;
+      }
+      try {
+        Thread.sleep(250);
+      } catch (InterruptedException e) {
+        // ignore
+      }
+    }
+    return false;
+  }
+}

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=901030&r1=901029&r2=901030&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java Wed Jan 20 01:42:12
2010
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.master;
 
 import java.io.IOException;
+import java.io.File;
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
 import java.lang.reflect.Constructor;
@@ -59,6 +60,7 @@
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.MiniZooKeeperCluster;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.Result;
@@ -1196,6 +1198,25 @@
           // If 'local', defer to LocalHBaseCluster instance.  Starts master
           // and regionserver both in the one JVM.
           if (LocalHBaseCluster.isLocal(conf)) {
+            // TODO make zookeepercluster a field and do an orderly shutdown
+            MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster();
+            File zkDataPath = new File(conf.get("hbase.zookeeper.property.dataDir"));
+            int zkClientPort = conf.getInt("hbase.zookeeper.property.clientPort", 0);
+            if (zkClientPort == 0) {
+              throw new IOException("No config value for hbase.zookeeper.property.clientPort");
+            }
+
+            zooKeeperCluster.setTickTime(conf.getInt("hbase.zookeeper.property.tickTime",
3000));
+            zooKeeperCluster.setClientPort(zkClientPort);
+            int clientPort = zooKeeperCluster.startup(zkDataPath);
+            if (clientPort != zkClientPort) {
+              String errorMsg = "Couldnt start ZK at requested address of " +
+                  zkClientPort + ", instead got: " + clientPort + ". Aborting. Why? " +
+                  "Because clients (eg shell) wont be able to find this ZK quorum";
+              System.err.println(errorMsg);
+              throw new IOException(errorMsg);
+            }
+            conf.set("hbase.zookeeper.property.clientPort", Integer.toString(clientPort));
             (new LocalHBaseCluster(conf)).startup();
           } else {
             Constructor<? extends HMaster> c =



Mime
View raw message