hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1326045 - in /hbase/branches/0.89-fb: bin/ bin/test/ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/master/ src/main/java/org/apache/hadoop/hbase/regionserver/ src/test/java/org/apache/hadoop/hbase/ src/test/j...
Date Sat, 14 Apr 2012 02:46:27 GMT
Author: mbautin
Date: Sat Apr 14 02:46:26 2012
New Revision: 1326045

URL: http://svn.apache.org/viewvc?rev=1326045&view=rev
Log:
[jira] [HBASE-5684] [89-fb] Make ProcessBasedLocalHBaseCluster run HDFS and make it more robust

Summary:
Currently ProcessBasedLocalHBaseCluster runs on top of raw local filesystem. We
need it to start a process-based HDFS cluster as well. We also need to make the
whole thing more stable so we can use it in unit tests.

Also all logs of local HBase cluster daemons are now tailed to the primary log
of the unit test with the appropriate prefixes to make debugging easier.

Test Plan:
Run the new unit test multiple times (10x or 50x).
Run all unit tests.

Reviewers: kannan, liyintang, pkhemani, kranganathan, madhuvaidya, amirshim, tedyu

Reviewed By: liyintang

CC: tedyu

Differential Revision: https://reviews.facebook.net/D2709

Added:
    hbase/branches/0.89-fb/bin/test/
    hbase/branches/0.89-fb/bin/test/process_based_cluster.sh   (with props)
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HTestConst.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/HBaseHomePath.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestProcessBasedCluster.java
Modified:
    hbase/branches/0.89-fb/bin/hbase-config.sh
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedCloseRegion.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedReopenRegion.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/ProcessBasedLocalHBaseCluster.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java

Modified: hbase/branches/0.89-fb/bin/hbase-config.sh
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/bin/hbase-config.sh?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/bin/hbase-config.sh (original)
+++ hbase/branches/0.89-fb/bin/hbase-config.sh Sat Apr 14 02:46:26 2012
@@ -115,9 +115,19 @@ HBASE_JMX_OPTS="-Dcom.sun.management.jmx
 HBASE_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.authenticate=false"
 # HBASE_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.password.file=$HBASE_HOME/conf/jmxremote.passwd"
 # HBASE_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.access.file=$HBASE_HOME/conf/jmxremote.access"
-HBASE_MASTER_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=8090"
-HBASE_REGIONSERVER_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=8091"
-HBASE_ZOOKEEPER_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=8092"
+
+# JMX options. Set default values if not overridden in hbase-env.sh.
+if [ -z "$HBASE_MASTER_JMX_OPTS" ]; then
+  HBASE_MASTER_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=8090"
+fi
+
+if [ -z "$HBASE_REGIONSERVER_JMX_OPTS" ]; then
+  HBASE_REGIONSERVER_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=8091"
+fi
+
+if [ -z "$HBASE_ZOOKEEPER_JMX_OPTS" ]; then
+  HBASE_ZOOKEEPER_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=8092"
+fi
 
 # YourKit Java Profiling
 # Note that you need to have yjpagent.so & yjp.jar on your computer and have
@@ -131,20 +141,27 @@ HBASE_GC_OPTIONS="-verbose:gc -XX:+Print
 HBASE_MASTER_JDWP_OPTIONS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070"
 HBASE_REGIONSERVER_JDWP_OPTIONS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071"
 
-# Java debugging options.
-# By default, local hbase instances are setup with the remote debugging
-# enabled for the HMaster (which also functions as regionserver) on port 1044
-# Zookeeper debug is configured on port 1045
-HBASE_MASTER_DBG_OPTS="$HBASE_MASTER_JDWP_OPTIONS $HBASE_GC_OPTIONS"
-HBASE_REGIONSERVER_DBG_OPTS="$HBASE_REGIONSERVER_JDWP_OPTIONS $HBASE_GC_OPTIONS"
-HBASE_ZOOKEEPER_DBG_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072"
+# Java debugging options. Set default values if not overridden in hbase-env.sh.
+if [ -z "$HBASE_MASTER_DBG_OPTS" ]; then
+  HBASE_MASTER_DBG_OPTS="$HBASE_MASTER_JDWP_OPTIONS $HBASE_GC_OPTIONS"
+fi
+
+if [ -z "$HBASE_REGIONSERVER_DBG_OPTS" ]; then
+  HBASE_REGIONSERVER_DBG_OPTS="$HBASE_REGIONSERVER_JDWP_OPTIONS $HBASE_GC_OPTIONS"
+fi
+
+if [ -z "$HBASE_ZOOKEEPER_DBG_OPTS" ]; then
+  HBASE_ZOOKEEPER_DBG_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072"
+fi
 
 if [ -z "$HBASE_MASTER_OPTS" ]; then
   export HBASE_MASTER_OPTS="$HBASE_MASTER_DBG_OPTS $HBASE_MASTER_JMX_OPTS"
 fi
+
 if [ -z "$HBASE_REGIONSERVER_OPTS" ]; then
   export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_DBG_OPTS $HBASE_REGIONSERVER_JMX_OPTS"
 fi
+
 if [ -z "$HBASE_ZOOKEEPER_OPTS" ]; then
   export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_DBG_OPTS $HBASE_ZOOKEEPER_JMX_OPTS"
 fi

Added: hbase/branches/0.89-fb/bin/test/process_based_cluster.sh
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/bin/test/process_based_cluster.sh?rev=1326045&view=auto
==============================================================================
--- hbase/branches/0.89-fb/bin/test/process_based_cluster.sh (added)
+++ hbase/branches/0.89-fb/bin/test/process_based_cluster.sh Sat Apr 14 02:46:26 2012
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+set -e -u -o pipefail
+
+SCRIPT_NAME=${0##*/}
+SCRIPT_DIR=$(cd `dirname $0` && pwd )
+
+print_usage() {
+  cat >&2 <<EOT
+Usage: $SCRIPT_NAME <options>
+Options:
+  --kill
+    Kill local process-based HBase cluster using pid files.
+  --show
+    Show HBase processes running on this machine
+EOT
+  exit 1
+}
+
+show_processes() {
+  ps -ef | grep -P "(HRegionServer|HMaster|HQuorumPeer) start" | grep -v grep
+}
+
+cmd_specified() {
+  if [ "$CMD_SPECIFIED" ]; then
+    echo "Only one command can be specified" >&2
+    exit 1
+  fi
+  CMD_SPECIFIED=1
+}
+
+list_pid_files() {
+  LOCAL_CLUSTER_DIR=$SCRIPT_DIR/../../target/local_cluster
+  LOCAL_CLUSTER_DIR=$( cd $LOCAL_CLUSTER_DIR && pwd )
+  find $LOCAL_CLUSTER_DIR -name "*.pid"
+}
+
+if [ $# -eq 0 ]; then
+  print_usage
+fi
+
+IS_KILL=""
+IS_SHOW=""
+CMD_SPECIFIED=""
+
+while [ $# -ne 0 ]; do
+  case "$1" in
+    -h|--help)
+      print_usage ;;
+    --kill)
+      IS_KILL=1
+      cmd_specified ;;
+    --show)
+      IS_SHOW=1
+      cmd_specified ;;
+    *)
+      echo "Invalid option: $1" >&2
+      exit 1
+  esac
+  shift
+done
+
+if [ "$IS_KILL" ]; then
+  list_pid_files | \
+    while read F; do
+      PID=`cat $F`
+      echo "Killing pid $PID from file $F"
+      # Kill may fail but that's OK, so turn off error handling for a moment.
+      set +e
+      kill -9 $PID
+      set -e
+    done
+elif [ "$IS_SHOW" ]; then
+  PIDS=""
+  for F in `list_pid_files`; do
+    PID=`cat $F`
+    if [ -n "$PID" ]; then
+      if [ -n "$PIDS" ]; then
+        PIDS="$PIDS,"
+      fi
+      PIDS="$PIDS$PID"
+    fi
+  done
+  ps -p $PIDS
+else
+  echo "No command specified" >&2
+  exit 1
+fi

Propchange: hbase/branches/0.89-fb/bin/test/process_based_cluster.sh
------------------------------------------------------------------------------
    svn:executable = *

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Sat Apr 14
02:46:26 2012
@@ -82,6 +82,9 @@ public final class HConstants {
   /** default port for master web api */
   public static final int DEFAULT_MASTER_INFOPORT = 60010;
 
+  /** Configuration key for master web API port */
+  public static final String MASTER_INFO_PORT = "hbase.master.info.port";
+
   /** Parameter name for the master type being backup (waits for primary to go inactive).
*/
   public static final String MASTER_TYPE_BACKUP = "hbase.master.backup";
 
@@ -156,9 +159,13 @@ public final class HConstants {
   /** default port for region server web api */
   public static final int DEFAULT_REGIONSERVER_INFOPORT = 60030;
 
+  /** A configuration key for regionserver info port */
+  public static final String REGIONSERVER_INFO_PORT =
+    "hbase.regionserver.info.port";
+
   /** A flag that enables automatic selection of regionserver info port */
   public static final String REGIONSERVER_INFO_PORT_AUTO =
-    "hbase.regionserver.info.port.auto";
+      REGIONSERVER_INFO_PORT + ".auto";
 
   /** Parameter name for what region server interface to use. */
   public static final String REGION_SERVER_CLASS = "hbase.regionserver.class";
@@ -466,6 +473,8 @@ public final class HConstants {
   /** Host name of the local machine */
   public static final String LOCALHOST = "localhost";
 
+  public static final String LOCALHOST_IP = "127.0.0.1";
+
   /** Conf key that enables distributed log splitting */
   public static final String DISTRIBUTED_LOG_SPLITTING_KEY =
       "hbase.master.distributed.log.splitting";

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Sat Apr
14 02:46:26 2012
@@ -1148,7 +1148,7 @@ public class HMaster extends Thread impl
     try {
       this.regionManager.start();
       // Put up info server.
-      int port = this.conf.getInt("hbase.master.info.port", 60010);
+      int port = this.conf.getInt(HConstants.MASTER_INFO_PORT, 60010);
       if (port >= 0) {
         String a = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0");
         this.infoServer = new InfoServer(MASTER, a, port, false, conf);

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Sat Apr 14 02:46:26 2012
@@ -366,7 +366,7 @@ public class HRegionServer implements HR
     this.serverInfo = new HServerInfo(new HServerAddress(
       new InetSocketAddress(address.getBindAddress(),
       this.server.getListenerAddress().getPort())), System.currentTimeMillis(),
-      this.conf.getInt("hbase.regionserver.info.port", 60030), machineName);
+      this.conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 60030), machineName);
     if (this.serverInfo.getServerAddress() == null) {
       throw new NullPointerException("Server address cannot be null; " +
         "hbase-958 debugging");
@@ -1246,7 +1246,7 @@ public class HRegionServer implements HR
     this.leases.setName(n + ".leaseChecker");
     this.leases.start();
     // Put up info server.
-    int port = this.conf.getInt("hbase.regionserver.info.port", 60030);
+    int port = this.conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 60030);
     // -1 is for disabling info server
     if (port >= 0) {
       String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0");

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Sat Apr 14 02:46:26 2012
@@ -37,11 +37,14 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableSet;
 import java.util.Random;
+import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentSkipListSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -138,6 +141,9 @@ public class HBaseTestingUtility {
   /** Filesystem URI used for map-reduce mini-cluster setup */
   private static String fsURI;
 
+  /** A set of ports that have been claimed using {@link #randomFreePort()}. */
+  private static final Set<Integer> takenRandomPorts = new HashSet<Integer>();
+
   /** Compression algorithms to use in parameterized JUnit 4 tests */
   public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
     Arrays.asList(new Object[][] {
@@ -1348,7 +1354,10 @@ REGION_LOOP:
         Bytes.toBytes(String.format(keyFormat, splitStartKey)),
         Bytes.toBytes(String.format(keyFormat, splitEndKey)),
         numRegions);
-    hbaseCluster.flushcache(HConstants.META_TABLE_NAME);
+
+    if (hbaseCluster != null) {
+      hbaseCluster.flushcache(HConstants.META_TABLE_NAME);
+    }
 
     for (int iFlush = 0; iFlush < numFlushes; ++iFlush) {
       for (int iRow = 0; iRow < numRowsPerFlush; ++iRow) {
@@ -1383,7 +1392,9 @@ REGION_LOOP:
       }
       LOG.info("Initiating flush #" + iFlush + " for table " + tableName);
       table.flushCommits();
-      hbaseCluster.flushcache(tableNameBytes);
+      if (hbaseCluster != null) {
+        hbaseCluster.flushcache(tableNameBytes);
+      }
     }
 
     return table;
@@ -1401,10 +1412,19 @@ REGION_LOOP:
         + new Random().nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
   }
 
+  /**
+   * Returns a random free port and marks that port as taken. Not thread-safe. Expected to
be
+   * called from single-threaded test setup code/
+   */
   public static int randomFreePort() {
     int port = 0;
     do {
       port = randomPort();
+      if (takenRandomPorts.contains(port)) {
+        continue;
+      }
+      takenRandomPorts.add(port);
+
       try {
         ServerSocket sock = new ServerSocket(port);
         sock.close();

Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HTestConst.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HTestConst.java?rev=1326045&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HTestConst.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HTestConst.java Sat Apr 14
02:46:26 2012
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.Collections;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/** Similar to {@link HConstants} but for tests. */
+public class HTestConst {
+
+  private HTestConst() {
+  }
+
+  public static final String DEFAULT_TABLE_STR = "MyTestTable";
+  public static final byte[] DEFAULT_TABLE_BYTES = Bytes.toBytes(DEFAULT_TABLE_STR);
+
+  public static final String DEFAULT_CF_STR = "MyDefaultCF";
+  public static final byte[] DEFAULT_CF_BYTES = Bytes.toBytes(DEFAULT_CF_STR);
+
+  public static final Set<String> DEFAULT_CF_STR_SET =
+      Collections.unmodifiableSet(new HashSet<String>(
+          Arrays.asList(new String[] { DEFAULT_CF_STR })));
+
+}

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java Sat
Apr 14 02:46:26 2012
@@ -38,8 +38,8 @@ public class TestInfoServers extends HBa
   protected void preHBaseClusterSetup() {
     // The info servers do not run in tests by default.
     // Set them to ephemeral ports so they will start
-    conf.setInt("hbase.master.info.port", 0);
-    conf.setInt("hbase.regionserver.info.port", 0);
+    conf.setInt(HConstants.MASTER_INFO_PORT, 0);
+    conf.setInt(HConstants.REGIONSERVER_INFO_PORT, 0);
   }
 
   /**

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
Sat Apr 14 02:46:26 2012
@@ -89,7 +89,7 @@ public class TestDistributedLogSplitting
     ZKSplitLog.Counters.resetCounters();
     LOG.info("Starting cluster");
     conf = HBaseConfiguration.create();
-    conf.setInt("hbase.regionserver.info.port", -1);
+    conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
     conf.setFloat("hbase.regions.slop", (float)100.0); // no load balancing
     conf.setBoolean(HConstants.DISTRIBUTED_LOG_SPLITTING_KEY, true);
     TEST_UTIL = new HBaseTestingUtility(conf);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedCloseRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedCloseRegion.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedCloseRegion.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedCloseRegion.java
Sat Apr 14 02:46:26 2012
@@ -63,7 +63,7 @@ public class TestZKBasedCloseRegion {
   @BeforeClass public static void beforeAllTests() throws Exception {
     Configuration c = TEST_UTIL.getConfiguration();
     c.setBoolean("dfs.support.append", true);
-    c.setInt("hbase.regionserver.info.port", 0);
+    c.setInt(HConstants.REGIONSERVER_INFO_PORT, 0);
     c.setInt("hbase.master.meta.thread.rescanfrequency", 5*1000);
     TEST_UTIL.startMiniCluster(2);
     TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedReopenRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedReopenRegion.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedReopenRegion.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedReopenRegion.java
Sat Apr 14 02:46:26 2012
@@ -64,7 +64,7 @@ public class TestZKBasedReopenRegion {
   @BeforeClass public static void beforeAllTests() throws Exception {
     Configuration c = TEST_UTIL.getConfiguration();
     c.setBoolean("dfs.support.append", true);
-    c.setInt("hbase.regionserver.info.port", 0);
+    c.setInt(HConstants.REGIONSERVER_INFO_PORT, 0);
     c.setInt("hbase.master.meta.thread.rescanfrequency", 5*1000);
     TEST_UTIL.startMiniCluster(2);
     TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
Sat Apr 14 02:46:26 2012
@@ -267,7 +267,7 @@ public class TestScanner extends HBaseTe
 
       // Store some new information
 
-      HServerAddress address = new HServerAddress("127.0.0.1:" +
+      HServerAddress address = new HServerAddress(HConstants.LOCALHOST_IP + ":" +
           HBaseTestingUtility.randomFreePort());
 
       put = new Put(ROW_KEY, System.currentTimeMillis(), null);

Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/HBaseHomePath.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/HBaseHomePath.java?rev=1326045&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/HBaseHomePath.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/HBaseHomePath.java Sat
Apr 14 02:46:26 2012
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.net.URL;
+
+import org.apache.hadoop.hbase.HConstants;
+
+/** Determines HBase home path from either class or jar directory */
+public class HBaseHomePath {
+
+  private static final String TARGET_CLASSES = "/target/classes";
+  private static final String JAR_SUFFIX = ".jar!";
+  private static final String FILE_PREFIX = "file:";
+
+  private HBaseHomePath() {
+  }
+
+  public static String getHomePath() {
+    String className = HConstants.class.getName();  // This could have been any HBase class.
+    String relPathForClass = className.replace(".", "/") + ".class";
+    URL url = ClassLoader.getSystemResource(relPathForClass);
+    relPathForClass = "/" + relPathForClass;
+    if (url == null) {
+      throw new RuntimeException("Could not lookup class location for " + className);
+    }
+
+    String path = url.getPath();
+    if (!path.endsWith(relPathForClass)) {
+      throw new RuntimeException("Got invalid path trying to look up class " + className
+
+          ": " + path);
+    }
+    path = path.substring(0, path.length() - relPathForClass.length());
+
+    if (path.startsWith(FILE_PREFIX)) {
+      path = path.substring(FILE_PREFIX.length());
+    }
+
+    if (path.endsWith(TARGET_CLASSES)) {
+      path = path.substring(0, path.length() - TARGET_CLASSES.length());
+    } else if (path.endsWith(JAR_SUFFIX)) {
+      int slashIndex = path.lastIndexOf("/");
+      if (slashIndex != -1) {
+        throw new RuntimeException("Expected to find slash in jar path " + path);
+      }
+      path = path.substring(0, slashIndex);
+    } else {
+      throw new RuntimeException("Cannot identify HBase source directory or installation
path " +
+          "from " + path);
+    }
+    return path;
+  }
+
+}

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/ProcessBasedLocalHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/ProcessBasedLocalHBaseCluster.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/ProcessBasedLocalHBaseCluster.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/ProcessBasedLocalHBaseCluster.java
Sat Apr 14 02:46:26 2012
@@ -19,27 +19,35 @@ package org.apache.hadoop.hbase.util;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileWriter;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
+import java.util.Set;
 import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 
 /**
  * A helper class for process-based mini-cluster tests. Unlike
@@ -48,17 +56,12 @@ import org.apache.hadoop.hbase.zookeeper
  */
 public class ProcessBasedLocalHBaseCluster {
 
-  private static final String DEFAULT_WORKDIR =
-      "/tmp/hbase-" + System.getenv("USER");
-
-  private final String hbaseHome;
-  private final String workDir;
+  private final String hbaseHome, workDir;
+  private final Configuration conf;
+  private final int numMasters, numRegionServers, numDataNodes;
+  private final List<Integer> rsPorts, masterPorts;
 
-  private int numRegionServers;
   private final int zkClientPort;
-  private final int masterPort;
-
-  private final Configuration conf;
 
   private static final int MAX_FILE_SIZE_OVERRIDE = 10 * 1000 * 1000;
 
@@ -72,39 +75,93 @@ public class ProcessBasedLocalHBaseClust
 
   private String hbaseDaemonScript;
 
+  private MiniDFSCluster dfsCluster;
+
+  private HBaseTestingUtility testUtil;
+
+  private Thread logTailerThread;
+
+  private List<String> logTailDirs = Collections.synchronizedList(new ArrayList<String>());
+
+  private static enum ServerType {
+    MASTER("master"),
+    RS("regionserver"),
+    ZK("zookeeper");
+
+    private final String fullName;
+
+    private ServerType(String fullName) {
+      this.fullName = fullName;
+    }
+  }
+
   /**
    * Constructor. Modifies the passed configuration.
    * @param hbaseHome the top directory of the HBase source tree
    */
-  public ProcessBasedLocalHBaseCluster(Configuration conf, String hbaseHome,
-      int numRegionServers) {
+  public ProcessBasedLocalHBaseCluster(Configuration conf,
+      int numDataNodes, int numRegionServers) {
     this.conf = conf;
-    this.hbaseHome = hbaseHome;
+    this.hbaseHome = HBaseHomePath.getHomePath();
+    this.numMasters = 1;
     this.numRegionServers = numRegionServers;
-    this.workDir = DEFAULT_WORKDIR;
+    this.workDir = hbaseHome + "/target/local_cluster";
+    this.numDataNodes = numDataNodes;
 
     hbaseDaemonScript = hbaseHome + "/bin/hbase-daemon.sh";
     zkClientPort = HBaseTestingUtility.randomFreePort();
-    masterPort = HBaseTestingUtility.randomFreePort();
+
+    this.rsPorts = sortedPorts(numRegionServers);
+    this.masterPorts = sortedPorts(numMasters);
 
     conf.set(HConstants.ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
     conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
   }
 
-  public void start() throws IOException {
+  /**
+   * Makes this local HBase cluster use a mini-DFS cluster. Must be called before
+   * {@link #startHBase()}.
+   * @throws IOException
+   */
+  public void startMiniDFS() throws IOException {
+    if (testUtil == null) {
+      testUtil = new HBaseTestingUtility(conf);
+    }
+    dfsCluster = testUtil.startMiniDFSCluster(numDataNodes);
+  }
+
+  /**
+   * Generates a list of random port numbers in the sorted order. A sorted
+   * order makes sense if we ever want to refer to these servers by their index
+   * in the returned array, e.g. server #0, #1, etc.
+   */
+  private static List<Integer> sortedPorts(int n) {
+    List<Integer> ports = new ArrayList<Integer>(n);
+    for (int i = 0; i < n; ++i) {
+      ports.add(HBaseTestingUtility.randomFreePort());
+    }
+    Collections.sort(ports);
+    return ports;
+  }
+
+  public void startHBase() throws IOException {
+    startDaemonLogTailer();
     cleanupOldState();
 
     // start ZK
-    LOG.info("Starting ZooKeeper");
+    LOG.info("Starting ZooKeeper on port " + zkClientPort);
     startZK();
 
     HBaseTestingUtility.waitForHostPort(HConstants.LOCALHOST, zkClientPort);
 
-    startMaster();
+    for (int masterPort : masterPorts) {
+      startMaster(masterPort);
+    }
+
     ZKUtil.waitForBaseZNode(conf);
 
-    for (int idx = 0; idx < numRegionServers; idx++) {
-      startRegionServer(HBaseTestingUtility.randomFreePort());
+    for (int rsPort : rsPorts) {
+      startRegionServer(rsPort);
     }
 
     LOG.info("Waiting for HBase startup by scanning META");
@@ -124,32 +181,32 @@ public class ProcessBasedLocalHBaseClust
   }
 
   public void startRegionServer(int port) {
-    startServer("regionserver", port);
+    startServer(ServerType.RS, port);
   }
 
-  public void startMaster() {
-    startServer("master", 0);
+  public void startMaster(int port) {
+    startServer(ServerType.MASTER, port);
   }
 
   public void killRegionServer(int port) throws IOException {
-    killServer("regionserver", port);
+    killServer(ServerType.RS, port);
   }
 
   public void killMaster() throws IOException {
-    killServer("master", 0);
+    killServer(ServerType.MASTER, 0);
   }
 
   public void startZK() {
-    startServer("zookeeper", 0);
+    startServer(ServerType.ZK, 0);
   }
 
   private void executeCommand(String command) {
-    ensureShutdownHookInstalled();
     executeCommand(command, null);
   }
 
   private void executeCommand(String command, Map<String,
       String> envOverrides) {
+    ensureShutdownHookInstalled();
     LOG.debug("Command : " + command);
 
     try {
@@ -195,7 +252,7 @@ public class ProcessBasedLocalHBaseClust
       try {
         pid = readPidFromFile(pidFile);
       } catch (IOException ex) {
-        LOG.error("Could not kill process with pid from " + pidFile);
+        LOG.error("Could not read pid from file " + pidFile);
       }
 
       if (pid > 0) {
@@ -203,9 +260,6 @@ public class ProcessBasedLocalHBaseClust
         killProcess(pid);
       }
     }
-
-    LOG.info("Waiting a bit to let processes terminate");
-    Threads.sleep(5000);
   }
 
   private void ensureShutdownHookInstalled() {
@@ -237,18 +291,12 @@ public class ProcessBasedLocalHBaseClust
     }
   }
 
-  private String serverWorkingDir(String serverName, int port) {
-    String dir;
-    if (serverName.equals("regionserver")) {
-      dir = workDir + "/" + serverName + "-" + port;
-    } else {
-      dir = workDir + "/" + serverName;
-    }
-    return dir;
+  private String serverWorkingDir(ServerType serverType, int port) {
+    return workDir + "/" + serverType + "-" + port;
   }
 
-  private int getServerPID(String serverName, int port) throws IOException {
-    String pidFile = pidFilePath(serverName, port);
+  private int getServerPID(ServerType serverType, int port) throws IOException {
+    String pidFile = pidFilePath(serverType, port);
     return readPidFromFile(pidFile);
   }
 
@@ -261,18 +309,18 @@ public class ProcessBasedLocalHBaseClust
     }
   }
 
-  private String pidFilePath(String serverName, int port) {
-    String dir = serverWorkingDir(serverName, port);
+  private String pidFilePath(ServerType serverType, int port) {
+    String dir = serverWorkingDir(serverType, port);
     String user = System.getenv("USER");
     String pidFile = String.format("%s/hbase-%s-%s.pid",
-                                   dir, user, serverName);
+                                   dir, user, serverType.fullName);
     return pidFile;
   }
 
-  private void killServer(String serverName, int port) throws IOException {
-    int pid = getServerPID(serverName, port);
+  private void killServer(ServerType serverType, int port) throws IOException {
+    int pid = getServerPID(serverType, port);
     if (pid > 0) {
-      LOG.info("Killing " + serverName + "; pid=" + pid);
+      LOG.info("Killing " + serverType + "; pid=" + pid);
       killProcess(pid);
     }
   }
@@ -282,14 +330,31 @@ public class ProcessBasedLocalHBaseClust
     executeCommand(cmd);
   }
 
-  private void startServer(String serverName, int rsPort) {
-    String conf = generateConfig(rsPort);
+  private void startServer(ServerType serverType, int rsPort) {
+    String conf = generateConfig(serverType, rsPort);
 
     // create working directory for this region server.
-    String dir = serverWorkingDir(serverName, rsPort);
-    executeCommand("mkdir -p " + dir);
-
-    writeStringToFile(conf, dir + "/" + "hbase-site.xml");
+    String dir = serverWorkingDir(serverType, rsPort);
+    LOG.debug("Creating directory " + dir);
+    new File(dir).mkdirs();
+
+    writeStringToFile(conf, dir + "/hbase-site.xml");
+
+    // Set debug options to an empty string so that hbase-config.sh does not configure them
+    // using default ports. If we want to run remote debugging on process-based local cluster's
+    // daemons, we can automatically choose non-conflicting JDWP and JMX ports for each daemon
+    // and specify them here.
+    writeStringToFile(
+        "unset HBASE_MASTER_OPTS\n" +
+        "unset HBASE_REGIONSERVER_OPTS\n" +
+        "unset HBASE_ZOOKEEPER_OPTS\n" +
+        "HBASE_MASTER_DBG_OPTS=' '\n" +
+        "HBASE_REGIONSERVER_DBG_OPTS=' '\n" +
+        "HBASE_ZOOKEEPER_DBG_OPTS=' '\n" +
+        "HBASE_MASTER_JMX_OPTS=' '\n" +
+        "HBASE_REGIONSERVER_JMX_OPTS=' '\n" +
+        "HBASE_ZOOKEEPER_JMX_OPTS=' '\n",
+        dir + "/hbase-env.sh");
 
     Map<String, String> envOverrides = new HashMap<String, String>();
     envOverrides.put("HBASE_LOG_DIR", dir);
@@ -303,23 +368,39 @@ public class ProcessBasedLocalHBaseClust
     }
 
     executeCommand(hbaseDaemonScript + " --config " + dir +
-                   " start " + serverName, envOverrides);
-    daemonPidFiles.add(pidFilePath(serverName, rsPort));
+                   " start " + serverType.fullName, envOverrides);
+    daemonPidFiles.add(pidFilePath(serverType, rsPort));
+    logTailDirs.add(dir);
   }
 
-  private final String generateConfig(int rsPort) {
+  private final String generateConfig(ServerType serverType, int rpcPort) {
     StringBuilder sb = new StringBuilder();
     Map<String, Object> confMap = new TreeMap<String, Object>();
     confMap.put(HConstants.CLUSTER_DISTRIBUTED, true);
-    if (rsPort > 0) {
-      confMap.put(HConstants.REGIONSERVER_PORT, rsPort);
-      confMap.put(HConstants.REGIONSERVER_INFO_PORT_AUTO, true);
+
+    if (serverType == ServerType.MASTER) {
+      confMap.put(HConstants.MASTER_PORT, rpcPort);
+
+      int masterInfoPort = HBaseTestingUtility.randomFreePort();
+      reportWebUIPort("master", masterInfoPort);
+      confMap.put(HConstants.MASTER_INFO_PORT, masterInfoPort);
+    } else if (serverType == ServerType.RS) {
+      confMap.put(HConstants.REGIONSERVER_PORT, rpcPort);
+
+      int rsInfoPort = HBaseTestingUtility.randomFreePort();
+      reportWebUIPort("region server", rsInfoPort);
+      confMap.put(HConstants.REGIONSERVER_INFO_PORT, rsInfoPort);
     }
 
     confMap.put(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
-    confMap.put(HConstants.MASTER_PORT, masterPort);
     confMap.put(HConstants.HREGION_MAX_FILESIZE, MAX_FILE_SIZE_OVERRIDE);
-    confMap.put("fs.file.impl", RawLocalFileSystem.class.getName());
+
+    if (dfsCluster != null) {
+      String fsURL = "hdfs://" + HConstants.LOCALHOST + ":" + dfsCluster.getNameNodePort();
+      confMap.put("fs.default.name", fsURL);
+      confMap.put("fs.defaultFS", fsURL);
+      confMap.put("hbase.rootdir", fsURL + "/hbase_test");
+    }
 
     sb.append("<configuration>\n");
     for (Map.Entry<String, Object> entry : confMap.entrySet()) {
@@ -332,8 +413,134 @@ public class ProcessBasedLocalHBaseClust
     return sb.toString();
   }
 
+  private static void reportWebUIPort(String daemon, int port) {
+    LOG.info("Local " + daemon + " web UI is at " + "http://"
+        + HConstants.LOCALHOST + ":" + port);
+  }
+
   public Configuration getConf() {
     return conf;
   }
 
+  public void shutdown() {
+    if (dfsCluster != null) {
+      dfsCluster.shutdown();
+    }
+    shutdownAllProcesses();
+  }
+
+  private static final Pattern TO_REMOVE_FROM_LOG_LINES_RE =
+      Pattern.compile("org\\.apache\\.hadoop\\.hbase\\.");
+
+  private static final Pattern LOG_PATH_FORMAT_RE =
+      Pattern.compile("^.*/([A-Z]+)-(\\d+)/[^/]+$");
+
+  private static String processLine(String line) {
+    Matcher m = TO_REMOVE_FROM_LOG_LINES_RE.matcher(line);
+    return m.replaceAll("");
+  }
+
+  private final class LocalDaemonLogTailer implements Runnable {
+    private final Set<String> tailedFiles = new HashSet<String>();
+    private final List<String> dirList = new ArrayList<String>();
+    private final Object printLock = new Object();
+
+    private final FilenameFilter LOG_FILES = new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        return name.endsWith(".out") || name.endsWith(".log");
+      }
+    };
+
+    @Override
+    public void run() {
+      try {
+        runInternal();
+      } catch (IOException ex) {
+        LOG.error(ex);
+      }
+    }
+
+    private void runInternal() throws IOException {
+      Thread.currentThread().setName(getClass().getSimpleName());
+      while (true) {
+        scanDirs();
+        try {
+          Thread.sleep(500);
+        } catch (InterruptedException e) {
+          LOG.error("Log tailer thread interrupted", e);
+          break;
+        }
+      }
+    }
+
+    private void scanDirs() throws FileNotFoundException {
+      dirList.clear();
+      dirList.addAll(logTailDirs);
+      for (String d : dirList) {
+        for (File f : new File(d).listFiles(LOG_FILES)) {
+          String filePath = f.getAbsolutePath();
+          if (!tailedFiles.contains(filePath)) {
+            tailedFiles.add(filePath);
+            startTailingFile(filePath);
+          }
+        }
+      }
+    }
+
+    private void startTailingFile(final String filePath) throws FileNotFoundException {
+      final PrintStream dest = filePath.endsWith(".log") ? System.err : System.out;
+      final ServerType serverType;
+      final int serverPort;
+      Matcher m = LOG_PATH_FORMAT_RE.matcher(filePath);
+      if (m.matches()) {
+        serverType = ServerType.valueOf(m.group(1));
+        serverPort = Integer.valueOf(m.group(2));
+      } else {
+        LOG.error("Unrecognized log path format: " + filePath);
+        return;
+      }
+      final String logMsgPrefix =
+          "[" + serverType + (serverPort != 0 ? ":" + serverPort : "") + "] ";
+
+      LOG.debug("Tailing " + filePath);
+      Thread t = new Thread(new Runnable() {
+        @Override
+        public void run() {
+          try {
+            FileInputStream fis = new FileInputStream(filePath);
+            BufferedReader br = new BufferedReader(new InputStreamReader(fis));
+            String line;
+            while (true) {
+              try {
+                Thread.sleep(200);
+              } catch (InterruptedException e) {
+                LOG.error("Tailer for " + filePath + " interrupted");
+                break;
+              }
+              while ((line = br.readLine()) != null) {
+                line = logMsgPrefix + processLine(line);
+                synchronized (printLock) {
+                  dest.println(line);
+                }
+              }
+            }
+          } catch (IOException ex) {
+            LOG.error("Failed tailing " + filePath, ex);
+          }
+        }
+      });
+      t.setDaemon(true);
+      t.setName("Tailer for " + filePath);
+      t.start();
+    }
+
+  }
+
+  private void startDaemonLogTailer() {
+    logTailerThread = new Thread(new LocalDaemonLogTailer());
+    logTailerThread.setDaemon(true);
+    logTailerThread.start();
+  }
+
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java?rev=1326045&r1=1326044&r2=1326045&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
Sat Apr 14 02:46:26 2012
@@ -16,7 +16,6 @@
  */
 package org.apache.hadoop.hbase.util;
 
-import java.io.File;
 import java.io.IOException;
 
 import org.apache.commons.cli.CommandLine;
@@ -52,14 +51,12 @@ public class RestartMetaTest extends Abs
   /** The actual number of region servers */
   private int numRegionServers;
 
-  /** HBase home source tree home directory */
-  private String hbaseHome;
-
-  private static final String OPT_HBASE_HOME = "hbase_home";
   private static final String OPT_NUM_RS = "num_rs";
-  
+
   private static final byte[] COLUMN_FAMILY = Bytes.toBytes("testCF");
 
+  private static final int NUM_DATANODES = 3;
+
   /** Loads data into the table using the multi-threaded writer. */
   private void loadData() throws IOException {
     long startKey = 0;
@@ -93,10 +90,11 @@ public class RestartMetaTest extends Abs
   @Override
   protected void doWork() throws IOException {
     ProcessBasedLocalHBaseCluster hbaseCluster =
-        new ProcessBasedLocalHBaseCluster(conf, hbaseHome, numRegionServers);
+        new ProcessBasedLocalHBaseCluster(conf, NUM_DATANODES, numRegionServers);
+    hbaseCluster.startMiniDFS();
 
     // start the process based HBase cluster
-    hbaseCluster.start();
+    hbaseCluster.startHBase();
 
     // create tables if needed
     HBaseTestingUtility.createPreSplitLoadTestTable(conf, TABLE_NAME,
@@ -136,19 +134,11 @@ public class RestartMetaTest extends Abs
 
   @Override
   protected void addOptions() {
-    addRequiredOptWithArg(OPT_HBASE_HOME, "HBase home directory");
     addOptWithArg(OPT_NUM_RS, "Number of Region Servers");
   }
 
   @Override
   protected void processOptions(CommandLine cmd) {
-    hbaseHome = cmd.getOptionValue(OPT_HBASE_HOME);
-    if (hbaseHome == null || !new File(hbaseHome).isDirectory()) {
-      throw new IllegalArgumentException("Invalid HBase home directory: " +
-          hbaseHome);
-    }
-
-    LOG.info("Using HBase home directory " + hbaseHome);
     numRegionServers = Integer.parseInt(cmd.getOptionValue(OPT_NUM_RS,
         String.valueOf(DEFAULT_NUM_RS)));
   }

Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestProcessBasedCluster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestProcessBasedCluster.java?rev=1326045&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestProcessBasedCluster.java
(added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestProcessBasedCluster.java
Sat Apr 14 02:46:26 2012
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTestConst;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.junit.Test;
+
+/**
+ * A basic unit test that spins up a local HBase cluster.
+ */
+public class TestProcessBasedCluster {
+
+  private static final Log LOG = LogFactory.getLog(TestProcessBasedCluster.class);
+
+  private static final int COLS_PER_ROW = 5;
+  private static final int FLUSHES = 5;
+  private static final int NUM_REGIONS = 5;
+  private static final int ROWS_PER_FLUSH = 5;
+
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @Test(timeout=300 * 1000)
+  public void testProcessBasedCluster() throws Exception {
+    ProcessBasedLocalHBaseCluster cluster = new ProcessBasedLocalHBaseCluster(
+        TEST_UTIL.getConfiguration(), 2, 3);
+    cluster.startMiniDFS();
+    cluster.startHBase();
+    try {
+      TEST_UTIL.createRandomTable(HTestConst.DEFAULT_TABLE_STR,
+          HTestConst.DEFAULT_CF_STR_SET,
+          HColumnDescriptor.DEFAULT_VERSIONS, COLS_PER_ROW, FLUSHES, NUM_REGIONS,
+          ROWS_PER_FLUSH);
+      HTable table = new HTable(TEST_UTIL.getConfiguration(), HTestConst.DEFAULT_TABLE_BYTES);
+      ResultScanner scanner = table.getScanner(HTestConst.DEFAULT_CF_BYTES);
+      Result result;
+      int rows = 0;
+      int cols = 0;
+      while ((result = scanner.next()) != null) {
+        ++rows;
+        cols += result.getFamilyMap(HTestConst.DEFAULT_CF_BYTES).size();
+      }
+      LOG.info("Read " + rows + " rows, " + cols + " columns");
+      scanner.close();
+      table.close();
+
+      // These numbers are deterministic, seeded by table name.
+      assertEquals(19, rows);
+      assertEquals(35, cols);
+    } catch (Exception ex) {
+      LOG.error(ex);
+      throw ex;
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  @Test
+  public void testHomePath() {
+    File pom = new File(HBaseHomePath.getHomePath(), "pom.xml");
+    assertTrue(pom.getPath() + " does not exist", pom.exists());
+  }
+
+}



Mime
View raw message