hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1310612 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/master/ main/java/org/apache/hadoop/hbase/regionserver/ test/java/org/apache/hadoop/hbase/ test/java/org/apache/hadoop/hbase/cli...
Date Fri, 06 Apr 2012 22:13:18 GMT
Author: mbautin
Date: Fri Apr  6 22:13:16 2012
New Revision: 1310612

URL: http://svn.apache.org/viewvc?rev=1310612&view=rev
Log:
[jira] [HBASE-5708] [89-fb] Improving robustness of map-reduce-related and other unit tests

Summary:
Some map-reduce-based tests are failing when executed concurrently in 89-fb
because mini-map-reduce cluster uses /tmp/hadoop-<username> for temporary
data.

Test Plan:
Run unit tests
Run map-reduce-based unit tests 50 times each

Reviewers: kannan, kranganathan, liyintang, pkhemani

Reviewed By: liyintang

CC: stack

Differential Revision: https://reviews.facebook.net/D2601

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
    hbase/branches/0.89-fb/src/test/resources/hbase-site.xml

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Fri Apr  6 22:13:16 2012
@@ -479,6 +479,8 @@ public final class HConstants {
   /** Delay when waiting for a variable (HBASE-4712) */
   public static final int VARIABLE_WAIT_TIME_MS = 40;
 
+  public static final String LOAD_BALANCER_SLOP_KEY = "hbase.regions.slop";
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java Fri Apr  6 22:13:16 2012
@@ -95,6 +95,10 @@ public class OldLogsCleaner extends Chor
   protected void chore() {
     try {
       FileStatus[] files = this.fs.listStatus(this.oldLogDir);
+      if (files == null) {
+        // We don't have any files to process.
+        return;
+      }
       int nbDeletedLog = 0;
       for (FileStatus file : files) {
         Path filePath = file.getPath();

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java Fri Apr  6 22:13:16 2012
@@ -1789,7 +1789,7 @@ public class RegionManager {
 
     LoadBalancer() {
       Configuration conf = master.getConfiguration();
-      float confSlop = conf.getFloat("hbase.regions.slop", (float)0.3);
+      float confSlop = conf.getFloat(HConstants.LOAD_BALANCER_SLOP_KEY, (float) 0.3);
       this.slop = confSlop <= 0 ? 1 : confSlop;
       this.maxRegToClose = conf.getInt("hbase.regions.close.max", -1);
     }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java Fri Apr  6 22:13:16 2012
@@ -1005,6 +1005,10 @@ public class SplitLogManager implements 
       byte[] newData = RecoverableZooKeeper.removeMetaData(data);
       tot_mgr_get_data_result.incrementAndGet();
       if (rc != 0) {
+        if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) {
+          LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries.");
+          return;
+        }
         if (rc == KeeperException.Code.NONODE.intValue()) {
           tot_mgr_get_data_nonode.incrementAndGet();
           // The task znode has been deleted. Must be some pending delete
@@ -1093,6 +1097,10 @@ public class SplitLogManager implements 
     @Override
     public void processResult(int rc, String path, Object ctx, String name) {
       if (rc != 0) {
+        if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) {
+          LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries.");
+          return;
+        }
         Long retry_count = (Long)ctx;
         LOG.warn("rc=" + KeeperException.Code.get(rc) + " for "+ path +
             " retry=" + retry_count);

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri Apr  6 22:13:16 2012
@@ -1582,21 +1582,25 @@ public class HRegionServer implements HR
 
             case MSG_REGION_OPEN:
               // Open a region
+              boolean requeued = false;
               if (!haveRootRegion.get() && !info.isRootRegion()) {
                 // root region is not online yet. requeue this task
                 LOG.info("putting region open request back into queue because" +
                     " root region is not yet available");
                 try {
                   toDo.put(e);
+                  requeued = true;
                 } catch (InterruptedException ex) {
                   LOG.warn("insertion into toDo queue was interrupted", ex);
                   break;
                 }
               }
-              if (e.msg.getMessage() != null && e.msg.getMessage().length > 0) {
-                openRegion(info, new String(e.msg.getMessage()));
-              } else {
-                openRegion(info, null);
+              if (!requeued) {
+                if (e.msg.getMessage() != null && e.msg.getMessage().length > 0) {
+                  openRegion(info, new String(e.msg.getMessage()));
+                } else {
+                  openRegion(info, null);
+                }
               }
               break;
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java Fri Apr  6 22:13:16 2012
@@ -19,7 +19,6 @@
  */
 package org.apache.hadoop.hbase;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.util.Iterator;
@@ -53,9 +52,6 @@ import org.apache.hadoop.hdfs.MiniDFSClu
 public abstract class HBaseTestCase extends TestCase {
   private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
 
-  /** configuration parameter name for test directory */
-  public static final String TEST_DIRECTORY_KEY = "test.build.data";
-
   protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
   protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
   protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
@@ -73,9 +69,7 @@ public abstract class HBaseTestCase exte
   protected String START_KEY;
   protected static final int MAXVERSIONS = 3;
 
-  static {
-    initialize();
-  }
+  protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
 
   public volatile HBaseConfiguration conf;
 
@@ -147,8 +141,7 @@ public abstract class HBaseTestCase exte
   }
 
   protected Path getUnitTestdir(String testName) {
-    return new Path(
-        conf.get(TEST_DIRECTORY_KEY, "target/test/data"), testName);
+    return testUtil.getTestDir(testName);
   }
 
   protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
@@ -600,21 +593,6 @@ public abstract class HBaseTestCase exte
     }
 
   /**
-   * Initializes parameters used in the test environment:
-   *
-   * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set.
-   * Sets the boolean debugging if "DEBUGGING" is set in the environment.
-   * If debugging is enabled, reconfigures logging so that the root log level is
-   * set to WARN and the logging level for the package is set to DEBUG.
-   */
-  public static void initialize() {
-    if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
-      System.setProperty(TEST_DIRECTORY_KEY, new File(
-          "build/hbase/test").getAbsolutePath());
-    }
-  }
-
-  /**
    * Common method to close down a MiniDFSCluster and the associated file system
    *
    * @param cluster

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Fri Apr  6 22:13:16 2012
@@ -26,6 +26,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
@@ -53,7 +54,6 @@ import org.apache.hadoop.hbase.client.De
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.NoServerForRegionException;
 import org.apache.hadoop.hbase.client.Put;
@@ -73,7 +73,6 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.RegionSplitter;
@@ -85,6 +84,7 @@ import org.apache.hadoop.hdfs.Distribute
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.TaskLog;
 import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.zookeeper.ZooKeeper;
@@ -114,10 +114,17 @@ public class HBaseTestingUtility {
   private MiniDFSCluster dfsCluster = null;
   private MiniHBaseCluster hbaseCluster = null;
   private MiniMRCluster mrCluster = null;
-  // If non-null, then already a cluster running.
+
+  /** The root directory for all mini-cluster test data for this testing utility instance. */
   private File clusterTestBuildDir = null;
+
+  /** If there is a mini cluster running for this testing utility instance. */
+  private boolean miniClusterRunning;
+
   private HBaseAdmin hbaseAdmin = null;
 
+  private String hadoopLogDir;
+
   /**
    * System property key to get test directory value.
    */
@@ -128,6 +135,9 @@ public class HBaseTestingUtility {
    */
   public static final String DEFAULT_TEST_DIRECTORY = "target/build/data";
 
+  /** Filesystem URI used for map-reduce mini-cluster setup */
+  private static String fsURI;
+
   /** Compression algorithms to use in parameterized JUnit 4 tests */
   public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
     Arrays.asList(new Object[][] {
@@ -182,25 +192,13 @@ public class HBaseTestingUtility {
   }
 
   /**
-   * Makes sure the test directory is set up so that {@link #getTestDir()}
-   * returns a valid directory. Useful in unit tests that do not run a
-   * mini-cluster.
-   */
-  public void initTestDir() {
-    if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
-      clusterTestBuildDir = setupClusterTestBuildDir();
-      System.setProperty(TEST_DIRECTORY_KEY, clusterTestBuildDir.getPath());
-    }
-  }
-
-  /**
    * @return Where to write test data on local filesystem; usually
    * {@link #DEFAULT_TEST_DIRECTORY}
    * @see #setupClusterTestBuildDir()
    */
-  public static Path getTestDir() {
-    return new Path(System.getProperty(TEST_DIRECTORY_KEY,
-      DEFAULT_TEST_DIRECTORY));
+  public Path getTestDir() {
+    setupClusterTestBuildDir();
+    return new Path(clusterTestBuildDir.getPath());
   }
 
   /**
@@ -209,7 +207,7 @@ public class HBaseTestingUtility {
    * {@link #getTestDir()}.
    * @see #setupClusterTestBuildDir()
    */
-  public static Path getTestDir(final String subdirName) {
+  public Path getTestDir(final String subdirName) {
     return new Path(getTestDir(), subdirName);
   }
 
@@ -223,33 +221,18 @@ public class HBaseTestingUtility {
    * single instance only is how the minidfscluster works.
    * @return The calculated cluster test build directory.
    */
-  File setupClusterTestBuildDir() {
-    String randomStr = UUID.randomUUID().toString();
-    String dirStr = getTestDir(randomStr).toString();
-    File dir = new File(dirStr).getAbsoluteFile();
-    // Have it cleaned up on exit
-    dir.deleteOnExit();
-    return dir;
-  }
-
-  /**
-   * @throws IOException If a cluster -- zk, dfs, or hbase -- already running.
-   */
-  void isRunningCluster() throws IOException {
-    if (this.clusterTestBuildDir == null) return;
-    throw new IOException("Cluster already running at " +
-      this.clusterTestBuildDir);
-  }
+  public void setupClusterTestBuildDir() {
+    if (clusterTestBuildDir != null) {
+      return;
+    }
 
-  /**
-   * Start a minidfscluster.
-   * @param servers How many DNs to start.
-   * @throws Exception
-   * @see {@link #shutdownMiniDFSCluster()}
-   * @return The mini dfs cluster created.
-   */
-  public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception {
-    return startMiniDFSCluster(servers, null);
+    String randomStr = UUID.randomUUID().toString();
+    String dirStr = new Path(DEFAULT_TEST_DIRECTORY, randomStr).toString();
+    clusterTestBuildDir = new File(dirStr).getAbsoluteFile();
+    clusterTestBuildDir.mkdirs();
+    clusterTestBuildDir.deleteOnExit();
+    conf.set(TEST_DIRECTORY_KEY, clusterTestBuildDir.getPath());
+    LOG.info("Created new mini-cluster data directory: " + clusterTestBuildDir);
   }
 
   /**
@@ -260,31 +243,44 @@ public class HBaseTestingUtility {
    * @see {@link #shutdownMiniDFSCluster()}
    * @return The mini dfs cluster created.
    */
-  public MiniDFSCluster startMiniDFSCluster(int servers, final File dir)
-      throws IOException {
-    // This does the following to home the minidfscluster
-    //     base_dir = new File(System.getProperty("test.build.data", "build/test/data"), "dfs/");
-    // Some tests also do this:
-    //  System.getProperty("test.cache.data", "build/test/cache");
-    if (dir == null) this.clusterTestBuildDir = setupClusterTestBuildDir();
-    else this.clusterTestBuildDir = dir;
-    System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestBuildDir.toString());
-    System.setProperty("test.cache.data", this.clusterTestBuildDir.toString());
+  public MiniDFSCluster startMiniDFSCluster(int servers) throws IOException {
+    createDirsAndSetProperties();
     this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
       true, null, null, null, null);
     return this.dfsCluster;
   }
 
+  public MiniDFSCluster startMiniDFSClusterForTestHLog(int namenodePort) throws IOException {
+    createDirsAndSetProperties();
+    dfsCluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null,
+        null, null, null);
+    return dfsCluster;
+  }
+
+  /** This is used before starting HDFS and map-reduce mini-clusters */
+  private void createDirsAndSetProperties() {
+    setupClusterTestBuildDir();
+    System.setProperty(TEST_DIRECTORY_KEY, clusterTestBuildDir.getPath());
+    createDirAndSetProperty("cache_data", "test.cache.data");
+    createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
+    hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
+    createDirAndSetProperty("mapred_output", "mapred.output.dir");
+    createDirAndSetProperty("mapred_local", "mapred.local.dir");
+    createDirAndSetProperty("mapred_system", "mapred.system.dir");
+    createDirAndSetProperty("mapred_temp", "mapred.temp.dir");
+  }
+
   /**
-   * Shuts down instance created by call to {@link #startMiniDFSCluster(int, File)}
+   * Shuts down instance created by call to {@link #startMiniDFSCluster(int)}
    * or does nothing.
+   * @throws IOException
    * @throws Exception
    */
-  public void shutdownMiniDFSCluster() throws Exception {
+  public void shutdownMiniDFSCluster() throws IOException {
     if (this.dfsCluster != null) {
-      FileSystem.closeAll();
       // The below throws an exception per dn, AsynchronousCloseException.
       this.dfsCluster.shutdown();
+      dfsCluster = null;
     }
   }
 
@@ -296,8 +292,8 @@ public class HBaseTestingUtility {
    * @return zk cluster started.
    */
   public MiniZooKeeperCluster startMiniZKCluster() throws Exception {
-    return startMiniZKCluster(setupClusterTestBuildDir());
-
+    setupClusterTestBuildDir();
+    return startMiniZKCluster(clusterTestBuildDir);
   }
 
   private MiniZooKeeperCluster startMiniZKCluster(final File dir)
@@ -373,14 +369,18 @@ public class HBaseTestingUtility {
       final int numSlaves) throws IOException, InterruptedException {
     LOG.info("Starting up minicluster");
     // If we already put up a cluster, fail.
-    isRunningCluster();
+    if (miniClusterRunning) {
+      throw new IllegalStateException("A mini-cluster is already running");
+    }
+    miniClusterRunning = true;
+
     // Make a new random dir to home everything in.  Set it as system property.
     // minidfs reads home from system property.
-    this.clusterTestBuildDir = setupClusterTestBuildDir();
+    setupClusterTestBuildDir();
     System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestBuildDir.getPath());
     // Bring up mini dfs cluster. This spews a bunch of warnings about missing
     // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'.
-    startMiniDFSCluster(numSlaves, this.clusterTestBuildDir);
+    startMiniDFSCluster(numSlaves);
 
     // Mangle conf so fs parameter points to minidfs we just started up
     FileSystem fs = this.dfsCluster.getFileSystem();
@@ -445,23 +445,14 @@ public class HBaseTestingUtility {
       this.hbaseCluster.shutdown();
       // Wait till hbase is down before going on to shutdown zk.
       this.hbaseCluster.join();
+      hbaseCluster = null;
     }
     shutdownMiniZKCluster();
-    if (this.dfsCluster != null) {
-      // The below throws an exception per dn, AsynchronousCloseException.
-      this.dfsCluster.shutdown();
-    }
-    // Clean up our directory.
-    if (this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) {
-      // Need to use deleteDirectory because File.delete required dir is empty.
-      if (!FSUtils.deleteDirectory(FileSystem.getLocal(this.conf),
-          new Path(this.clusterTestBuildDir.toString()))) {
-        LOG.warn("Failed delete of " + this.clusterTestBuildDir.toString());
-      }
-    }
-    clusterTestBuildDir = null;
+    shutdownMiniDFSCluster();
+
+    cleanupTestDir();
+    miniClusterRunning = false;
     LOG.info("Minicluster is down");
-    clusterTestBuildDir = null;
   }
 
   /**
@@ -845,29 +836,76 @@ public class HBaseTestingUtility {
    *
    * @throws IOException When starting the cluster fails.
    */
-  public void startMiniMapReduceCluster() throws IOException {
+  public MiniMRCluster startMiniMapReduceCluster() throws IOException {
     startMiniMapReduceCluster(2);
+    return mrCluster;
+  }
+
+  /**
+   * Tasktracker has a bug where changing the hadoop.log.dir system property
+   * will not change its internal static LOG_DIR variable.
+   */
+  private void forceChangeTaskLogDir() {
+    Field logDirField;
+    try {
+      logDirField = TaskLog.class.getDeclaredField("LOG_DIR");
+      logDirField.setAccessible(true);
+
+      Field modifiersField = Field.class.getDeclaredField("modifiers");
+      modifiersField.setAccessible(true);
+      modifiersField.setInt(logDirField, logDirField.getModifiers() & ~Modifier.FINAL);
+
+      logDirField.set(null, new File(hadoopLogDir, "userlogs"));
+    } catch (SecurityException e) {
+      throw new RuntimeException(e);
+    } catch (NoSuchFieldException e) {
+      // TODO Auto-generated catch block
+      throw new RuntimeException(e);
+    } catch (IllegalArgumentException e) {
+      throw new RuntimeException(e);
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException(e);
+    }
   }
 
   /**
-   * Starts a <code>MiniMRCluster</code>.
+   * Starts a <code>MiniMRCluster</code>. Call {@link #setFileSystemURI(String)} to use a different
+   * filesystem.
    *
    * @param servers  The number of <code>TaskTracker</code>'s to start.
    * @throws IOException When starting the cluster fails.
    */
-  public void startMiniMapReduceCluster(final int servers) throws IOException {
+  private void startMiniMapReduceCluster(final int servers) throws IOException {
+    if (mrCluster != null) {
+      throw new IllegalStateException("MiniMRCluster is already running");
+    }
     LOG.info("Starting mini mapreduce cluster...");
     // These are needed for the new and improved Map/Reduce framework
     Configuration c = getConfiguration();
-    System.setProperty("hadoop.log.dir", c.get("hadoop.log.dir"));
-    c.set("mapred.output.dir", c.get("hadoop.tmp.dir"));
+
+    setupClusterTestBuildDir();
+    createDirsAndSetProperties();
+
+    forceChangeTaskLogDir();
+
+    // Allow the user to override FS URI for this map-reduce cluster to use.
     mrCluster = new MiniMRCluster(servers,
-      FileSystem.get(c).getUri().toString(), 1);
+        fsURI != null ? fsURI : FileSystem.get(c).getUri().toString(), 1);
+
     LOG.info("Mini mapreduce cluster started");
     c.set("mapred.job.tracker",
         mrCluster.createJobConf().get("mapred.job.tracker"));
   }
 
+  private String createDirAndSetProperty(final String relPath, String property) {
+    String path = clusterTestBuildDir.getPath() + "/" + relPath;
+    System.setProperty(property, path);
+    conf.set(property, path);
+    new File(path).mkdirs();
+    LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
+    return path;
+  }
+
   /**
    * Stops the previously started <code>MiniMRCluster</code>.
    */
@@ -875,6 +913,7 @@ public class HBaseTestingUtility {
     LOG.info("Stopping mini mapreduce cluster...");
     if (mrCluster != null) {
       mrCluster.shutdown();
+      mrCluster = null;
     }
     // Restore configuration to point to local jobtracker
     conf.set("mapred.job.tracker", "local");
@@ -920,15 +959,14 @@ public class HBaseTestingUtility {
     int sessionTimeout = nodeZK.getSessionTimeout();
     byte[] password = nodeZK.getSessionPassword();
     long sessionID = nodeZK.getSessionID();
-    final long sleep = sessionTimeout * 10L;
-    final int maxRetryNum = 50;
+    final int sleep = 1000;
+    final int maxRetryNum = 60 * 1000 / sleep;  // Wait for a total of up to one minute.
     int retryNum = maxRetryNum;
 
     ZooKeeper zk = new ZooKeeper(quorumServers,
         sessionTimeout, EmptyWatcher.instance, sessionID, password);
+    LOG.debug("Closing ZK");
     zk.close();
-    Thread.sleep(sleep);
-		LOG.debug("ZooKeeper is closed");
 
     while (!nodeZK.isAborted() && retryNum != 0) {
       Thread.sleep(sleep);
@@ -937,6 +975,7 @@ public class HBaseTestingUtility {
     if (retryNum == 0) {
       fail("ZooKeeper is not aborted after " + maxRetryNum + " attempts.");
     }
+    LOG.debug("ZooKeeper is closed");
   }
 
   /**
@@ -1022,10 +1061,6 @@ public class HBaseTestingUtility {
     return FileSystem.get(conf);
   }
 
-  public void cleanupTestDir() throws IOException {
-    getTestDir().getFileSystem(conf).delete(getTestDir(), true);
-  }
-
   public void waitTableAvailable(byte[] table, long timeoutMillis)
   throws InterruptedException, IOException {
     HBaseAdmin admin = new HBaseAdmin(conf);
@@ -1471,4 +1506,20 @@ REGION_LOOP:
     return region;
   }
 
+  /**
+   * Useful for tests that do not start a full cluster, e.g. those that use a mini MR cluster only.
+   */
+  public void cleanupTestDir() throws IOException {
+    if (this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) {
+      // Need to use deleteDirectory because File.delete required dir is empty.
+      if (!FSUtils.deleteDirectory(FileSystem.getLocal(this.conf),
+          new Path(this.clusterTestBuildDir.toString()))) {
+        LOG.warn("Failed delete of " + this.clusterTestBuildDir.toString());
+      }
+    }
+  }
+
+  public void setFileSystemURI(String fsURI) {
+    this.fsURI = fsURI;
+  }
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java Fri Apr  6 22:13:16 2012
@@ -59,6 +59,7 @@ public class TestFullLogReconstruction {
     c.setInt("ipc.client.connect.max.retries", 1);
     c.setInt("dfs.client.block.recovery.retries", 1);
     c.setInt("hbase.regionserver.flushlogentries", 1);
+    c.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 1000);
     TEST_UTIL.startMiniCluster(NUM_REGIONSERVER);
   }
 
@@ -118,8 +119,10 @@ public class TestFullLogReconstruction {
     }
 
     TEST_UTIL.expireRegionServerSession(0);
+
     // wait for meta region comes online
-    Thread.sleep(1000);
+    TEST_UTIL.waitTableAvailable(HConstants.META_TABLE_NAME, 10000);
+
     scan = new Scan();
     results = table.getScanner(scan);
     int newCount = 0;

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java Fri Apr  6 22:13:16 2012
@@ -61,10 +61,11 @@ public class TestZooKeeper {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     // Test we can first start the ZK cluster by itself
+    conf = TEST_UTIL.getConfiguration();
     TEST_UTIL.startMiniZKCluster();
-    TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
+    conf.setBoolean("dfs.support.append", true);
+    conf.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 1000);
     TEST_UTIL.startMiniCluster(NUM_REGIONSERVER);
-    conf = TEST_UTIL.getConfiguration();
   }
 
   /**

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java Fri Apr  6 22:13:16 2012
@@ -3822,7 +3822,7 @@ public class TestFromClientSide {
      // Create multiple regions for this table
      TEST_UTIL.createMultiRegions(table, FAMILY);
 
-     Path tempPath = new Path(HBaseTestingUtility.getTestDir(), "regions.dat");
+     Path tempPath = new Path(TEST_UTIL.getTestDir(), "regions.dat");
 
      final String tempFileName = tempPath.toString();
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java Fri Apr  6 22:13:16 2012
@@ -38,8 +38,8 @@ public class TestColumnPrefixFilter {
     HTableDescriptor htd = new HTableDescriptor("TestColumnPrefixFilter");
     htd.addFamily(new HColumnDescriptor(family));
     HRegionInfo info = new HRegionInfo(htd, null, null, false);
-    HRegion region = HRegion.createHRegion(info, HBaseTestingUtility.
-        getTestDir(), TEST_UTIL.getConfiguration());
+    HRegion region = HRegion.createHRegion(info,
+        TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
     String row;
     Put p;
     row = "001";
@@ -73,8 +73,8 @@ public class TestColumnPrefixFilter {
     HTableDescriptor htd = new HTableDescriptor("TestColumnPrefixFilter");
     htd.addFamily(new HColumnDescriptor(family));
     HRegionInfo info = new HRegionInfo(htd, null, null, false);
-    HRegion region = HRegion.createHRegion(info, HBaseTestingUtility.
-        getTestDir(), TEST_UTIL.getConfiguration());
+    HRegion region = HRegion.createHRegion(info,
+        TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
 
     List<String> rows = generateRandomWords(100, "row");
     List<String> columns = generateRandomWords(10000, "column");
@@ -129,8 +129,8 @@ public class TestColumnPrefixFilter {
     HTableDescriptor htd = new HTableDescriptor("TestColumnPrefixFilter");
     htd.addFamily(new HColumnDescriptor(family));
     HRegionInfo info = new HRegionInfo(htd, null, null, false);
-    HRegion region = HRegion.createHRegion(info, HBaseTestingUtility.
-        getTestDir(), TEST_UTIL.getConfiguration());
+    HRegion region = HRegion.createHRegion(info,
+        TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
 
     List<String> rows = generateRandomWords(100, "row");
     List<String> columns = generateRandomWords(10000, "column");

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java Fri Apr  6 22:13:16 2012
@@ -59,7 +59,7 @@ public class TestHalfStoreFileReader {
   @Test
   public void testHalfScanAndReseek() throws IOException {
     HBaseTestingUtility test_util = new HBaseTestingUtility();
-    String root_dir = HBaseTestingUtility.getTestDir("TestHalfStoreFile").toString();
+    String root_dir = test_util.getTestDir("TestHalfStoreFile").toString();
     Path p = new Path(root_dir, "test");
 
     Configuration conf = test_util.getConfiguration();

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java Fri Apr  6 22:13:16 2012
@@ -260,7 +260,7 @@ public class TestCacheOnWrite {
   }
 
   public void writeStoreFile() throws IOException {
-    Path storeFileParentDir = new Path(HBaseTestingUtility.getTestDir(),
+    Path storeFileParentDir = new Path(TEST_UTIL.getTestDir(),
         "test_cache_on_write");
     StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, cacheConf, fs,
         DATA_BLOCK_SIZE)

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java Fri Apr  6 22:13:16 2012
@@ -127,7 +127,7 @@ public class TestFixedFileTrailer {
     }
 
     // Now check what happens if the trailer is corrupted.
-    Path trailerPath = new Path(HBaseTestingUtility.getTestDir(), "trailer_"
+    Path trailerPath = new Path(util.getTestDir(), "trailer_"
         + version);
 
     {

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java Fri Apr  6 22:13:16 2012
@@ -51,8 +51,9 @@ import org.apache.hadoop.io.Writable;
 public class TestHFile extends HBaseTestCase {
   static final Log LOG = LogFactory.getLog(TestHFile.class);
 
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static String ROOT_DIR =
-    HBaseTestingUtility.getTestDir("TestHFile").toString();
+      TEST_UTIL.getTestDir("TestHFile").toString();
   private final int minBlockSize = 512;
   private static String localFormatter = "%010d";
   private Map<String, Long> startingMetrics;

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java Fri Apr  6 22:13:16 2012
@@ -100,7 +100,7 @@ public class TestHFileBlock {
   @Before
   public void setUp() throws IOException {
     fs = FileSystem.get(TEST_UTIL.getConfiguration());
-    TEST_UTIL.initTestDir();
+    TEST_UTIL.setupClusterTestBuildDir();
   }
 
   public void writeTestBlockContents(DataOutputStream dos) throws IOException {
@@ -249,7 +249,7 @@ public class TestHFileBlock {
     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
       for (boolean pread : new boolean[] { false, true }) {
         byte[] block = createTestV1Block(algo);
-        Path path = new Path(HBaseTestingUtility.getTestDir(), "blocks_v1_"
+        Path path = new Path(TEST_UTIL.getTestDir(), "blocks_v1_"
             + algo);
         LOG.info("Creating temporary file at " + path);
         FSDataOutputStream os = fs.create(path);
@@ -283,7 +283,7 @@ public class TestHFileBlock {
   public void testReaderV2() throws IOException {
     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
       for (boolean pread : new boolean[] { false, true }) {
-        Path path = new Path(HBaseTestingUtility.getTestDir(), "blocks_v2_"
+        Path path = new Path(TEST_UTIL.getTestDir(), "blocks_v2_"
             + algo);
         FSDataOutputStream os = fs.create(path);
         HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
@@ -343,7 +343,7 @@ public class TestHFileBlock {
     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
       for (boolean pread : new boolean[] { false, true }) {
         for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
-          Path path = new Path(HBaseTestingUtility.getTestDir(), "blocks_v2_"
+          Path path = new Path(TEST_UTIL.getTestDir(), "blocks_v2_"
               + algo + "_" + encoding.toString());
           FSDataOutputStream os = fs.create(path);
           HFileDataBlockEncoder dataBlockEncoder =
@@ -470,7 +470,7 @@ public class TestHFileBlock {
         for (boolean cacheOnWrite : BOOLEAN_VALUES) {
           Random rand = defaultRandom();
           LOG.info("Compression algorithm: " + algo + ", pread=" + pread);
-          Path path = new Path(HBaseTestingUtility.getTestDir(), "prev_offset");
+          Path path = new Path(TEST_UTIL.getTestDir(), "prev_offset");
           List<Long> expectedOffsets = new ArrayList<Long>();
           List<Long> expectedPrevOffsets = new ArrayList<Long>();
           List<BlockType> expectedTypes = new ArrayList<BlockType>();
@@ -626,7 +626,7 @@ public class TestHFileBlock {
   public void testConcurrentReading() throws Exception {
     for (Compression.Algorithm compressAlgo : COMPRESSION_ALGORITHMS) {
       Path path =
-          new Path(HBaseTestingUtility.getTestDir(), "concurrent_reading");
+          new Path(TEST_UTIL.getTestDir(), "concurrent_reading");
       Random rand = defaultRandom();
       List<Long> offsets = new ArrayList<Long>();
       List<BlockType> types = new ArrayList<BlockType>();

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java Fri Apr  6 22:13:16 2012
@@ -114,7 +114,7 @@ public class TestHFileBlockIndex {
 
   @Test
   public void testBlockIndex() throws IOException {
-    path = new Path(HBaseTestingUtility.getTestDir(), "block_index_" + compr);
+    path = new Path(TEST_UTIL.getTestDir(), "block_index_" + compr);
     writeWholeIndex();
     readIndex();
   }
@@ -461,7 +461,7 @@ public class TestHFileBlockIndex {
    */
   @Test
   public void testHFileWriterAndReader() throws IOException {
-    Path hfilePath = new Path(HBaseTestingUtility.getTestDir(),
+    Path hfilePath = new Path(TEST_UTIL.getTestDir(),
         "hfile_for_block_index");
     CacheConfig cacheConf = new CacheConfig(conf);
     BlockCache blockCache = cacheConf.getBlockCache();

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java Fri Apr  6 22:13:16 2012
@@ -74,14 +74,6 @@ public class TestHFileDataBlockEncoder {
   }
 
   /**
-   * Cleanup after JUnit test.
-   */
-  @After
-  public void tearDown() throws IOException {
-    TEST_UTIL.cleanupTestDir();
-  }
-
-  /**
    * Test putting and taking out blocks into cache with different
    * encoding options.
    */

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java Fri Apr  6 22:13:16 2012
@@ -73,8 +73,9 @@ class RawKeyValue implements Comparable<
  * instead.</p>
  */
 public class TestHFilePerformance extends TestCase {
+  private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static String ROOT_DIR =
-    HBaseTestingUtility.getTestDir("TestHFilePerformance").toString();
+    TEST_UTIL.getTestDir("TestHFilePerformance").toString();
   private FileSystem fs;
   private Configuration conf;
   private long startTimeEpoch;

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java Fri Apr  6 22:13:16 2012
@@ -60,6 +60,8 @@ public class TestHFileSeek extends TestC
   private RandomDistribution.DiscreteRNG keyLenGen;
   private KVGenerator kvGen;
 
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
   @Override
   public void setUp() throws IOException {
     if (options == null) {
@@ -239,7 +241,7 @@ public class TestHFileSeek extends TestC
     int maxWordLen = 20;
 
     String rootDir =
-      HBaseTestingUtility.getTestDir("TestTFileSeek").toString();
+      TEST_UTIL.getTestDir("TestTFileSeek").toString();
     String file = "TestTFileSeek";
     // String compress = "lzo"; DISABLED
     String compress = "none";

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java Fri Apr  6 22:13:16 2012
@@ -68,7 +68,7 @@ public class TestHFileWriterV2 {
 
   @Test
   public void testHFileFormatV2() throws IOException {
-    Path hfilePath = new Path(HBaseTestingUtility.getTestDir(),
+    Path hfilePath = new Path(TEST_UTIL.getTestDir(),
         "testHFileFormatV2");
 
     final Compression.Algorithm COMPRESS_ALGO = Compression.Algorithm.GZ;

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java Fri Apr  6 22:13:16 2012
@@ -40,7 +40,7 @@ public class TestReseekTo {
   @Test
   public void testReseekTo() throws Exception {
 
-    Path ncTFile = new Path(HBaseTestingUtility.getTestDir(), "basic.hfile");
+    Path ncTFile = new Path(TEST_UTIL.getTestDir(), "basic.hfile");
     FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
     CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
     HFile.Writer writer = HFile.getWriterFactory(

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java Fri Apr  6 22:13:16 2012
@@ -63,7 +63,7 @@ public class TestScannerSelectionUsingTT
   private static String FAMILY = "myCF";
   private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
 
-  private static final int TTL_SECONDS = 2;
+  private static final int TTL_SECONDS = 10;
   private static final int TTL_MS = TTL_SECONDS * 1000;
 
   private static final int NUM_EXPIRED_FILES = 2;
@@ -103,7 +103,7 @@ public class TestScannerSelectionUsingTT
     htd.addFamily(hcd);
     HRegionInfo info = new HRegionInfo(htd, null, null, false);
     HRegion region = HRegion.createHRegion(
-        info, HBaseTestingUtility.getTestDir(), TEST_UTIL.getConfiguration());
+        info, TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
 
     for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
       if (iFile == NUM_EXPIRED_FILES) {

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java Fri Apr  6 22:13:16 2012
@@ -19,20 +19,23 @@
  */
 package org.apache.hadoop.hbase.mapred;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.Map;
 import java.util.NavigableMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MultiRegionTable;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapred.JobClient;
@@ -126,7 +129,9 @@ public class TestLegacyTableMapReduce ex
   }
 
   private void runTestOnTable(HTable table) throws IOException {
-    MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1);
+    HBaseTestingUtility testUtil = new HBaseTestingUtility();
+    testUtil.setupClusterTestBuildDir();
+    MiniMRCluster mrCluster = testUtil.startMiniMapReduceCluster();
 
     JobConf jobConf = null;
     try {
@@ -148,9 +153,6 @@ public class TestLegacyTableMapReduce ex
       verify(Bytes.toString(table.getTableName()));
     } finally {
       mrCluster.shutdown();
-      if (jobConf != null) {
-        FileUtil.fullyDelete(new File(jobConf.get("hadoop.tmp.dir")));
-      }
     }
   }
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java Fri Apr  6 22:13:16 2012
@@ -56,12 +56,10 @@ import org.apache.hadoop.hbase.client.Ro
 import org.apache.hadoop.hbase.client.RowMutation;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
-import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -74,7 +72,6 @@ import org.apache.hadoop.mapreduce.Recor
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
 
@@ -158,11 +155,6 @@ public class TestHFileOutputFormat  {
     }
   }
 
-  @Before
-  public void cleanupDir() throws IOException {
-    util.cleanupTestDir();
-  }
-
   private void setupRandomGeneratorMapper(Job job) {
     job.setInputFormatClass(NMapInputFormat.class);
     job.setMapperClass(RandomKVGeneratingMapper.class);
@@ -255,7 +247,7 @@ public class TestHFileOutputFormat  {
     conf.setInt("io.sort.mb", 20);
     conf.setInt("mapred.map.tasks", 1);
 
-    Path dir = HBaseTestingUtility.getTestDir("testRowSortReducer");
+    Path dir = util.getTestDir("testRowSortReducer");
 
     try {
       Job job = new Job(conf);
@@ -403,7 +395,7 @@ public class TestHFileOutputFormat  {
     RecordWriter<ImmutableBytesWritable, KeyValue> writer = null;
     TaskAttemptContext context = null;
     Path dir =
-      HBaseTestingUtility.getTestDir("test_LATEST_TIMESTAMP_isReplaced");
+      util.getTestDir("test_LATEST_TIMESTAMP_isReplaced");
     try {
       Job job = new Job(conf);
       FileOutputFormat.setOutputPath(job, dir);
@@ -442,7 +434,7 @@ public class TestHFileOutputFormat  {
   @Test
   public void testWritingPEData() throws Exception {
     Configuration conf = util.getConfiguration();
-    Path testDir = HBaseTestingUtility.getTestDir("testWritingPEData");
+    Path testDir = util.getTestDir("testWritingPEData");
     FileSystem fs = testDir.getFileSystem(conf);
 
     // Set down this value or we OOME in eclipse.
@@ -496,18 +488,21 @@ public class TestHFileOutputFormat  {
 
   @Test
   public void testMRIncrementalLoad() throws Exception {
+    LOG.info("\nStarting test testMRIncrementalLoad\n");
     doIncrementalLoadTest(false);
   }
 
   @Test
   public void testMRIncrementalLoadWithSplit() throws Exception {
+    LOG.info("\nStarting test testMRIncrementalLoadWithSplit\n");
     doIncrementalLoadTest(true);
   }
 
   private void doIncrementalLoadTest(
       boolean shouldChangeRegions) throws Exception {
+    util = new HBaseTestingUtility();
     Configuration conf = util.getConfiguration();
-    Path testDir = HBaseTestingUtility.getTestDir("testLocalMRIncrementalLoad");
+    Path testDir = util.getTestDir("testLocalMRIncrementalLoad");
     byte[][] startKeys = generateRandomStartKeys(5);
 
     try {
@@ -566,9 +561,7 @@ public class TestHFileOutputFormat  {
           expectedRows, util.countRows(table));
       Scan scan = new Scan();
       ResultScanner results = table.getScanner(scan);
-      int count = 0;
       for (Result res : results) {
-        count++;
         assertEquals(FAMILIES.length, res.raw().length);
         KeyValue first = res.raw()[0];
         for (KeyValue kv : res.raw()) {
@@ -692,7 +685,7 @@ public class TestHFileOutputFormat  {
     RecordWriter<ImmutableBytesWritable, KeyValue> writer = null;
     TaskAttemptContext context = null;
     Path dir =
-        HBaseTestingUtility.getTestDir("testColumnFamilyCompression");
+        util.getTestDir("testColumnFamilyCompression");
 
     HTable table = Mockito.mock(HTable.class);
 
@@ -812,7 +805,7 @@ public class TestHFileOutputFormat  {
     RecordWriter<ImmutableBytesWritable, KeyValue> writer = null;
     TaskAttemptContext context = null;
     Path dir =
-        HBaseTestingUtility.getTestDir("testColumnFamilyBloomFilter");
+        util.getTestDir("testColumnFamilyBloomFilter");
 
     HTable table = Mockito.mock(HTable.class);
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java Fri Apr  6 22:13:16 2012
@@ -90,7 +90,7 @@ public class TestLoadIncrementalHFiles {
 
   private void runTest(String testName, byte[][][] hfileRanges)
   throws Exception {
-    Path dir = HBaseTestingUtility.getTestDir(testName);
+    Path dir = util.getTestDir(testName);
     FileSystem fs = util.getTestFileSystem();
     dir = dir.makeQualified(fs);
     Path familyDir = new Path(dir, Bytes.toString(FAMILY));
@@ -126,7 +126,7 @@ public class TestLoadIncrementalHFiles {
 
   @Test
   public void testSplitStoreFile() throws IOException {
-    Path dir = HBaseTestingUtility.getTestDir("testSplitHFile");
+    Path dir = util.getTestDir("testSplitHFile");
     FileSystem fs = util.getTestFileSystem();
     Path testIn = new Path(dir, "testhfile");
     HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java Fri Apr  6 22:13:16 2012
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -118,7 +119,9 @@ public class TestTableMapReduce extends 
 
   private void runTestOnTable(HTable table)
   throws IOException, InterruptedException, ClassNotFoundException {
-    MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1);
+    HBaseTestingUtility testUtil = new HBaseTestingUtility();
+    testUtil.setFileSystemURI(fs.getUri().toString());
+    MiniMRCluster mrCluster = testUtil.startMiniMapReduceCluster();
     LOG.debug("ZK client port before runing MR job: " +
         ZooKeeperWrapper.getZKClientPort(conf));
 
@@ -147,10 +150,7 @@ public class TestTableMapReduce extends 
       verify(Bytes.toString(table.getTableName()));
     } finally {
       mrCluster.shutdown();
-      if (job != null) {
-        FileUtil.fullyDelete(
-          new File(job.getConfiguration().get("hadoop.tmp.dir")));
-      }
+      testUtil.cleanupTestDir();
     }
   }
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java Fri Apr  6 22:13:16 2012
@@ -31,6 +31,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.master.SplitLogManager.Task;
 import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
@@ -70,6 +71,12 @@ public class TestSplitLogManager {
 
   private volatile boolean stopped = false;
 
+  /**
+   * Additional amount of time we wait for events to happen. Added where unit
+   * test failures have been observed.
+   */
+  private static final int EXTRA_TOLERANCE_MS = 200;
+
   private Stoppable stopper = new Stoppable() {
     @Override
     public boolean isStopped() {
@@ -100,7 +107,9 @@ public class TestSplitLogManager {
   @Before
   public void setup() throws Exception {
     conf = TEST_UTIL.getConfiguration();
-    zkw = ZooKeeperWrapper.createInstance(conf, "split-log-manager-tests");
+    // Use a different ZK wrapper instance for each tests.
+    zkw = ZooKeeperWrapper.createInstance(conf, "split-log-manager-tests-" +
+        UUID.randomUUID().toString());
     zkw.deleteChildrenRecursively(zkw.parentZNode);
     zkw.createZNodeIfNotExists(zkw.parentZNode);
     assertTrue(zkw.checkExists(zkw.parentZNode) != -1);
@@ -217,7 +226,7 @@ public class TestSplitLogManager {
     assertTrue((task.last_update <= curt) &&
         (task.last_update > (curt - 1000)));
     LOG.info("waiting for manager to resubmit the orphan task");
-    waitForCounter(tot_mgr_resubmit, 0, 1, to + 100);
+    waitForCounter(tot_mgr_resubmit, 0, 1, to + 300);
     assertTrue(task.isUnassigned());
     waitForCounter(tot_mgr_rescan, 0, 1, to + 100);
   }
@@ -271,18 +280,18 @@ public class TestSplitLogManager {
 
     zkw.setData(tasknode, TaskState.TASK_OWNED.get("worker1"));
     waitForCounter(tot_mgr_heartbeat, 0, 1, 1000);
-    waitForCounter(tot_mgr_resubmit, 0, 1, to + 100);
+    waitForCounter(tot_mgr_resubmit, 0, 1, to + EXTRA_TOLERANCE_MS);
     int version1 = zkw.checkExists(tasknode);
     assertTrue(version1 > version);
     zkw.setData(tasknode, TaskState.TASK_OWNED.get("worker2"));
     waitForCounter(tot_mgr_heartbeat, 1, 2, 1000);
-    waitForCounter(tot_mgr_resubmit, 1, 2, to + 100);
+    waitForCounter(tot_mgr_resubmit, 1, 2, to + EXTRA_TOLERANCE_MS);
     int version2 = zkw.checkExists(tasknode);
     assertTrue(version2 > version1);
     zkw.setData(tasknode, TaskState.TASK_OWNED.get("worker3"));
     waitForCounter(tot_mgr_heartbeat, 1, 2, 1000);
-    waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + 100);
-    Thread.sleep(to + 100);
+    waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + EXTRA_TOLERANCE_MS);
+    Thread.sleep(to + EXTRA_TOLERANCE_MS);
     assertEquals(2L, tot_mgr_resubmit.get());
   }
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java Fri Apr  6 22:13:16 2012
@@ -43,7 +43,9 @@ public class TestBlocksRead extends HBas
   }
 
   HRegion region = null;
-  private final String DIR = HBaseTestingUtility.getTestDir() + "/TestHRegion/";
+
+  private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
+  private final String DIR = testUtil.getTestDir() + "/TestHRegion/";
 
   /**
    * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java Fri Apr  6 22:13:16 2012
@@ -65,7 +65,7 @@ public class TestColumnSeeking {
     htd.addFamily(hcd);
     HRegionInfo info = new HRegionInfo(htd, null, null, false);
     HRegion region =
-        HRegion.createHRegion(info, HBaseTestingUtility.getTestDir(), TEST_UTIL
+        HRegion.createHRegion(info, TEST_UTIL.getTestDir(), TEST_UTIL
             .getConfiguration());
 
     List<String> rows = generateRandomWords(10, "row");
@@ -169,7 +169,7 @@ public class TestColumnSeeking {
     htd.addFamily(new HColumnDescriptor(family));
     HRegionInfo info = new HRegionInfo(htd, null, null, false);
     HRegion region =
-        HRegion.createHRegion(info, HBaseTestingUtility.getTestDir(), TEST_UTIL
+        HRegion.createHRegion(info, TEST_UTIL.getTestDir(), TEST_UTIL
             .getConfiguration());
 
     List<String> rows = generateRandomWords(10, "row");

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java Fri Apr  6 22:13:16 2012
@@ -52,7 +52,7 @@ public class TestCompactSelection extend
   private Configuration conf;
   private Store store;
   private static final String DIR
-    = HBaseTestingUtility.getTestDir() + "/TestCompactSelection/";
+    = TEST_UTIL.getTestDir() + "/TestCompactSelection/";
   private static Path TEST_FILE;
 
   private static final int minFiles = 3;

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java Fri Apr  6 22:13:16 2012
@@ -295,7 +295,7 @@ public class TestCompoundBloomFilter {
 
     StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConf, fs,
         BLOCK_SIZES[t])
-            .withOutputDir(HBaseTestingUtility.getTestDir())
+            .withOutputDir(TEST_UTIL.getTestDir())
             .withBloomType(bt)
             .build();
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java Fri Apr  6 22:13:16 2012
@@ -67,7 +67,7 @@ public class TestFSErrorsExposed {
   @Test
   public void testHFileScannerThrowsErrors() throws IOException {
     Path hfilePath = new Path(new Path(
-        HBaseTestingUtility.getTestDir("internalScannerExposesErrors"),
+        util.getTestDir("internalScannerExposesErrors"),
         "regionname"), "familyname");
     FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem());
     CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
@@ -114,7 +114,7 @@ public class TestFSErrorsExposed {
   @Test
   public void testStoreFileScannerThrowsErrors() throws IOException {
     Path hfilePath = new Path(new Path(
-        HBaseTestingUtility.getTestDir("internalScannerExposesErrors"),
+        util.getTestDir("internalScannerExposesErrors"),
         "regionname"), "familyname");
     FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem());
     CacheConfig cacheConf = new CacheConfig(util.getConfiguration());

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Fri Apr  6 22:13:16 2012
@@ -85,7 +85,8 @@ public class TestHRegion extends HBaseTe
   private static final String COLUMN_FAMILY = "MyCF";
 
   HRegion region = null;
-  private final String DIR = HBaseTestingUtility.getTestDir() +
+  private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final String DIR = TEST_UTIL.getTestDir() +
     "/TestHRegion/";
 
   private final int MAX_VERSIONS = 2;

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java Fri Apr  6 22:13:16 2012
@@ -354,7 +354,7 @@ public class TestMultiColumnScanner {
     htd.addFamily(hcd);
     HRegionInfo info = new HRegionInfo(htd, null, null, false);
     HRegion region = HRegion.createHRegion(
-        info, HBaseTestingUtility.getTestDir(), TEST_UTIL.getConfiguration());
+        info, TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
     return region;
   }
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java Fri Apr  6 22:13:16 2012
@@ -34,7 +34,8 @@ public class TestScannerResets extends H
   }
 
   HRegion region = null;
-  private final String DIR = HBaseTestingUtility.getTestDir() + "/TestScannerResets/";
+  private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final String DIR = TEST_UTIL.getTestDir() + "/TestScannerResets/";
 
   /**
    * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java Fri Apr  6 22:13:16 2012
@@ -89,7 +89,8 @@ public class TestStore extends TestCase 
   long id = System.currentTimeMillis();
   Get get = new Get(row);
 
-  private static final String DIR = HBaseTestingUtility.getTestDir() + "/TestStore/";
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final String DIR = TEST_UTIL.getTestDir() + "/TestStore/";
 
   private static final int MAX_VERSION = 4;
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Fri Apr  6 22:13:16 2012
@@ -68,6 +68,7 @@ public class TestStoreFile extends HBase
   private MiniDFSCluster cluster;
   private CacheConfig cacheConf;
   private Map<String, Long> startingMetrics;
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
   @Override
   public void setUp() throws Exception {
@@ -352,7 +353,7 @@ public class TestStoreFile extends HBase
   }
 
   private static String ROOT_DIR =
-    HBaseTestingUtility.getTestDir("TestStoreFile").toString();
+    TEST_UTIL.getTestDir("TestStoreFile").toString();
   private static String localFormatter = "%010d";
 
   private void bloomWriteRead(StoreFile.Writer writer, FileSystem fs)

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java Fri Apr  6 22:13:16 2012
@@ -338,10 +338,9 @@ public class TestHLog  {
     }
     // Now call sync to send the data to HDFS datanodes
     wal.sync(true);
-     int namenodePort = cluster.getNameNodePort();
+    int namenodePort = cluster.getNameNodePort();
     final Path walPath = wal.computeFilename();
 
-
     // Stop the cluster.  (ensure restart since we're sharing MiniDFSCluster)
     try {
       cluster.getNameNode().setSafeMode(SafeModeAction.SAFEMODE_ENTER);
@@ -361,10 +360,12 @@ public class TestHLog  {
         LOG.error("Waiting for cluster to go down");
         Thread.sleep(1000);
       }
-      cluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null);
+      LOG.info("Waiting a few seconds before re-starting HDFS");
+      Thread.sleep(5000);
+      cluster = TEST_UTIL.startMiniDFSClusterForTestHLog(namenodePort);
       cluster.waitActive();
       fs = cluster.getFileSystem();
-      LOG.info("START second instance.");
+      LOG.info("STARTED second instance.");
     }
 
     // set the lease period to be 1 second so that the

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java Fri Apr  6 22:13:16 2012
@@ -43,7 +43,7 @@ public class TestHLogMethods {
    */
   @Test public void testGetSplitEditFilesSorted() throws IOException {
     FileSystem fs = FileSystem.get(util.getConfiguration());
-    Path regiondir = HBaseTestingUtility.getTestDir("regiondir");
+    Path regiondir = util.getTestDir("regiondir");
     fs.delete(regiondir, true);
     fs.mkdirs(regiondir);
     Path recoverededits = HLog.getRegionDirRecoveredEditsDir(regiondir);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java Fri Apr  6 22:13:16 2012
@@ -112,9 +112,9 @@ public class TestReplicationSourceManage
         server.getZooKeeperWrapper(), conf,
         REPLICATING, "123456789");
     fs = FileSystem.get(conf);
-    oldLogDir = new Path(HBaseTestingUtility.getTestDir(),
+    oldLogDir = new Path(utility.getTestDir(),
         HConstants.HREGION_OLDLOGDIR_NAME);
-    logDir = new Path(HBaseTestingUtility.getTestDir(),
+    logDir = new Path(utility.getTestDir(),
         HConstants.HREGION_LOGDIR_NAME);
 
     manager = new ReplicationSourceManager(helper,

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java Fri Apr  6 22:13:16 2012
@@ -75,6 +75,9 @@ public class TestMiniClusterLoadSequenti
     this.isMultiPut = isMultiPut;
     this.dataBlockEncoding = dataBlockEncoding;
     conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
+
+    // We don't want any region reassignments by the load balancer during the test.
+    conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, 10.0f);
   }
 
   @Parameters

Modified: hbase/branches/0.89-fb/src/test/resources/hbase-site.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/resources/hbase-site.xml?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/resources/hbase-site.xml (original)
+++ hbase/branches/0.89-fb/src/test/resources/hbase-site.xml Fri Apr  6 22:13:16 2012
@@ -142,7 +142,7 @@
 
   <property>
     <name>zookeeper.session.timeout</name>
-    <value>1000</value>
+    <value>60000</value>
   </property>
 
   <property>
@@ -150,4 +150,11 @@
     <value>false</value>
   </property>
 
+  <property>
+    <name>hbase.client.retries.number</name>
+    <value>100</value>
+    <description>
+    Use a lot of retries in unit tests.
+    </description>
+  </property>
 </configuration>



Mime
View raw message