hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject [1/2] hbase git commit: HBASE-12645 HBaseTestingUtility is using ${$HOME} for rootDir (Varun Saxena)
Date Wed, 17 Dec 2014 00:58:56 GMT
Repository: hbase
Updated Branches:
  refs/heads/master d845a92fc -> 5b5c981d9


HBASE-12645 HBaseTestingUtility is using ${$HOME} for rootDir (Varun Saxena)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/89006f76
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/89006f76
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/89006f76

Branch: refs/heads/master
Commit: 89006f76f7f8b206c3575c146f76a3509a398190
Parents: d845a92
Author: stack <stack@apache.org>
Authored: Tue Dec 16 16:41:27 2014 -0800
Committer: stack <stack@apache.org>
Committed: Tue Dec 16 16:41:27 2014 -0800

----------------------------------------------------------------------
 .../hadoop/hbase/HBaseTestingUtility.java       | 122 ++++++++++++++++---
 .../hbase/client/TestTableSnapshotScanner.java  |   2 +-
 .../TableSnapshotInputFormatTestBase.java       |   2 +-
 .../replication/TestReplicationSource.java      |   8 +-
 .../apache/hadoop/hbase/util/TestMergeTool.java |   5 +-
 5 files changed, 111 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/89006f76/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 4619ef2..c5574e0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -442,7 +442,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
   }
 
   /**
-   * Sets up a path in test filesystem to be used by tests
+   * Sets up a path in test filesystem to be used by tests.
+   * Creates a new directory if not already setup.
    */
   private void setupDataTestDirOnTestFS() throws IOException {
     if (dataTestDirOnTestFS != null) {
@@ -450,22 +451,30 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
           + dataTestDirOnTestFS.toString());
       return;
     }
+    dataTestDirOnTestFS = getNewDataTestDirOnTestFS();
+  }
 
+  /**
+   * Sets up a new path in test filesystem to be used by tests.
+   */
+  private Path getNewDataTestDirOnTestFS() throws IOException {
     //The file system can be either local, mini dfs, or if the configuration
     //is supplied externally, it can be an external cluster FS. If it is a local
     //file system, the tests should use getBaseTestDir, otherwise, we can use
     //the working directory, and create a unique sub dir there
     FileSystem fs = getTestFileSystem();
+    Path newDataTestDir = null;
     if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) {
       File dataTestDir = new File(getDataTestDir().toString());
       if (deleteOnExit()) dataTestDir.deleteOnExit();
-      dataTestDirOnTestFS = new Path(dataTestDir.getAbsolutePath());
+      newDataTestDir = new Path(dataTestDir.getAbsolutePath());
     } else {
       Path base = getBaseTestDirOnTestFS();
       String randomStr = UUID.randomUUID().toString();
-      dataTestDirOnTestFS = new Path(base, randomStr);
-      if (deleteOnExit()) fs.deleteOnExit(dataTestDirOnTestFS);
+      newDataTestDir = new Path(base, randomStr);
+      if (deleteOnExit()) fs.deleteOnExit(newDataTestDir);
     }
+    return newDataTestDir;
   }
 
   /**
@@ -740,6 +749,19 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
   }
 
   /**
+   * Start up a minicluster of hbase, dfs, and zookeeper.
+   * Set the <code>create</code> flag to create root or data directory path or
not
+   * (will overwrite if dir already exists)
+   * @throws Exception
+   * @return Mini hbase cluster instance created.
+   * @see {@link #shutdownMiniDFSCluster()}
+   */
+  public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create)
+  throws Exception {
+    return startMiniCluster(1, numSlaves, create);
+  }
+
+  /**
    * Start up a minicluster of hbase, optionally dfs, and zookeeper.
    * Modifies Configuration.  Homes the cluster data directory under a random
    * subdirectory in a directory under System property test.build.data.
@@ -754,9 +776,21 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
    */
   public MiniHBaseCluster startMiniCluster(final int numSlaves)
   throws Exception {
-    return startMiniCluster(1, numSlaves);
+    return startMiniCluster(1, numSlaves, false);
   }
 
+  /**
+   * Start minicluster. Whether to create a new root or data dir path even if such a path
+   * has been created earlier is decided based on flag <code>create</code>
+   * @throws Exception
+   * @see {@link #shutdownMiniCluster()}
+   * @return Mini hbase cluster instance created.
+   */
+  public MiniHBaseCluster startMiniCluster(final int numMasters,
+      final int numSlaves, boolean create)
+    throws Exception {
+      return startMiniCluster(numMasters, numSlaves, null, create);
+  }
 
   /**
    * start minicluster
@@ -767,7 +801,14 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
   public MiniHBaseCluster startMiniCluster(final int numMasters,
     final int numSlaves)
   throws Exception {
-    return startMiniCluster(numMasters, numSlaves, null);
+    return startMiniCluster(numMasters, numSlaves, null, false);
+  }
+
+  public MiniHBaseCluster startMiniCluster(final int numMasters,
+      final int numSlaves, final String[] dataNodeHosts, boolean create)
+      throws Exception {
+    return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
+        null, null, create);
   }
 
   /**
@@ -796,7 +837,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
    */
   public MiniHBaseCluster startMiniCluster(final int numMasters,
       final int numSlaves, final String[] dataNodeHosts) throws Exception {
-    return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, null, null);
+    return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
+        null, null);
   }
 
   /**
@@ -843,15 +885,27 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
         numMasters, numSlaves, numSlaves, dataNodeHosts, masterClass, regionserverClass);
   }
 
+  public MiniHBaseCluster startMiniCluster(final int numMasters,
+      final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
+      Class<? extends HMaster> masterClass,
+      Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
+    throws Exception {
+    return startMiniCluster(numMasters, numSlaves, numDataNodes, dataNodeHosts,
+        masterClass, regionserverClass, false);
+  }
+
   /**
    * Same as {@link #startMiniCluster(int, int, String[], Class, Class)}, but with custom
    * number of datanodes.
    * @param numDataNodes Number of data nodes.
+   * @param create Set this flag to create a new
+   * root or data directory path or not (will overwrite if exists already).
    */
   public MiniHBaseCluster startMiniCluster(final int numMasters,
     final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
     Class<? extends HMaster> masterClass,
-    Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
+    Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
+    boolean create)
   throws Exception {
     if (dataNodeHosts != null && dataNodeHosts.length != 0) {
       numDataNodes = dataNodeHosts.length;
@@ -879,12 +933,13 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
     }
 
     // Start the MiniHBaseCluster
-    return startMiniHBaseCluster(numMasters, numSlaves, masterClass, regionserverClass);
+    return startMiniHBaseCluster(numMasters, numSlaves, masterClass,
+      regionserverClass, create);
   }
 
   public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves)
       throws IOException, InterruptedException{
-    return startMiniHBaseCluster(numMasters, numSlaves, null, null);
+    return startMiniHBaseCluster(numMasters, numSlaves, null, null, false);
   }
 
   /**
@@ -893,6 +948,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
    * Usually you won't want this.  You'll usually want {@link #startMiniCluster()}.
    * @param numMasters
    * @param numSlaves
+   * @param create Whether to create a
+   * root or data directory path or not; will overwrite if exists already.
    * @return Reference to the hbase mini hbase cluster.
    * @throws IOException
    * @throws InterruptedException
@@ -900,10 +957,11 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
    */
   public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
         final int numSlaves, Class<? extends HMaster> masterClass,
-        Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
+        Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
+        boolean create)
   throws IOException, InterruptedException {
     // Now do the mini hbase cluster.  Set the hbase.rootdir in config.
-    createRootDir();
+    createRootDir(create);
 
     // These settings will make the server waits until this exact number of
     // regions servers are connected.
@@ -1026,14 +1084,30 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility
{
   }
 
   /**
-   * Returns the path to the default root dir the minicluster uses.
+   * Returns the path to the default root dir the minicluster uses. If getNewDirPathIfExists
+   * is true, a new root directory path is fetched irrespective of whether it has been fetched
+   * before or not. If false, previous path is used.
+   * Note: this does not cause the root dir to be created.
+   * @return Fully qualified path for the default hbase root dir
+   * @throws IOException
+   */
+  public Path getDefaultRootDirPath(boolean create) throws IOException {
+    if (!create) {
+      return getDataTestDirOnTestFS();
+    } else {
+      return getNewDataTestDirOnTestFS();
+    }
+  }
+
+  /**
+   * Same as {{@link HBaseTestingUtility#getDefaultRootDirPath(boolean getNewDirPathIfExists)}
+   * except that getNewDirPathIfExists flag is false.
    * Note: this does not cause the root dir to be created.
    * @return Fully qualified path for the default hbase root dir
    * @throws IOException
    */
   public Path getDefaultRootDirPath() throws IOException {
-    FileSystem fs = FileSystem.get(this.conf);
-    return new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase");
+    return getDefaultRootDirPath(false);
   }
 
   /**
@@ -1041,12 +1115,16 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility
{
    * version file.  Normally you won't make use of this method.  Root hbasedir
    * is created for you as part of mini cluster startup.  You'd only use this
    * method if you were doing manual operation.
+   * @param getNewDirPathIfExists This flag decides whether to get a new
+   * root or data directory path or not, if it has been fetched already.
+   * Note : Directory will be made irrespective of whether path has been fetched or not.
+   * If directory already exists, it will be overwritten
    * @return Fully qualified path to hbase root dir
    * @throws IOException
    */
-  public Path createRootDir() throws IOException {
+  public Path createRootDir(boolean getNewDirPathIfExists) throws IOException {
     FileSystem fs = FileSystem.get(this.conf);
-    Path hbaseRootdir = getDefaultRootDirPath();
+    Path hbaseRootdir = getDefaultRootDirPath(getNewDirPathIfExists);
     FSUtils.setRootDir(this.conf, hbaseRootdir);
     fs.mkdirs(hbaseRootdir);
     FSUtils.setVersion(fs, hbaseRootdir);
@@ -1054,6 +1132,16 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility
{
   }
 
   /**
+   * Same as {@link HBaseTestingUtility#createRootDir(boolean getNewDirPathIfExists)}
+   * except that getNewDirPathIfExists flag is false.
+   * @return Fully qualified path to hbase root dir
+   * @throws IOException
+   */
+  public Path createRootDir() throws IOException {
+    return createRootDir(false);
+  }
+
+  /**
    * Flushes all caches in the mini hbase cluster
    * @throws IOException
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/89006f76/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
index 62211b2..1e3d1cf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
@@ -57,7 +57,7 @@ public class TestTableSnapshotScanner {
 
   public void setupCluster() throws Exception {
     setupConf(UTIL.getConfiguration());
-    UTIL.startMiniCluster(NUM_REGION_SERVERS);
+    UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
     rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
     fs = rootDir.getFileSystem(UTIL.getConfiguration());
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/89006f76/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 7c55ad0..1f8ccae 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -50,7 +50,7 @@ public abstract class TableSnapshotInputFormatTestBase {
 
   public void setupCluster() throws Exception {
     setupConf(UTIL.getConfiguration());
-    UTIL.startMiniCluster(NUM_REGION_SERVERS);
+    UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
     rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
     fs = rootDir.getFileSystem(UTIL.getConfiguration());
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/89006f76/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
index 7700e31..aac966e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
@@ -63,12 +63,10 @@ public class TestReplicationSource {
   public static void setUpBeforeClass() throws Exception {
     TEST_UTIL.startMiniDFSCluster(1);
     FS = TEST_UTIL.getDFSCluster().getFileSystem();
-    FSUtils.setRootDir(TEST_UTIL.getConfiguration(), FS.getHomeDirectory());
-    oldLogDir = new Path(FS.getHomeDirectory(),
-        HConstants.HREGION_OLDLOGDIR_NAME);
+    Path rootDir = TEST_UTIL.createRootDir();
+    oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
     if (FS.exists(oldLogDir)) FS.delete(oldLogDir, true);
-    logDir = new Path(FS.getHomeDirectory(),
-        HConstants.HREGION_LOGDIR_NAME);
+    logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
     if (FS.exists(logDir)) FS.delete(logDir, true);
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/89006f76/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
index e5cd212..0f6d697 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
@@ -139,10 +139,7 @@ public class TestMergeTool extends HBaseTestCase {
     this.fs = this.dfsCluster.getFileSystem();
     System.out.println("fs=" + this.fs);
     FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
-    Path parentdir = fs.getHomeDirectory();
-    FSUtils.setRootDir(conf, parentdir);
-    fs.mkdirs(parentdir);
-    FSUtils.setVersion(fs, parentdir);
+    TEST_UTIL.createRootDir();
 
     // Note: we must call super.setUp after starting the mini cluster or
     // we will end up with a local file system


Mime
View raw message