Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9B6B910D68 for ; Wed, 17 Dec 2014 05:20:46 +0000 (UTC) Received: (qmail 46993 invoked by uid 500); 17 Dec 2014 05:20:46 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 46947 invoked by uid 500); 17 Dec 2014 05:20:46 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 46938 invoked by uid 99); 17 Dec 2014 05:20:46 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 17 Dec 2014 05:20:46 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 32D1BA2EAA3; Wed, 17 Dec 2014 05:20:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: stack@apache.org To: commits@hbase.apache.org Message-Id: <68a6440afc184e0e8832907c94238c08@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-12645 HBaseTestingUtility is using ${$HOME} for rootDir (Varun Saxena) Date: Wed, 17 Dec 2014 05:20:46 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/branch-1 063570535 -> e82f048d7 HBASE-12645 HBaseTestingUtility is using ${$HOME} for rootDir (Varun Saxena) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e82f048d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e82f048d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e82f048d Branch: refs/heads/branch-1 Commit: e82f048d7b17b871f813bf6f4aa7ecda226d1b19 Parents: 0635705 Author: stack Authored: Tue Dec 16 16:41:27 2014 -0800 Committer: stack Committed: Tue Dec 16 21:20:32 2014 -0800 ---------------------------------------------------------------------- .../hadoop/hbase/HBaseTestingUtility.java | 122 ++++++++++++++++--- .../hbase/client/TestTableSnapshotScanner.java | 2 +- .../TableSnapshotInputFormatTestBase.java | 2 +- .../replication/TestReplicationSource.java | 8 +- .../apache/hadoop/hbase/util/TestMergeTool.java | 5 +- 5 files changed, 111 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index bf61208..5a72965 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -444,7 +444,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } /** - * Sets up a path in test filesystem to be used by tests + * Sets up a path in test filesystem to be used by tests. + * Creates a new directory if not already setup. */ private void setupDataTestDirOnTestFS() throws IOException { if (dataTestDirOnTestFS != null) { @@ -452,22 +453,30 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { + dataTestDirOnTestFS.toString()); return; } + dataTestDirOnTestFS = getNewDataTestDirOnTestFS(); + } + /** + * Sets up a new path in test filesystem to be used by tests. + */ + private Path getNewDataTestDirOnTestFS() throws IOException { //The file system can be either local, mini dfs, or if the configuration //is supplied externally, it can be an external cluster FS. If it is a local //file system, the tests should use getBaseTestDir, otherwise, we can use //the working directory, and create a unique sub dir there FileSystem fs = getTestFileSystem(); + Path newDataTestDir = null; if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) { File dataTestDir = new File(getDataTestDir().toString()); if (deleteOnExit()) dataTestDir.deleteOnExit(); - dataTestDirOnTestFS = new Path(dataTestDir.getAbsolutePath()); + newDataTestDir = new Path(dataTestDir.getAbsolutePath()); } else { Path base = getBaseTestDirOnTestFS(); String randomStr = UUID.randomUUID().toString(); - dataTestDirOnTestFS = new Path(base, randomStr); - if (deleteOnExit()) fs.deleteOnExit(dataTestDirOnTestFS); + newDataTestDir = new Path(base, randomStr); + if (deleteOnExit()) fs.deleteOnExit(newDataTestDir); } + return newDataTestDir; } /** @@ -742,6 +751,19 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } /** + * Start up a minicluster of hbase, dfs, and zookeeper. + * Set the create flag to create root or data directory path or not + * (will overwrite if dir already exists) + * @throws Exception + * @return Mini hbase cluster instance created. + * @see {@link #shutdownMiniDFSCluster()} + */ + public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create) + throws Exception { + return startMiniCluster(1, numSlaves, create); + } + + /** * Start up a minicluster of hbase, optionally dfs, and zookeeper. * Modifies Configuration. Homes the cluster data directory under a random * subdirectory in a directory under System property test.build.data. @@ -756,9 +778,21 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { */ public MiniHBaseCluster startMiniCluster(final int numSlaves) throws Exception { - return startMiniCluster(1, numSlaves); + return startMiniCluster(1, numSlaves, false); } + /** + * Start minicluster. Whether to create a new root or data dir path even if such a path + * has been created earlier is decided based on flag create + * @throws Exception + * @see {@link #shutdownMiniCluster()} + * @return Mini hbase cluster instance created. + */ + public MiniHBaseCluster startMiniCluster(final int numMasters, + final int numSlaves, boolean create) + throws Exception { + return startMiniCluster(numMasters, numSlaves, null, create); + } /** * start minicluster @@ -769,7 +803,14 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { public MiniHBaseCluster startMiniCluster(final int numMasters, final int numSlaves) throws Exception { - return startMiniCluster(numMasters, numSlaves, null); + return startMiniCluster(numMasters, numSlaves, null, false); + } + + public MiniHBaseCluster startMiniCluster(final int numMasters, + final int numSlaves, final String[] dataNodeHosts, boolean create) + throws Exception { + return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, + null, null, create); } /** @@ -798,7 +839,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { */ public MiniHBaseCluster startMiniCluster(final int numMasters, final int numSlaves, final String[] dataNodeHosts) throws Exception { - return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, null, null); + return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, + null, null); } /** @@ -845,15 +887,27 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { numMasters, numSlaves, numSlaves, dataNodeHosts, masterClass, regionserverClass); } + public MiniHBaseCluster startMiniCluster(final int numMasters, + final int numSlaves, int numDataNodes, final String[] dataNodeHosts, + Class masterClass, + Class regionserverClass) + throws Exception { + return startMiniCluster(numMasters, numSlaves, numDataNodes, dataNodeHosts, + masterClass, regionserverClass, false); + } + /** * Same as {@link #startMiniCluster(int, int, String[], Class, Class)}, but with custom * number of datanodes. * @param numDataNodes Number of data nodes. + * @param create Set this flag to create a new + * root or data directory path or not (will overwrite if exists already). */ public MiniHBaseCluster startMiniCluster(final int numMasters, final int numSlaves, int numDataNodes, final String[] dataNodeHosts, Class masterClass, - Class regionserverClass) + Class regionserverClass, + boolean create) throws Exception { if (dataNodeHosts != null && dataNodeHosts.length != 0) { numDataNodes = dataNodeHosts.length; @@ -881,12 +935,13 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } // Start the MiniHBaseCluster - return startMiniHBaseCluster(numMasters, numSlaves, masterClass, regionserverClass); + return startMiniHBaseCluster(numMasters, numSlaves, masterClass, + regionserverClass, create); } public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves) throws IOException, InterruptedException{ - return startMiniHBaseCluster(numMasters, numSlaves, null, null); + return startMiniHBaseCluster(numMasters, numSlaves, null, null, false); } /** @@ -895,6 +950,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. * @param numMasters * @param numSlaves + * @param create Whether to create a + * root or data directory path or not; will overwrite if exists already. * @return Reference to the hbase mini hbase cluster. * @throws IOException * @throws InterruptedException @@ -902,10 +959,11 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { */ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves, Class masterClass, - Class regionserverClass) + Class regionserverClass, + boolean create) throws IOException, InterruptedException { // Now do the mini hbase cluster. Set the hbase.rootdir in config. - createRootDir(); + createRootDir(create); // These settings will make the server waits until this exact number of // regions servers are connected. @@ -1028,14 +1086,30 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } /** - * Returns the path to the default root dir the minicluster uses. + * Returns the path to the default root dir the minicluster uses. If getNewDirPathIfExists + * is true, a new root directory path is fetched irrespective of whether it has been fetched + * before or not. If false, previous path is used. + * Note: this does not cause the root dir to be created. + * @return Fully qualified path for the default hbase root dir + * @throws IOException + */ + public Path getDefaultRootDirPath(boolean create) throws IOException { + if (!create) { + return getDataTestDirOnTestFS(); + } else { + return getNewDataTestDirOnTestFS(); + } + } + + /** + * Same as {{@link HBaseTestingUtility#getDefaultRootDirPath(boolean getNewDirPathIfExists)} + * except that getNewDirPathIfExists flag is false. * Note: this does not cause the root dir to be created. * @return Fully qualified path for the default hbase root dir * @throws IOException */ public Path getDefaultRootDirPath() throws IOException { - FileSystem fs = FileSystem.get(this.conf); - return new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase"); + return getDefaultRootDirPath(false); } /** @@ -1043,12 +1117,16 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { * version file. Normally you won't make use of this method. Root hbasedir * is created for you as part of mini cluster startup. You'd only use this * method if you were doing manual operation. + * @param getNewDirPathIfExists This flag decides whether to get a new + * root or data directory path or not, if it has been fetched already. + * Note : Directory will be made irrespective of whether path has been fetched or not. + * If directory already exists, it will be overwritten * @return Fully qualified path to hbase root dir * @throws IOException */ - public Path createRootDir() throws IOException { + public Path createRootDir(boolean getNewDirPathIfExists) throws IOException { FileSystem fs = FileSystem.get(this.conf); - Path hbaseRootdir = getDefaultRootDirPath(); + Path hbaseRootdir = getDefaultRootDirPath(getNewDirPathIfExists); FSUtils.setRootDir(this.conf, hbaseRootdir); fs.mkdirs(hbaseRootdir); FSUtils.setVersion(fs, hbaseRootdir); @@ -1056,6 +1134,16 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } /** + * Same as {@link HBaseTestingUtility#createRootDir(boolean getNewDirPathIfExists)} + * except that getNewDirPathIfExists flag is false. + * @return Fully qualified path to hbase root dir + * @throws IOException + */ + public Path createRootDir() throws IOException { + return createRootDir(false); + } + + /** * Flushes all caches in the mini hbase cluster * @throws IOException */ http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java index 10f7b9f..3e915e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java @@ -56,7 +56,7 @@ public class TestTableSnapshotScanner { public void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); - UTIL.startMiniCluster(NUM_REGION_SERVERS); + UTIL.startMiniCluster(NUM_REGION_SERVERS, true); rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); fs = rootDir.getFileSystem(UTIL.getConfiguration()); } http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java index 7c55ad0..1f8ccae 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java @@ -50,7 +50,7 @@ public abstract class TableSnapshotInputFormatTestBase { public void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); - UTIL.startMiniCluster(NUM_REGION_SERVERS); + UTIL.startMiniCluster(NUM_REGION_SERVERS, true); rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); fs = rootDir.getFileSystem(UTIL.getConfiguration()); } http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java index 1fb0312..458819d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java @@ -62,12 +62,10 @@ public class TestReplicationSource { public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniDFSCluster(1); FS = TEST_UTIL.getDFSCluster().getFileSystem(); - FSUtils.setRootDir(TEST_UTIL.getConfiguration(), FS.getHomeDirectory()); - oldLogDir = new Path(FS.getHomeDirectory(), - HConstants.HREGION_OLDLOGDIR_NAME); + Path rootDir = TEST_UTIL.createRootDir(); + oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); if (FS.exists(oldLogDir)) FS.delete(oldLogDir, true); - logDir = new Path(FS.getHomeDirectory(), - HConstants.HREGION_LOGDIR_NAME); + logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); if (FS.exists(logDir)) FS.delete(logDir, true); } http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java index ca09b39..d4c7fa8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java @@ -135,10 +135,7 @@ public class TestMergeTool extends HBaseTestCase { this.fs = this.dfsCluster.getFileSystem(); System.out.println("fs=" + this.fs); FSUtils.setFsDefault(this.conf, new Path(fs.getUri())); - Path parentdir = fs.getHomeDirectory(); - FSUtils.setRootDir(conf, parentdir); - fs.mkdirs(parentdir); - FSUtils.setVersion(fs, parentdir); + TEST_UTIL.createRootDir(); // Note: we must call super.setUp after starting the mini cluster or // we will end up with a local file system