hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject hbase git commit: HBASE-13846 Run MiniCluster on top of other MiniDfsCluster
Date Wed, 10 Jun 2015 20:07:15 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.0 6979c6f56 -> 670801101


HBASE-13846 Run MiniCluster on top of other MiniDfsCluster


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/67080110
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/67080110
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/67080110

Branch: refs/heads/branch-1.0
Commit: 670801101c7338bedff94ffa7edaf859af9cb1b9
Parents: 6979c6f
Author: Jesse Yates <jyates@apache.org>
Authored: Thu Jun 4 11:46:38 2015 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Wed Jun 10 12:58:29 2015 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/HBaseTestingUtility.java       | 35 ++++++--
 .../hbase/TestHBaseOnOtherDfsCluster.java       | 85 ++++++++++++++++++++
 2 files changed, 114 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/67080110/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 1e9f782..32561d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -567,8 +567,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       true, null, null, hosts, null);
 
     // Set this just-started cluster as our filesystem.
-    FileSystem fs = this.dfsCluster.getFileSystem();
-    FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
+    setFs();
 
     // Wait for the cluster to be totally up
     this.dfsCluster.waitClusterUp();
@@ -579,6 +578,14 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
     return this.dfsCluster;
   }
 
+  private void setFs() throws IOException {
+    if(this.dfsCluster == null){
+      LOG.info("Skipping setting fs because dfsCluster is null");
+      return;
+    }
+    FileSystem fs = this.dfsCluster.getFileSystem();
+    FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
+  }
 
   public MiniDFSCluster startMiniDFSCluster(int servers, final  String racks[], String hosts[])
       throws Exception {
@@ -937,7 +944,9 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
 
     // Bring up mini dfs cluster. This spews a bunch of warnings about missing
     // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'.
-    startMiniDFSCluster(numDataNodes, dataNodeHosts);
+    if(this.dfsCluster == null) {
+      dfsCluster = startMiniDFSCluster(numDataNodes, dataNodeHosts);
+    }
 
     // Start up a zk cluster.
     if (this.zkCluster == null) {
@@ -2829,11 +2838,25 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility
{
     return dfsCluster;
   }
 
-  public void setDFSCluster(MiniDFSCluster cluster) throws IOException {
-    if (dfsCluster != null && dfsCluster.isClusterUp()) {
-      throw new IOException("DFSCluster is already running! Shut it down first.");
+  public void setDFSCluster(MiniDFSCluster cluster) throws IllegalStateException, IOException
{
+    setDFSCluster(cluster, true);
+  }
+
+  /**
+   * Set the MiniDFSCluster
+   * @param cluster cluster to use
+   * @param requireDown require the that cluster not be "up" (MiniDFSCluster#isClusterUp)
before
+   * it is set.
+   * @throws IllegalStateException if the passed cluster is up when it is required to be
down
+   * @throws IOException if the FileSystem could not be set from the passed dfs cluster
+   */
+  public void setDFSCluster(MiniDFSCluster cluster, boolean requireDown)
+      throws IllegalStateException, IOException {
+    if (dfsCluster != null && requireDown && dfsCluster.isClusterUp()) {
+      throw new IllegalStateException("DFSCluster is already running! Shut it down first.");
     }
     this.dfsCluster = cluster;
+    this.setFs();
   }
 
   public FileSystem getTestFileSystem() throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/67080110/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseOnOtherDfsCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseOnOtherDfsCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseOnOtherDfsCluster.java
new file mode 100644
index 0000000..7135a24
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseOnOtherDfsCluster.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test that an HBase cluster can run on top of an existing MiniDfsCluster
+ */
+@Category(MediumTests.class)
+public class TestHBaseOnOtherDfsCluster {
+
+  @Test
+  public void testOveralyOnOtherCluster() throws Exception {
+    // just run HDFS
+    HBaseTestingUtility util1 = new HBaseTestingUtility();
+    MiniDFSCluster dfs = util1.startMiniDFSCluster(1);
+
+    // run HBase on that HDFS
+    HBaseTestingUtility util2 = new HBaseTestingUtility();
+    // set the dfs
+    util2.setDFSCluster(dfs, false);
+    util2.startMiniCluster();
+
+    //ensure that they are pointed at the same place
+    FileSystem fs = dfs.getFileSystem();
+    FileSystem targetFs = util2.getDFSCluster().getFileSystem();
+    assertFsSameUri(fs, targetFs);
+
+    fs = FileSystem.get(util1.getConfiguration());
+    targetFs = FileSystem.get(util2.getConfiguration());
+    assertFsSameUri(fs, targetFs);
+
+    Path randomFile = new Path("/"+UUID.randomUUID());
+    assertTrue(targetFs.createNewFile(randomFile));
+    assertTrue(fs.exists(randomFile));
+
+    // do a simple create/write to ensure the cluster works as expected
+    byte[] family = Bytes.toBytes("testfamily");
+    byte[] tablename = Bytes.toBytes("testtable");
+    HTable table = util2.createTable(tablename, family);
+    Put p = new Put(new byte[] { 1, 2, 3 });
+    p.add(family, null, new byte[] { 1 });
+    table.put(p);
+    table.flushCommits();
+
+    // shutdown and make sure cleanly shutting down
+    util2.shutdownMiniCluster();
+    util1.shutdownMiniDFSCluster();
+  }
+
+  private void assertFsSameUri(FileSystem sourceFs, FileSystem targetFs) {
+    Path source = new Path(sourceFs.getUri());
+    Path target = new Path(targetFs.getUri());
+    assertEquals(source, target);
+  }
+}
\ No newline at end of file


Mime
View raw message