giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject git commit: GIRAPH-510: Remove HBase Cruft
Date Fri, 22 Mar 2013 02:08:59 GMT
Updated Branches:
  refs/heads/trunk f4bd1996e -> 156682791


GIRAPH-510: Remove HBase Cruft


Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/15668279
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/15668279
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/15668279

Branch: refs/heads/trunk
Commit: 156682791504c2e433382551e1be301e1d9146cb
Parents: f4bd199
Author: Nitay Joffe <nitayj@fb.com>
Authored: Thu Mar 21 22:03:03 2013 -0400
Committer: Nitay Joffe <nitayj@fb.com>
Committed: Thu Mar 21 22:08:45 2013 -0400

----------------------------------------------------------------------
 CHANGELOG                                          |    2 +
 .../io/hbase/TestHBaseRootMarkerVertextFormat.java |   70 +++++++++++----
 2 files changed, 55 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/giraph/blob/15668279/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index a1446a7..db8f4ce 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
 Giraph Change Log
 
 Release 0.2.0 - unreleased
+  GIRAPH-510: Remove HBase Cruft (kelarini via nitay)
+
   GIRAPH-566: Make option for aggregators to be configurable (majakabiljo)
 
   GIRAPH-575: update hive-io (nitay)

http://git-wip-us.apache.org/repos/asf/giraph/blob/15668279/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java
----------------------------------------------------------------------
diff --git a/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java
b/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java
index 4d9588a..7415ddc 100644
--- a/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java
+++ b/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/TestHBaseRootMarkerVertextFormat.java
@@ -29,8 +29,10 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.Get;
@@ -51,6 +53,7 @@ import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.UUID;
 
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -65,7 +68,7 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
      *
      * @param testName name of the test case
      */
-    private HBaseTestingUtility testUtil = new HBaseTestingUtility();
+
     private final Logger log = Logger.getLogger(TestHBaseRootMarkerVertextFormat.class);
 
     private final String TABLE_NAME = "simple_graph";
@@ -73,12 +76,33 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
     private final String QUALIFER = "children";
     private final String OUTPUT_FIELD = "parent";
 
+    private HBaseTestingUtility testUtil;
+    private Path hbaseRootdir;
+
+
     public TestHBaseRootMarkerVertextFormat() {
         super(TestHBaseRootMarkerVertextFormat.class.getName());
+
+        // Let's set up the hbase root directory.
+        Configuration conf = HBaseConfiguration.create();
+        try {
+            FileSystem fs = FileSystem.get(conf);
+            String randomStr = UUID.randomUUID().toString();
+            String tmpdir = System.getProperty("java.io.tmpdir") +
+                randomStr + "/";
+            hbaseRootdir = fs.makeQualified(new Path(tmpdir));
+            conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
+            fs.mkdirs(hbaseRootdir);
+        } catch(IOException ioe) {
+            fail("Could not create hbase root directory.");
+        }
+
+        // Start the test utility.
+        testUtil = new HBaseTestingUtility(conf);
     }
 
     @Test
-    public void testHBaseInputOutput() throws Exception{
+    public void testHBaseInputOutput() throws Exception {
 
         if (System.getProperty("prop.mapred.job.tracker") != null) {
             if(log.isInfoEnabled())
@@ -93,7 +117,7 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
                     "Make sure you built the main Giraph artifact?.");
         }
 
-        String INPUT_FILE = "graph.csv";
+        String INPUT_FILE = hbaseRootdir.toString() + "/graph.csv";
         //First let's load some data using ImportTsv into our mock table.
         String[] args = new String[] {
                 "-Dimporttsv.columns=HBASE_ROW_KEY,cf:"+QUALIFER,
@@ -102,17 +126,23 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
                 INPUT_FILE
         };
 
-        MiniZooKeeperCluster zkCluster = testUtil.startMiniZKCluster();
-        MiniHBaseCluster cluster = testUtil.startMiniHBaseCluster(2, 2);        
-
-        GenericOptionsParser opts =
-                new GenericOptionsParser(cluster.getConfiguration(), args);
-        Configuration conf = opts.getConfiguration();
-        args = opts.getRemainingArgs();
+        MiniHBaseCluster cluster = null;
+        MiniZooKeeperCluster zkCluster = null;
+        FileSystem fs = null;
 
         try {
-
-            FileSystem fs = FileSystem.get(conf);
+            // using the restart method allows us to avoid having the hbase
+            // root directory overwritten by /home/$username
+            zkCluster = testUtil.startMiniZKCluster();
+            testUtil.restartHBaseCluster(2);
+            cluster = testUtil.getMiniHBaseCluster();
+
+            GenericOptionsParser opts =
+                    new GenericOptionsParser(cluster.getConfiguration(), args);
+            Configuration conf = opts.getConfiguration();
+            args = opts.getRemainingArgs();
+
+            fs = FileSystem.get(conf);
             FSDataOutputStream op = fs.create(new Path(INPUT_FILE), true);
             String line1 = "0001,0002\n";
             String line2 = "0002,0004\n";
@@ -132,8 +162,7 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
             HTableDescriptor desc = new HTableDescriptor(TAB);
             desc.addFamily(new HColumnDescriptor(FAM));
             HBaseAdmin hbaseAdmin=new HBaseAdmin(conf);
-            if (hbaseAdmin.isTableAvailable(TABLE_NAME))
-            {
+            if (hbaseAdmin.isTableAvailable(TABLE_NAME)) {
             	hbaseAdmin.disableTable(TABLE_NAME);
             	hbaseAdmin.deleteTable(TABLE_NAME);
             }
@@ -171,10 +200,17 @@ public class TestHBaseRootMarkerVertextFormat extends BspCase {
             assertNotNull(parentBytes);
             assertTrue(parentBytes.length > 0);
             Assert.assertEquals("0001", Bytes.toString(parentBytes));
-
         }   finally {
-            cluster.shutdown();
-            zkCluster.shutdown();
+            if (cluster != null) {
+                cluster.shutdown();
+            }
+            if (zkCluster != null) {
+                zkCluster.shutdown();
+            }
+            // clean test files
+            if (fs != null) {
+                fs.delete(hbaseRootdir);
+            }
         }
     }
 


Mime
View raw message