incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1225693 - in /incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase: ./ src/main/groovy/org/apache/bigtop/itest/hbase/system/ src/main/groovy/org/apache/bigtop/itest/smoke/ src/main/groovy/org/apache/bigtop/itest/system/ ...
Date Thu, 29 Dec 2011 23:45:53 GMT
Author: rvs
Date: Thu Dec 29 23:45:52 2011
New Revision: 1225693

URL: http://svn.apache.org/viewvc?rev=1225693&view=rev
Log:
BIGTOP-321. Make TestLoadAndVerify parameterizable and runnable from command line (Jonathan
Hsieh via rvs)

Removed:
    incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/smoke/IncrementalPELoad.java
    incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/smoke/TestHBaseCompression.groovy
    incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/smoke/TestHBaseSmoke.java
    incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/smoke/TestHFileOutputFormat.java
    incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/smoke/TestLoadIncrementalHFiles.java
    incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/system/TestLoadAndVerify.java
    incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/util/HBaseTestUtil.java
Modified:
    incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/pom.xml
    incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/pom.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/pom.xml?rev=1225693&r1=1225692&r2=1225693&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/pom.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/pom.xml Thu Dec
29 23:45:52 2011
@@ -31,13 +31,15 @@
   <name>hbasesmoke</name>
 
   <dependencies>
-     <dependency>
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-mapred</artifactId>
+      <artifactId>hadoop-common</artifactId>
+      <version>0.23.0-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <version>0.23.0-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hbase</groupId>

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java?rev=1225693&r1=1225692&r2=1225693&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java
(original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java
Thu Dec 29 23:45:52 2011
@@ -17,17 +17,20 @@
  */
 package org.apache.bigtop.itest.hbase.system;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Random;
-
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.bigtop.itest.hbase.util.HBaseTestUtil;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
@@ -45,20 +48,18 @@ import org.apache.hadoop.io.BytesWritabl
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 
-import org.apache.bigtop.itest.hbase.util.HBaseTestUtil;
-
-public class TestLoadAndVerify {
+public class TestLoadAndVerify  extends Configured implements Tool {
   private static final String TEST_NAME = "TestLoadAndVerify";
   private static final byte[] TEST_FAMILY = Bytes.toBytes("f1");
   private static final byte[] TEST_QUALIFIER = Bytes.toBytes("q1");
@@ -245,12 +246,11 @@ public class TestLoadAndVerify {
     }
   }
 
-  private void doLoad(HTableDescriptor htd) throws Exception {
+  private void doLoad(Configuration conf, HTableDescriptor htd) throws Exception {
     Path outputDir = 
       new Path(HBaseTestUtil.getMROutputDir(TEST_NAME),
           "load-output");
 
-    Configuration conf = HBaseConfiguration.create();
     NMapInputFormat.setNumMapTasks(conf, NUM_TASKS);
     conf.set(TABLE_NAME_KEY, htd.getNameAsString());
 
@@ -268,12 +268,11 @@ public class TestLoadAndVerify {
     assertTrue(job.waitForCompletion(true));
   }
 
-  private void doVerify(HTableDescriptor htd) throws Exception {
+  private void doVerify(Configuration conf, HTableDescriptor htd) throws Exception {
     Path outputDir = 
       new Path(HBaseTestUtil.getMROutputDir(TEST_NAME),
           "verify-output");
 
-    Configuration conf = HBaseConfiguration.create();
     Job job = new Job(conf);
     job.setJarByClass(this.getClass());
     job.setJobName(TEST_NAME + " Verification for " + htd.getNameAsString());
@@ -290,8 +289,7 @@ public class TestLoadAndVerify {
     FileOutputFormat.setOutputPath(job, outputDir);
     assertTrue(job.waitForCompletion(true));
 
-    long numOutputRecords = job.getCounters().findCounter(
-        org.apache.hadoop.mapred.Task.Counter.REDUCE_OUTPUT_RECORDS).getValue();
+    long numOutputRecords = job.getCounters().findCounter(TaskCounter.REDUCE_OUTPUT_RECORDS).getValue();
     assertEquals(0, numOutputRecords);
   }
 
@@ -303,12 +301,102 @@ public class TestLoadAndVerify {
     int numPreCreate = 40;
     admin.createTable(htd, Bytes.toBytes(0L), Bytes.toBytes(-1L), numPreCreate);
 
-    doLoad(htd);
-    doVerify(htd);
+    Configuration conf = HBaseConfiguration.create();
+
+    doLoad(conf, htd);
+    doVerify(conf, htd);
 
     // Only disable and drop if we succeeded to verify - otherwise it's useful
     // to leave it around for post-mortem
-    admin.disableTable(htd.getName());
+    deleteTable(admin, htd);
+  }
+
+  private void deleteTable(HBaseAdmin admin, HTableDescriptor htd) 
+    throws IOException, InterruptedException {
+    // Use disableTestAsync because disable can take a long time to complete
+    System.out.print("Disabling table " + htd.getNameAsString() +" ");
+    admin.disableTableAsync(htd.getName());
+    
+    long start = System.currentTimeMillis();
+    // NOTE tables can be both admin.isTableEnabled=false and 
+    // isTableDisabled=false, when disabling must use isTableDisabled!
+    while (!admin.isTableDisabled(htd.getName())) {
+      System.out.print(".");
+      Thread.sleep(1000);
+    }
+    long delta = System.currentTimeMillis() - start;
+    System.out.println(" " + delta +" ms");
+    System.out.println("Deleting table " + htd.getNameAsString() +" ");
     admin.deleteTable(htd.getName());
   }
+  
+  public void usage() {
+    System.err.println(this.getClass().getSimpleName() + " [-Doptions] <load|verify|loadAndVerify>");
+    System.err.println("  Loads a table with row dependencies and verifies the dependency
chains");
+    System.err.println("Options");
+    System.err.println("  -Dloadmapper.table=<name>        Table to write/verify (default
autogen)");
+    System.err.println("  -Dloadmapper.backrefs=<n>        Number of backreferences
per row (default 50)");
+    System.err.println("  -Dloadmapper.num_to_write=<n>    Number of rows per mapper
(default 100,000 per mapper)");
+    System.err.println("  -Dloadmapper.deleteAfter=<bool>  Delete after a successful
verify (default true)");
+    System.err.println("  -Dloadmapper.numPresplits=<n>    Number of presplit regions
to start with (default 40)");
+
+  }
+  
+  public int run(String argv[]) throws Exception {
+    if (argv.length < 1 || argv.length > 1) {
+      usage();
+      return 1;
+    }
+
+    boolean doLoad = false;
+    boolean doVerify = false;
+    boolean doDelete = getConf().getBoolean("loadmapper.deleteAfter",true);
+    int numPresplits = getConf().getInt("loadmapper.numPresplits", 40);
+
+    if (argv[0].equals("load")) {
+      doLoad = true;
+    } else if (argv[0].equals("verify")) {
+      doVerify= true;
+    } else if (argv[0].equals("loadAndVerify")) {
+      doLoad=true;
+      doVerify= true;
+    } else {
+      System.err.println("Invalid argument " + argv[0]);
+      usage();
+      return 1;
+    }
+
+    // create HTableDescriptor for specified table
+    String table = getConf().get(TABLE_NAME_KEY, "");
+    HTableDescriptor htd ;
+    if ("".equals(table)) {
+      // Just like the unit test.
+      htd = HBaseTestUtil.createTestTableDescriptor(TEST_NAME, TEST_FAMILY);
+    } else {
+      // overridden by the user.
+      htd = new HTableDescriptor(table);
+      htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+    }
+
+    TestLoadAndVerify verify = new TestLoadAndVerify();
+
+    HBaseAdmin admin = new HBaseAdmin(getConf());
+    if (doLoad) {
+      admin.createTable(htd, Bytes.toBytes(0L), Bytes.toBytes(-1L), numPresplits);
+      verify.doLoad(getConf(), htd);
+    }
+    if (doVerify) {
+      verify.doVerify(getConf(), htd);
+      if (doDelete) {
+        deleteTable(admin, htd);
+      }
+    }
+    return 0;
+  }
+
+  public static void main(String argv[]) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    int ret = ToolRunner.run(conf, new TestLoadAndVerify(), argv);
+    System.exit(ret);
+  }
 }



Mime
View raw message