hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmhs...@apache.org
Subject svn commit: r1466257 - in /hbase/branches/0.95/hbase-server/src: main/java/org/apache/hadoop/hbase/mapreduce/Export.java test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
Date Tue, 09 Apr 2013 21:53:39 GMT
Author: jmhsieh
Date: Tue Apr  9 21:53:39 2013
New Revision: 1466257

URL: http://svn.apache.org/r1466257
Log:
HBASE-6330 TestImportExport has been failing against hadoop 0.23/2.0 profile

Modified:
    hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
    hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
    hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java

Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java?rev=1466257&r1=1466256&r2=1466257&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
(original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
Tue Apr  9 21:53:39 2013
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.Result;
@@ -78,7 +79,7 @@ public class Export {
     job.setOutputFormatClass(SequenceFileOutputFormat.class);
     job.setOutputKeyClass(ImmutableBytesWritable.class);
     job.setOutputValueClass(Result.class);
-    FileOutputFormat.setOutputPath(job, outputDir);
+    FileOutputFormat.setOutputPath(job, outputDir); // job conf doesn't contain the conf
so doesn't have a default fs.
     return job;
   }
 

Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1466257&r1=1466256&r2=1466257&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
(original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Tue Apr  9 21:53:39 2013
@@ -1648,6 +1648,10 @@ public class HBaseTestingUtility extends
 
     forceChangeTaskLogDir();
 
+    // Tests were failing because this process used 6GB of virtual memory and was getting
killed.
+    // we up the VM usable so that processes don't get killed.
+    conf.setFloat("yarn.nodemanager.vmem-pmem-ratio", 8.0f);
+
     // Allow the user to override FS URI for this map-reduce cluster to use.
     mrCluster = new MiniMRCluster(servers,
       FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1,
@@ -1656,6 +1660,7 @@ public class HBaseTestingUtility extends
     if (jobConf == null) {
       jobConf = mrCluster.createJobConf();
     }
+
     jobConf.set("mapred.local.dir",
       conf.get("mapred.local.dir")); //Hadoop MiniMR overwrites this while it should not
     LOG.info("Mini mapreduce cluster started");
@@ -1664,14 +1669,14 @@ public class HBaseTestingUtility extends
     conf.set("mapred.job.tracker", jobConf.get("mapred.job.tracker"));
     // this for mrv2 support; mr1 ignores this
     conf.set("mapreduce.framework.name", "yarn");
-    String rmAdress = jobConf.get("yarn.resourcemanager.address");
-    if (rmAdress != null) {
-      conf.set("yarn.resourcemanager.address", rmAdress);
+    String rmAddress = jobConf.get("yarn.resourcemanager.address");
+    if (rmAddress != null) {
+      conf.set("yarn.resourcemanager.address", rmAddress);
     }
-    String schedulerAdress =
+    String schedulerAddress =
       jobConf.get("yarn.resourcemanager.scheduler.address");
-    if (schedulerAdress != null) {
-      conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress);
+    if (schedulerAddress != null) {
+      conf.set("yarn.resourcemanager.scheduler.address", schedulerAddress);
     }
   }
 

Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java?rev=1466257&r1=1466256&r2=1466257&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
(original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
Tue Apr  9 21:53:39 2013
@@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.HColumnDe
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
@@ -51,9 +50,10 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import java.util.Iterator;
-import java.util.Map;
 
+/**
+ * Tests the table import and table export MR job functionality
+ */
 @Category(MediumTests.class)
 public class TestImportExport {
   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
@@ -65,15 +65,16 @@ public class TestImportExport {
   private static final byte[] FAMILYB = Bytes.toBytes(FAMILYB_STRING);
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
+  private static String FQ_OUTPUT_DIR;
   private static final String EXPORT_BATCH_SIZE = "100";
 
-  private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    cluster = UTIL.startMiniCluster();
+    UTIL.startMiniCluster();
     UTIL.startMiniMapReduceCluster();
+    FQ_OUTPUT_DIR =  new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString();
   }
 
   @AfterClass
@@ -90,24 +91,39 @@ public class TestImportExport {
   }
 
   /**
-   * When running on Hadoop 2, we need to copy (or add) configuration values for keys
-   * that start with "yarn." (from the map reduce minicluster) to the
-   * configuration that will be used during the test (from the HBase minicluster). 
-   * YARN configuration values are set properly in the map reduce minicluster, 
-   * but not necessarily in the HBase mini cluster.
-   * @param srcConf the configuration to copy from (the map reduce minicluster version)
-   * @param destConf the configuration to copy to (the HBase minicluster version)
+   * Runs an export job with the specified command line args
+   * @param args
+   * @return true if job completed successfully
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws ClassNotFoundException
    */
-  private void copyConfigurationValues(Configuration srcConf, Configuration destConf) {
-    Iterator<Map.Entry<String,String>> it = srcConf.iterator();
-    while (it.hasNext()) {
-      Map.Entry<String, String> entry = it.next();
-      String key = entry.getKey();
-      String value = entry.getValue();
-      if (key.startsWith("yarn.") && !value.isEmpty()) {
-        destConf.set(key, value);
-      }
-    }
+  boolean runExport(String[] args) throws IOException, InterruptedException, ClassNotFoundException
{
+    // need to make a copy of the configuration because to make sure different temp dirs
are used.
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()),
args);
+    Configuration conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+    Job job = Export.createSubmittableJob(conf, args);
+    job.waitForCompletion(false);
+    return job.isSuccessful();
+  }
+
+  /**
+   * Runs an import job with the specified command line args
+   * @param args
+   * @return true if job completed successfully
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws ClassNotFoundException
+   */
+  boolean runImport(String[] args) throws IOException, InterruptedException, ClassNotFoundException
{
+    // need to make a copy of the configuration because to make sure different temp dirs
are used.
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()),
args);
+    Configuration conf = opts.getConfiguration();
+    args = opts.getRemainingArgs();
+    Job job = Import.createSubmittableJob(conf, args);
+    job.waitForCompletion(false);
+    return job.isSuccessful();
   }
 
   /**
@@ -131,42 +147,19 @@ public class TestImportExport {
 
     String[] args = new String[] {
         EXPORT_TABLE,
-        OUTPUT_DIR,
+        FQ_OUTPUT_DIR,
         "1000", // max number of key versions per key to export
     };
-
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()),
args);
-    Configuration conf = opts.getConfiguration();
-
-    // copy or add the necessary configuration values from the map reduce config to the hbase
config
-    copyConfigurationValues(UTIL.getConfiguration(), conf);
-    args = opts.getRemainingArgs();
-
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
-    job.waitForCompletion(false);
-    assertTrue(job.isSuccessful());
-
+    assertTrue(runExport(args));
 
     String IMPORT_TABLE = "importTableSimpleCase";
     t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
     args = new String[] {
         "-D" + Import.CF_RENAME_PROP + "="+FAMILYA_STRING+":"+FAMILYB_STRING,
         IMPORT_TABLE,
-        OUTPUT_DIR
+        FQ_OUTPUT_DIR
     };
-
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
-    conf = opts.getConfiguration();
-
-    // copy or add the necessary configuration values from the map reduce config to the hbase
config
-    copyConfigurationValues(UTIL.getConfiguration(), conf);
-    args = opts.getRemainingArgs();
-
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
-    job.waitForCompletion(false);
-    assertTrue(job.isSuccessful());
+    assertTrue(runImport(args));
 
     Get g = new Get(ROW1);
     g.setMaxVersions();
@@ -186,19 +179,8 @@ public class TestImportExport {
   @Test
   public void testMetaExport() throws Exception {
     String EXPORT_TABLE = ".META.";
-    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        cluster.getConfiguration()), args);
-    Configuration conf = opts.getConfiguration();
-
-    // copy or add the necessary configuration values from the map reduce config to the hbase
config
-    copyConfigurationValues(UTIL.getConfiguration(), conf);
-    args = opts.getRemainingArgs();
-
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
-    job.waitForCompletion(false);
-    assertTrue(job.isSuccessful());
+    String[] args = new String[] { EXPORT_TABLE, FQ_OUTPUT_DIR, "1", "0", "0" };
+    assertTrue(runExport(args));
   }
 
   /**
@@ -225,21 +207,12 @@ public class TestImportExport {
     String[] args = new String[] {
         "-D" + Export.EXPORT_BATCHING + "=" + EXPORT_BATCH_SIZE,  // added scanner batching
arg.
         BATCH_TABLE,
-        OUTPUT_DIR
+        FQ_OUTPUT_DIR
     };
- 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()),
args);
-    Configuration conf = opts.getConfiguration();
-    args = opts.getRemainingArgs();
-    assertEquals(conf.get(Export.EXPORT_BATCHING), EXPORT_BATCH_SIZE);
+    assertTrue(runExport(args));
 
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
-    job.waitForCompletion(false);
-    assertTrue(job.isSuccessful());
-    
     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
-    fs.delete(new Path(OUTPUT_DIR), true);
+    fs.delete(new Path(FQ_OUTPUT_DIR), true);
   }
 
   @Test
@@ -266,26 +239,14 @@ public class TestImportExport {
     d = new Delete(ROW1);
     d.deleteColumns(FAMILYA, QUAL, now+2);
     t.delete(d);
-    
+
     String[] args = new String[] {
         "-D" + Export.RAW_SCAN + "=true",
         EXPORT_TABLE,
-        OUTPUT_DIR,
+        FQ_OUTPUT_DIR,
         "1000", // max number of key versions per key to export
     };
-
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()),
args);
-    Configuration conf = opts.getConfiguration();
-
-    // copy or add the necessary configuration values from the map reduce config to the hbase
config
-    copyConfigurationValues(UTIL.getConfiguration(), conf);
-    args = opts.getRemainingArgs();
-
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
-    job.waitForCompletion(false);
-    assertTrue(job.isSuccessful());
-
+    assertTrue(runExport(args));
 
     String IMPORT_TABLE = "importWithDeletes";
     desc = new HTableDescriptor(IMPORT_TABLE);
@@ -298,20 +259,9 @@ public class TestImportExport {
     t = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
     args = new String[] {
         IMPORT_TABLE,
-        OUTPUT_DIR
+        FQ_OUTPUT_DIR
     };
-
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
-    conf = opts.getConfiguration();
-
-    // copy or add the necessary configuration values from the map reduce config to the hbase
config
-    copyConfigurationValues(UTIL.getConfiguration(), conf);
-    args = opts.getRemainingArgs();
-
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
-    job.waitForCompletion(false);
-    assertTrue(job.isSuccessful());
+    assertTrue(runImport(args));
 
     Scan s = new Scan();
     s.setMaxVersions();
@@ -329,8 +279,13 @@ public class TestImportExport {
     t.close();
   }
 
+  /**
+   * Create a simple table, run an Export Job on it, Import with filtering on,  verify counts,
+   * attempt with invalid values.
+   */
   @Test
   public void testWithFilter() throws Exception {
+    // Create simple table to export
     String EXPORT_TABLE = "exportSimpleCase_ImportWithFilter";
     HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
     desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
@@ -345,18 +300,11 @@ public class TestImportExport {
     p.add(FAMILYA, QUAL, now + 4, QUAL);
     exportTable.put(p);
 
-    String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };
-
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
-        cluster.getConfiguration()), args);
-    Configuration conf = opts.getConfiguration();
-    args = opts.getRemainingArgs();
-
-    Job job = Export.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
-    job.waitForCompletion(false);
-    assertTrue(job.isSuccessful());
+    // Export the simple table
+    String[] args = new String[] { EXPORT_TABLE, FQ_OUTPUT_DIR, "1000" };
+    assertTrue(runExport(args));
 
+    // Import to a new table
     String IMPORT_TABLE = "importWithFilter";
     desc = new HTableDescriptor(IMPORT_TABLE);
     desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
@@ -364,17 +312,9 @@ public class TestImportExport {
 
     HTable importTable = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
     args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(),
-        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
+        "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, FQ_OUTPUT_DIR,
         "1000" };
-
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
-    conf = opts.getConfiguration();
-    args = opts.getRemainingArgs();
-
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
-    job.waitForCompletion(false);
-    assertTrue(job.isSuccessful());
+    assertTrue(runImport(args));
 
     // get the count of the source table for that time range
     PrefixFilter filter = new PrefixFilter(ROW1);
@@ -388,16 +328,8 @@ public class TestImportExport {
 
     args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(),
         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
-        OUTPUT_DIR, "1000" };
-
-    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
-    conf = opts.getConfiguration();
-    args = opts.getRemainingArgs();
-
-    job = Import.createSubmittableJob(conf, args);
-    job.getConfiguration().set("mapreduce.framework.name", "yarn");
-    job.waitForCompletion(false);
-    assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());
+        FQ_OUTPUT_DIR, "1000" };
+    assertFalse(runImport(args));
 
     // cleanup
     exportTable.close();



Mime
View raw message