hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmhs...@apache.org
Subject svn commit: r1357844 - in /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase: HBaseTestingUtility.java mapreduce/MapreduceTestingShim.java mapreduce/TestImportExport.java
Date Thu, 05 Jul 2012 19:56:18 GMT
Author: jmhsieh
Date: Thu Jul  5 19:56:17 2012
New Revision: 1357844

URL: http://svn.apache.org/viewvc?rev=1357844&view=rev
Log:
Revert HBASE-6330 - hadoop 2.0 was fixed but this broke hadoop 1.0 build.

Modified:
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1357844&r1=1357843&r2=1357844&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Thu Jul  5 19:56:17 2012
@@ -330,6 +330,21 @@ public class HBaseTestingUtility {
     createSubDirAndSystemProperty(
       "hadoop.log.dir",
       testPath, "hadoop-log-dir");
+
+    // This is defaulted in core-default.xml to /tmp/hadoop-${user.name}, but
+    //  we want our own value to ensure uniqueness on the same machine
+    createSubDirAndSystemProperty(
+      "hadoop.tmp.dir",
+      testPath, "hadoop-tmp-dir");
+
+    // Read and modified in org.apache.hadoop.mapred.MiniMRCluster
+    createSubDir(
+      "mapred.local.dir",
+      testPath, "mapred-local-dir");
+
+    createSubDirAndSystemProperty(
+      "mapred.working.dir",
+      testPath, "mapred-working-dir");
   }
 
   private void createSubDir(String propertyName, Path parent, String subDirName){
@@ -422,7 +437,7 @@ public class HBaseTestingUtility {
 
     return this.dfsCluster;
   }
-  
+
   public MiniDFSCluster startMiniDFSClusterForTestHLog(int namenodePort) throws IOException
{
     createDirsAndSetProperties();
     dfsCluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null,
@@ -430,14 +445,17 @@ public class HBaseTestingUtility {
     return dfsCluster;
   }
 
-
   /** This is used before starting HDFS and map-reduce mini-clusters */
   private void createDirsAndSetProperties() {
     setupClusterTestDir();
     System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
+    createDirAndSetProperty("cache_data", "test.cache.data");
     createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
+    hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
     createDirAndSetProperty("mapred_output", MapreduceTestingShim.getMROutputDirProp());
     createDirAndSetProperty("mapred_local", "mapred.local.dir");
+    createDirAndSetProperty("mapred_system", "mapred.system.dir");
+    createDirAndSetProperty("mapred_temp", "mapred.temp.dir");
   }
 
   private String createDirAndSetProperty(final String relPath, String property) {
@@ -1315,12 +1333,16 @@ public class HBaseTestingUtility {
     // Allow the user to override FS URI for this map-reduce cluster to use.
     mrCluster = new MiniMRCluster(servers,
       FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1);
-    JobConf jobConf = mrCluster.createJobConf();
+    JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
+    if (jobConf == null) {
+      jobConf = mrCluster.createJobConf();
+    }
+    jobConf.set("mapred.local.dir",
+      conf.get("mapred.local.dir")); //Hadoop MiniMR overwrites this while it should not
     LOG.info("Mini mapreduce cluster started");
 
-    // This fixes TestImportTsv but breaks TestImportExport tests
-    conf.set("mapred.job.tracker",
-        mrCluster.createJobConf().get("mapred.job.tracker"));
+    // Needed for TestImportTsv.
+    conf.set("mapred.job.tracker", jobConf.get("mapred.job.tracker"));
     // this for mrv2 support; mr1 ignores this 
     conf.set("mapreduce.framework.name", "yarn");
     String rmAdress = jobConf.get("yarn.resourcemanager.address");

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java?rev=1357844&r1=1357843&r2=1357844&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
Thu Jul  5 19:56:17 2012
@@ -19,9 +19,12 @@ package org.apache.hadoop.hbase.mapreduc
 
 import java.io.IOException;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobID;
@@ -34,6 +37,7 @@ import org.apache.hadoop.mapreduce.JobID
  */
 abstract public class MapreduceTestingShim {
   private static MapreduceTestingShim instance;
+  private static Class[] emptyParam = new Class[] {};
 
   static {
     try {
@@ -48,6 +52,8 @@ abstract public class MapreduceTestingSh
 
   abstract public JobContext newJobContext(Configuration jobConf)
       throws IOException;
+  
+  abstract public JobConf obtainJobConf(MiniMRCluster cluster);
 
   abstract public String obtainMROutputDirProp();
   
@@ -55,6 +61,10 @@ abstract public class MapreduceTestingSh
       throws IOException {
     return instance.newJobContext(jobConf);
   }
+  
+  public static JobConf getJobConf(MiniMRCluster cluster) {
+    return instance.obtainJobConf(cluster);
+  }
 
   public static String getMROutputDirProp() {
     return instance.obtainMROutputDirProp();
@@ -74,6 +84,23 @@ abstract public class MapreduceTestingSh
             "Failed to instantiate new JobContext(jobConf, new JobID())", e);
       }
     }
+    
+    public JobConf obtainJobConf(MiniMRCluster cluster) {
+      if (cluster == null) return null;
+      try {
+        Object runner = cluster.getJobTrackerRunner();
+        Method meth = runner.getClass().getDeclaredMethod("getJobTracker", emptyParam);
+        Object tracker = meth.invoke(runner, new Object []{});
+        Method m = tracker.getClass().getDeclaredMethod("getConf", emptyParam);
+        return (JobConf) m.invoke(tracker, new Object []{});
+      } catch (NoSuchMethodException nsme) {
+        return null;
+      } catch (InvocationTargetException ite) {
+        return null;
+      } catch (IllegalAccessException iae) {
+        return null;
+      }
+    }
 
     @Override
     public String obtainMROutputDirProp() {
@@ -94,6 +121,19 @@ abstract public class MapreduceTestingSh
             "Failed to return from Job.getInstance(jobConf)");
       }
     }
+    
+    public JobConf obtainJobConf(MiniMRCluster cluster) {
+      try {
+        Method meth = MiniMRCluster.class.getMethod("getJobTrackerConf", emptyParam);
+        return (JobConf) meth.invoke(cluster, new Object []{});
+      } catch (NoSuchMethodException nsme) {
+        return null;
+      } catch (InvocationTargetException ite) {
+        return null;
+      } catch (IllegalAccessException iae) {
+        return null;
+      }
+    }
 
     @Override
     public String obtainMROutputDirProp() {

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java?rev=1357844&r1=1357843&r2=1357844&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
Thu Jul  5 19:56:17 2012
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import org.apache.hadoop.conf.Configuration;
@@ -28,6 +27,7 @@ import org.apache.hadoop.hbase.HColumnDe
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.client.Re
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.HBaseFsck;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.junit.After;
@@ -45,6 +44,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import static org.junit.Assert.assertEquals;
 
 @Category(MediumTests.class)
 public class TestImportExport {
@@ -58,11 +58,12 @@ public class TestImportExport {
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
 
+  private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    UTIL.startMiniCluster();
+    cluster = UTIL.startMiniCluster();
     UTIL.startMiniMapReduceCluster();
   }
 
@@ -104,16 +105,16 @@ public class TestImportExport {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()),
args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()),
args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
     Job job = Export.createSubmittableJob(conf, args);
     job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
-    HBaseFsck.debugLsr(conf, new Path("."));
     assertTrue(job.isSuccessful());
 
+
     String IMPORT_TABLE = "importTableSimpleCase";
     t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
     args = new String[] {
@@ -122,14 +123,13 @@ public class TestImportExport {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
     job = Import.createSubmittableJob(conf, args);
     job.getConfiguration().set("mapreduce.framework.name", "yarn");
     job.waitForCompletion(false);
-    HBaseFsck.debugLsr(conf, new Path("."));
     assertTrue(job.isSuccessful());
 
     Get g = new Get(ROW1);
@@ -174,7 +174,7 @@ public class TestImportExport {
         "1000"
     };
 
-    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()),
args);
+    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()),
args);
     Configuration conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 
@@ -198,7 +198,7 @@ public class TestImportExport {
         OUTPUT_DIR
     };
 
-    opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+    opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
     conf = opts.getConfiguration();
     args = opts.getRemainingArgs();
 



Mime
View raw message