hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject svn commit: r957290 - in /hadoop/hive/branches/branch-0.6: ./ ant/src/org/apache/hadoop/hive/ant/ hbase-handler/src/test/org/apache/hadoop/hive/hbase/ ql/ ql/src/test/org/apache/hadoop/hive/ql/ ql/src/test/templates/
Date Wed, 23 Jun 2010 18:23:24 GMT
Author: jvs
Date: Wed Jun 23 18:23:23 2010
New Revision: 957290

URL: http://svn.apache.org/viewvc?rev=957290&view=rev
Log:
HIVE-1359. Unit test should be shim-aware
(Ning Zhang via jvs)


Modified:
    hadoop/hive/branches/branch-0.6/CHANGES.txt
    hadoop/hive/branches/branch-0.6/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
    hadoop/hive/branches/branch-0.6/build-common.xml
    hadoop/hive/branches/branch-0.6/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
    hadoop/hive/branches/branch-0.6/ql/build.xml
    hadoop/hive/branches/branch-0.6/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
    hadoop/hive/branches/branch-0.6/ql/src/test/templates/TestCliDriver.vm

Modified: hadoop/hive/branches/branch-0.6/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/CHANGES.txt?rev=957290&r1=957289&r2=957290&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.6/CHANGES.txt Wed Jun 23 18:23:23 2010
@@ -227,6 +227,9 @@ Release 0.6.0 -  Unreleased
     HIVE-1383. Allow HBase WAL to be disabled.
     (John Sichi via Ning Zhang)
 
+    HIVE-1359. Unit test should be shim-aware
+    (Ning Zhang via jvs)
+
   OPTIMIZATIONS
 
     HIVE-1348. Move inputFileChanged() from ExecMapper to where it is needed

Modified: hadoop/hive/branches/branch-0.6/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java?rev=957290&r1=957289&r2=957290&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (original)
+++ hadoop/hive/branches/branch-0.6/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java Wed
Jun 23 18:23:23 2010
@@ -95,7 +95,17 @@ public class QTestGenTask extends Task {
   private String logFile;
 
   private String clusterMode;
+  
+  private String hadoopVersion;
+
+  public void setHadoopVersion(String ver) {
+    this.hadoopVersion = ver;
+  }
 
+  public String getHadoopVersion() {
+    return hadoopVersion;
+  }
+  
   public void setClusterMode(String clusterMode) {
     this.clusterMode = clusterMode;
   }
@@ -291,6 +301,9 @@ public class QTestGenTask extends Task {
       if (clusterMode == null) {
         clusterMode = new String("");
       }
+      if (hadoopVersion == null) {
+        hadoopVersion = "";
+      }
 
       // For each of the qFiles generate the test
       VelocityContext ctx = new VelocityContext();
@@ -299,6 +312,7 @@ public class QTestGenTask extends Task {
       ctx.put("resultsDir", resultsDir);
       ctx.put("logDir", logDir);
       ctx.put("clusterMode", clusterMode);
+      ctx.put("hadoopVersion", hadoopVersion);
 
       File outFile = new File(outDir, className + ".java");
       FileWriter writer = new FileWriter(outFile);

Modified: hadoop/hive/branches/branch-0.6/build-common.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/build-common.xml?rev=957290&r1=957289&r2=957290&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/build-common.xml (original)
+++ hadoop/hive/branches/branch-0.6/build-common.xml Wed Jun 23 18:23:23 2010
@@ -52,6 +52,7 @@
   <property name="test.junit.output.usefile" value="true"/>
   <property name="minimr.query.files" value="join1.q,groupby1.q"/>
   <property name="test.silent" value="true"/>
+  <property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
   <property name="test.serialize.qplan" value="true"/>
 
   <path id="test.classpath">

Modified: hadoop/hive/branches/branch-0.6/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java?rev=957290&r1=957289&r2=957290&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
(original)
+++ hadoop/hive/branches/branch-0.6/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
Wed Jun 23 18:23:23 2010
@@ -27,7 +27,7 @@ public class HBaseQTestUtil extends QTes
     String outDir, String logDir, boolean miniMr, HBaseTestSetup setup)
     throws Exception {
 
-    super(outDir, logDir, miniMr);
+    super(outDir, logDir, miniMr, null);
     setup.preTest(conf);
     super.init();
   }

Modified: hadoop/hive/branches/branch-0.6/ql/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/build.xml?rev=957290&r1=957289&r2=957290&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/build.xml (original)
+++ hadoop/hive/branches/branch-0.6/ql/build.xml Wed Jun 23 18:23:23 2010
@@ -88,7 +88,9 @@
               clusterMode="${clustermode}"
               resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestCliDriver"
               logFile="${test.log.dir}/testclidrivergen.log"
-              logDirectory="${test.log.dir}/clientpositive"/>
+              logDirectory="${test.log.dir}/clientpositive"
+              hadoopVersion="${hadoopVersion}"
+    />
 
     <qtestgen outputDirectory="${test.build.src}/org/apache/hadoop/hive/cli" 
               templatePath="${ql.test.template.dir}" template="TestNegativeCliDriver.vm"


Modified: hadoop/hive/branches/branch-0.6/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=957290&r1=957289&r2=957290&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/branches/branch-0.6/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Wed
Jun 23 18:23:23 2010
@@ -28,6 +28,7 @@ import java.io.PrintStream;
 import java.io.Serializable;
 import java.net.URI;
 import java.util.ArrayList;
+import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -96,6 +97,7 @@ public class QTestUtil {
   private MiniMRCluster mr = null;
   private HadoopShims.MiniDFSShim dfs = null;
   private boolean miniMr = false;
+  private String hadoopVer = null;
 
   public boolean deleteDirectory(File path) {
     if (path.exists()) {
@@ -167,14 +169,27 @@ public class QTestUtil {
   }
 
   public QTestUtil(String outDir, String logDir) throws Exception {
-    this(outDir, logDir, false);
+    this(outDir, logDir, false, "0.20");
   }
 
-  public QTestUtil(String outDir, String logDir, boolean miniMr) throws Exception {
+  private String getHadoopMainVersion(String input) {
+    if (input == null) {
+      return null;
+    }
+    Pattern p = Pattern.compile("^(\\d+\\.\\d+).*");
+    Matcher m = p.matcher(input);
+    if (m.matches()) {
+      return m.group(1);
+    }
+    return null;
+  }
+
+  public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer) throws
Exception {
     this.outDir = outDir;
     this.logDir = logDir;
     conf = new HiveConf(Driver.class);
     this.miniMr = miniMr;
+    this.hadoopVer = getHadoopMainVersion(hadoopVer);
     qMap = new TreeMap<String, String>();
     qSkipSet = new HashSet<String>();
 
@@ -706,6 +721,7 @@ public class QTestUtil {
 
   }
 
+
   public int checkResults(String tname) throws Exception {
     Path warehousePath = new Path(FileSystem.get(conf).getUri().getPath());
     warehousePath = new Path(warehousePath, (new URI(testWarehouse)).getPath());
@@ -783,10 +799,47 @@ public class QTestUtil {
     return exitVal;
   }
 
+  /**
+   * Given the current configurations (e.g., hadoop version and execution mode), return
+   * the correct file name to compare with the current test run output.
+   * @param outDir The directory where the reference log files are stored.
+   * @param testName The test file name (terminated by ".out").
+   * @return The file name appended with the configuration values if it exists.
+   */
+  public String outPath(String outDir, String testName) {
+    String ret = testName;
+    // List of configurations. Currently the list consists of hadoop version and execution
mode only
+    List<String> configs = new ArrayList<String>();
+    configs.add(this.hadoopVer);
+
+    Deque<String> stack = new LinkedList<String>();
+    StringBuilder sb = new StringBuilder();
+    sb.append(testName);
+    stack.push(sb.toString());
+
+    // example file names are input1.q.out_0.20.0_minimr or input2.q.out_0.17
+    for (String s: configs) {
+      sb.append('_');
+      sb.append(s);
+      stack.push(sb.toString());
+    }
+    while (stack.size() > 0) {
+      String fileName = stack.pop();
+      File f = new File(outDir, fileName);
+      if (f.exists()) {
+        ret = f.getPath();
+        break;
+      }
+    }
+   return ret;
+  }
+
   public int checkCliDriverResults(String tname) throws Exception {
     String[] cmdArray;
     assert(qMap.containsKey(tname));
 
+    String outFileName = outPath(outDir, tname + ".out");
+
     cmdArray = new String[] {
         "diff", "-a",
         "-I", "file:",
@@ -804,7 +857,7 @@ public class QTestUtil {
         "-I", "Caused by:",
         "-I", "[.][.][.] [0-9]* more",
         (new File(logDir, tname + ".out")).getPath(),
-        (new File(outDir, tname + ".out")).getPath()};
+        outFileName };
 
     System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' '));
 
@@ -825,7 +878,7 @@ public class QTestUtil {
       cmdArray = new String[3];
       cmdArray[0] = "cp";
       cmdArray[1] = (new File(logDir, tname + ".out")).getPath();
-      cmdArray[2] = (new File(outDir, tname + ".out")).getPath();
+      cmdArray[2] = outFileName;
       executor = Runtime.getRuntime().exec(cmdArray);
       exitVal = executor.waitFor();
     }

Modified: hadoop/hive/branches/branch-0.6/ql/src/test/templates/TestCliDriver.vm
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/test/templates/TestCliDriver.vm?rev=957290&r1=957289&r2=957290&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/test/templates/TestCliDriver.vm (original)
+++ hadoop/hive/branches/branch-0.6/ql/src/test/templates/TestCliDriver.vm Wed Jun 23 18:23:23
2010
@@ -33,10 +33,13 @@ public class $className extends TestCase
   protected void setUp() {
     try {
       boolean miniMR = false;
+      String hadoopVer;
+
       if ("$clusterMode".equals("miniMR"))
         miniMR = true;
+      hadoopVer = "$hadoopVersion";
 
-      qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()",
miniMR);
+      qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()",
miniMR, hadoopVer);
 
     }
     catch (Exception e) {



Mime
View raw message