hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject hive git commit: HIVE-10514 : Fix MiniCliDriver tests failure (Hari Subramaniyan, reviewed by Szehon Ho, Sushanth Sowmyan)
Date Fri, 01 May 2015 08:47:09 GMT
Repository: hive
Updated Branches:
  refs/heads/master aa1321aaa -> 283466374


HIVE-10514 : Fix MiniCliDriver tests failure (Hari Subramaniyan, reviewed by Szehon Ho, Sushanth
Sowmyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/28346637
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/28346637
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/28346637

Branch: refs/heads/master
Commit: 2834663742b42d2bb82e30714a3b96d1faaa3ff5
Parents: aa1321a
Author: Sushanth Sowmyan <khorgath@gmail.com>
Authored: Fri May 1 01:46:25 2015 -0700
Committer: Sushanth Sowmyan <khorgath@gmail.com>
Committed: Fri May 1 01:46:25 2015 -0700

----------------------------------------------------------------------
 .../src/test/templates/TestAccumuloCliDriver.vm | 19 +++++----
 .../apache/hadoop/hive/ant/QTestGenTask.java    | 25 ++++++++++++
 .../src/test/templates/TestHBaseCliDriver.vm    | 18 ++++----
 .../templates/TestHBaseNegativeCliDriver.vm     | 19 +++++----
 .../org/apache/hadoop/hive/ql/QTestUtil.java    | 43 ++++++++++++++++++++
 ql/src/test/templates/TestCliDriver.vm          | 18 ++++----
 ql/src/test/templates/TestCompareCliDriver.vm   | 21 +++++-----
 ql/src/test/templates/TestNegativeCliDriver.vm  | 18 ++++----
 ql/src/test/templates/TestParseNegative.vm      | 17 ++++----
 9 files changed, 139 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm b/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm
index dd1ee64..9872291 100644
--- a/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm
+++ b/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm
@@ -26,6 +26,7 @@ import java.util.*;
 
 import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil;
 import org.apache.hadoop.hive.accumulo.AccumuloTestSetup;
+import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.apache.hadoop.hive.ql.session.SessionState;
 
@@ -35,6 +36,12 @@ public class $className extends TestCase {
   private AccumuloQTestUtil qt;
   private AccumuloTestSetup setup;
 
+  public static class TestAccumuloCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor
{
+    public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+      suite.addTest(new $className("testCliDriver_"+tName, (AccumuloTestSetup)setup));
+    }
+  }
+
   public $className(String name, AccumuloTestSetup setup) {
     super(name);
     qt = null;
@@ -83,16 +90,12 @@ public class $className extends TestCase {
         }
       }
     }
+
     TestSuite suite = new TestSuite();
     AccumuloTestSetup setup = new AccumuloTestSetup(suite);
-#foreach ($qf in $qfiles)
-  #set ($fname = $qf.getName())
-  #set ($eidx = $fname.indexOf('.'))
-  #set ($tname = $fname.substring(0, $eidx))
-    if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
-      suite.addTest(new $className("testCliDriver_$tname", setup));
-    }
-#end
+
+    QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+      suite, setup, new TestAccumuloCliDriverAddTestFromQFiles());
     return setup;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
----------------------------------------------------------------------
diff --git a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
index ca959c0..60c7764 100644
--- a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
+++ b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.ant;
 
 
+import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileFilter;
 import java.io.FileWriter;
@@ -453,6 +454,29 @@ public class QTestGenTask extends Task {
         hadoopVersion = "";
       }
 
+      File qFileNames = new File(outputDirectory, className + "QFileNames.txt");
+      String qFileNamesFile = qFileNames.getCanonicalPath();
+
+      if (qFileNames.exists()) {
+        if (!qFileNames.delete()) {
+          throw new Exception("Could not delete old query file names containing file " +
+            qFileNamesFile);
+        }
+      }
+      if (!qFileNames.createNewFile()) {
+        throw new Exception("Could not create query file names containing file " +
+          qFileNamesFile);
+      }
+
+      FileWriter fw = new FileWriter(qFileNames.getCanonicalFile());
+      BufferedWriter bw = new BufferedWriter(fw);
+
+      for (File qFile: qFiles) {
+        bw.write(qFile.getName());
+        bw.newLine();
+      }
+      bw.close();
+
       // For each of the qFiles generate the test
       System.out.println("hiveRootDir = " + hiveRootDir);
       VelocityContext ctx = new VelocityContext();
@@ -464,6 +488,7 @@ public class QTestGenTask extends Task {
       System.out.println("queryDir = " + strQueryDir);
       ctx.put("queryDir", strQueryDir);
       ctx.put("qfiles", qFiles);
+      ctx.put("qFileNamesFile", qFileNamesFile);
       ctx.put("qfilesMap", qFilesMap);
       if (resultsDir != null) {
         ctx.put("resultsDir", relativePath(hiveRootDir, resultsDir));

http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
index 69fad97..de0be32 100644
--- a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
+++ b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
@@ -24,6 +24,7 @@ import junit.framework.TestSuite;
 import java.io.*;
 import java.util.*;
 
+import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
 import org.apache.hadoop.hive.hbase.HBaseTestSetup;
@@ -34,6 +35,12 @@ public class $className extends TestCase {
   private HBaseQTestUtil qt;
   private HBaseTestSetup setup;
 
+  public static class TestHBaseCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor
{
+    public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+      suite.addTest(new $className("testCliDriver_"+tName, (HBaseTestSetup)setup));
+    }
+  }
+
   public $className(String name, HBaseTestSetup setup) {
     super(name);
     qt = null;
@@ -84,14 +91,9 @@ public class $className extends TestCase {
     }
     TestSuite suite = new TestSuite();
     HBaseTestSetup setup = new HBaseTestSetup(suite);
-#foreach ($qf in $qfiles)
-  #set ($fname = $qf.getName())
-  #set ($eidx = $fname.indexOf('.'))
-  #set ($tname = $fname.substring(0, $eidx))
-    if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
-      suite.addTest(new $className("testCliDriver_$tname", setup));
-    }
-#end
+
+    QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+      suite, setup, new TestHBaseCliDriverAddTestFromQFiles());
     return setup;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
index 82f0584..b402585 100644
--- a/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
+++ b/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
@@ -25,6 +25,7 @@ import junit.framework.TestSuite;
 import java.io.*;
 import java.util.*;
 
+import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
 import org.apache.hadoop.hive.hbase.HBaseTestSetup;
@@ -35,6 +36,12 @@ public class $className extends TestCase {
   private HBaseQTestUtil qt;
   private HBaseTestSetup setup;
 
+  public static class TestHBaseNegativeCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor
{
+    public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+      suite.addTest(new $className("testCliDriver_"+tName, (HBaseTestSetup)setup));
+    }
+  }
+
   public $className(String name, HBaseTestSetup setup) {
     super(name);
     qt = null;
@@ -83,16 +90,12 @@ public class $className extends TestCase {
         }
       }
     }
+
     TestSuite suite = new TestSuite();
     HBaseTestSetup setup = new HBaseTestSetup(suite);
-#foreach ($qf in $qfiles)
-  #set ($fname = $qf.getName())
-  #set ($eidx = $fname.indexOf('.'))
-  #set ($tname = $fname.substring(0, $eidx))
-    if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
-      suite.addTest(new $className("testCliDriver_$tname", setup));
-    }
-#end
+
+    QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+      suite, setup, new TestHBaseNegativeCliDriverAddTestFromQFiles());
     return setup;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index f6081f4..d1104b3 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -55,6 +55,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import junit.framework.Assert;
+import junit.framework.TestSuite;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.io.FileUtils;
@@ -158,6 +159,10 @@ public class QTestUtil {
   private final String initScript;
   private final String cleanupScript;
 
+  public interface SuiteAddTestFunctor {
+    public void addTestToSuite(TestSuite suite, Object setup, String tName);
+  }
+
   static {
     for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) {
       srcTable = srcTable.trim();
@@ -1992,4 +1997,42 @@ public class QTestUtil {
         (command != null ? " running " + command : "") +
         (debugHint != null ? debugHint : ""));
   }
+
+  public static void addTestsToSuiteFromQfileNames(
+    String qFileNamesFile,
+    Set<String> qFilesToExecute,
+    TestSuite suite,
+    Object setup,
+    SuiteAddTestFunctor suiteAddTestCallback) {
+    try {
+      File qFileNames = new File(qFileNamesFile);
+      FileReader fr = new FileReader(qFileNames.getCanonicalFile());
+      BufferedReader br = new BufferedReader(fr);
+      String fName = null;
+
+      while ((fName = br.readLine()) != null) {
+        if (fName.isEmpty() || fName.trim().equals("")) {
+          continue;
+        }
+
+        int eIdx = fName.indexOf('.');
+
+        if (eIdx == -1) {
+          continue;
+        }
+
+        String tName = fName.substring(0, eIdx);
+
+        if (qFilesToExecute.isEmpty() || qFilesToExecute.contains(fName)) {
+          suiteAddTestCallback.addTestToSuite(suite, setup, tName);
+        }
+      }
+      br.close();
+    } catch (Exception e) {
+      System.err.println("Exception: " + e.getMessage());
+      e.printStackTrace();
+      System.err.flush();
+      Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm
index 8827f12..ae449c5 100644
--- a/ql/src/test/templates/TestCliDriver.vm
+++ b/ql/src/test/templates/TestCliDriver.vm
@@ -33,6 +33,12 @@ public class $className extends TestCase {
   private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
   private static QTestUtil qt;
 
+  public static class TestCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor
{
+    public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+      suite.addTest(new $className("testCliDriver_"+tName));
+    }
+  }
+
   static {
 
     MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
@@ -108,15 +114,11 @@ public class $className extends TestCase {
         }
       }
     }
+
     TestSuite suite = new TestSuite();
-#foreach ($qf in $qfiles)
-  #set ($fname = $qf.getName())
-  #set ($eidx = $fname.indexOf('.'))
-  #set ($tname = $fname.substring(0, $eidx))
-    if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
-      suite.addTest(new $className("testCliDriver_$tname"));
-    }
-#end
+
+    QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+      suite, null, new TestCliDriverAddTestFromQFiles());
     suite.addTest(new $className("testCliDriver_shutdown"));
     return suite;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestCompareCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestCompareCliDriver.vm b/ql/src/test/templates/TestCompareCliDriver.vm
index 6b367d6..4712708 100644
--- a/ql/src/test/templates/TestCompareCliDriver.vm
+++ b/ql/src/test/templates/TestCompareCliDriver.vm
@@ -34,6 +34,12 @@ public class $className extends TestCase {
   private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
   private static QTestUtil qt;
 
+  public static class TestCompareCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor
{
+    public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+      suite.addTest(new $className("testCompareCliDriver_"+tName));
+    }
+  }
+
   static {
 
     MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
@@ -109,18 +115,11 @@ public class $className extends TestCase {
         }
       }
     }
+
     TestSuite suite = new TestSuite();
-    
-    
-    
-#foreach ($qf in $qfiles)
-  #set ($fname = $qf.getName())
-  #set ($eidx = $fname.indexOf('.'))
-  #set ($tname = $fname.substring(0, $eidx))
-    if (qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
-      suite.addTest(new $className("testCompareCliDriver_$tname"));
-    }
-#end
+
+    QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+      suite, null, new TestCompareCliDriverAddTestFromQFiles());
     suite.addTest(new $className("testCompareCliDriver_shutdown"));
     return suite;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestNegativeCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestNegativeCliDriver.vm b/ql/src/test/templates/TestNegativeCliDriver.vm
index 742044a..5f8ee8e 100644
--- a/ql/src/test/templates/TestNegativeCliDriver.vm
+++ b/ql/src/test/templates/TestNegativeCliDriver.vm
@@ -33,6 +33,12 @@ public class $className extends TestCase {
   private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
   private static QTestUtil qt;
 
+  public static class TestNegativeCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor
{
+    public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+      suite.addTest(new $className("testNegativeCliDriver_"+tName));
+    }
+  }
+
   static {
     MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
     String initScript = "$initScript";
@@ -95,15 +101,11 @@ public class $className extends TestCase {
         }
       }
     }
+
     TestSuite suite = new TestSuite();
-#foreach ($qf in $qfiles)
-  #set ($fname = $qf.getName())
-  #set ($eidx = $fname.indexOf('.'))
-  #set ($tname = $fname.substring(0, $eidx))
-    if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
-      suite.addTest(new $className("testNegativeCliDriver_$tname"));
-    }
-#end
+
+    QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+      suite, null, new TestNegativeCliDriverAddTestFromQFiles());
     suite.addTest(new $className("testNegativeCliDriver_shutdown"));
     return suite;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestParseNegative.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestParseNegative.vm b/ql/src/test/templates/TestParseNegative.vm
index 33b238e..c5e7bdf 100755
--- a/ql/src/test/templates/TestParseNegative.vm
+++ b/ql/src/test/templates/TestParseNegative.vm
@@ -32,6 +32,12 @@ public class $className extends TestCase {
 
   private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
   private static QTestUtil qt;
+
+  public static class TestParseNegativeAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor
{
+    public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+      suite.addTest(new $className("testParseNegative_"+tName));
+    }
+  }
  
   static {
 
@@ -88,16 +94,11 @@ public class $className extends TestCase {
         }
       }
     }
+
     TestSuite suite = new TestSuite();
 
-#foreach ($qf in $qfiles)
-  #set ($fname = $qf.getName())
-  #set ($eidx = $fname.indexOf('.'))
-  #set ($tname = $fname.substring(0, $eidx))
-    if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
-      suite.addTest(new $className("testParseNegative_$tname"));
-    }
-#end
+    QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+      suite, null, new TestParseNegativeAddTestFromQFiles());
     suite.addTest(new $className("testParseNegative_shutdown"));
     return suite;
   }


Mime
View raw message