Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0C4BC17CA9 for ; Fri, 1 May 2015 08:47:10 +0000 (UTC) Received: (qmail 67698 invoked by uid 500); 1 May 2015 08:47:09 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 67653 invoked by uid 500); 1 May 2015 08:47:09 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 67642 invoked by uid 99); 1 May 2015 08:47:09 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 May 2015 08:47:09 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AFDB4DFF8D; Fri, 1 May 2015 08:47:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: khorgath@apache.org To: commits@hive.apache.org Message-Id: <4b0310f5029144d794afaeb3b23da132@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-10514 : Fix MiniCliDriver tests failure (Hari Subramaniyan, reviewed by Szehon Ho, Sushanth Sowmyan) Date: Fri, 1 May 2015 08:47:09 +0000 (UTC) Repository: hive Updated Branches: refs/heads/master aa1321aaa -> 283466374 HIVE-10514 : Fix MiniCliDriver tests failure (Hari Subramaniyan, reviewed by Szehon Ho, Sushanth Sowmyan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/28346637 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/28346637 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/28346637 Branch: refs/heads/master Commit: 2834663742b42d2bb82e30714a3b96d1faaa3ff5 Parents: aa1321a Author: Sushanth Sowmyan Authored: Fri May 1 01:46:25 2015 -0700 Committer: Sushanth Sowmyan Committed: Fri May 1 01:46:25 2015 -0700 ---------------------------------------------------------------------- .../src/test/templates/TestAccumuloCliDriver.vm | 19 +++++---- .../apache/hadoop/hive/ant/QTestGenTask.java | 25 ++++++++++++ .../src/test/templates/TestHBaseCliDriver.vm | 18 ++++---- .../templates/TestHBaseNegativeCliDriver.vm | 19 +++++---- .../org/apache/hadoop/hive/ql/QTestUtil.java | 43 ++++++++++++++++++++ ql/src/test/templates/TestCliDriver.vm | 18 ++++---- ql/src/test/templates/TestCompareCliDriver.vm | 21 +++++----- ql/src/test/templates/TestNegativeCliDriver.vm | 18 ++++---- ql/src/test/templates/TestParseNegative.vm | 17 ++++---- 9 files changed, 139 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm ---------------------------------------------------------------------- diff --git a/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm b/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm index dd1ee64..9872291 100644 --- a/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm +++ b/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm @@ -26,6 +26,7 @@ import java.util.*; import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil; import org.apache.hadoop.hive.accumulo.AccumuloTestSetup; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.ql.session.SessionState; @@ -35,6 +36,12 @@ public class $className extends TestCase { private AccumuloQTestUtil qt; private AccumuloTestSetup setup; + public static class TestAccumuloCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCliDriver_"+tName, (AccumuloTestSetup)setup)); + } + } + public $className(String name, AccumuloTestSetup setup) { super(name); qt = null; @@ -83,16 +90,12 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); AccumuloTestSetup setup = new AccumuloTestSetup(suite); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCliDriver_$tname", setup)); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, setup, new TestAccumuloCliDriverAddTestFromQFiles()); return setup; } http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java ---------------------------------------------------------------------- diff --git a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java index ca959c0..60c7764 100644 --- a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java +++ b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ant; +import java.io.BufferedWriter; import java.io.File; import java.io.FileFilter; import java.io.FileWriter; @@ -453,6 +454,29 @@ public class QTestGenTask extends Task { hadoopVersion = ""; } + File qFileNames = new File(outputDirectory, className + "QFileNames.txt"); + String qFileNamesFile = qFileNames.getCanonicalPath(); + + if (qFileNames.exists()) { + if (!qFileNames.delete()) { + throw new Exception("Could not delete old query file names containing file " + + qFileNamesFile); + } + } + if (!qFileNames.createNewFile()) { + throw new Exception("Could not create query file names containing file " + + qFileNamesFile); + } + + FileWriter fw = new FileWriter(qFileNames.getCanonicalFile()); + BufferedWriter bw = new BufferedWriter(fw); + + for (File qFile: qFiles) { + bw.write(qFile.getName()); + bw.newLine(); + } + bw.close(); + // For each of the qFiles generate the test System.out.println("hiveRootDir = " + hiveRootDir); VelocityContext ctx = new VelocityContext(); @@ -464,6 +488,7 @@ public class QTestGenTask extends Task { System.out.println("queryDir = " + strQueryDir); ctx.put("queryDir", strQueryDir); ctx.put("qfiles", qFiles); + ctx.put("qFileNamesFile", qFileNamesFile); ctx.put("qfilesMap", qFilesMap); if (resultsDir != null) { ctx.put("resultsDir", relativePath(hiveRootDir, resultsDir)); http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/hbase-handler/src/test/templates/TestHBaseCliDriver.vm ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm index 69fad97..de0be32 100644 --- a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm +++ b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm @@ -24,6 +24,7 @@ import junit.framework.TestSuite; import java.io.*; import java.util.*; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; @@ -34,6 +35,12 @@ public class $className extends TestCase { private HBaseQTestUtil qt; private HBaseTestSetup setup; + public static class TestHBaseCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCliDriver_"+tName, (HBaseTestSetup)setup)); + } + } + public $className(String name, HBaseTestSetup setup) { super(name); qt = null; @@ -84,14 +91,9 @@ public class $className extends TestCase { } TestSuite suite = new TestSuite(); HBaseTestSetup setup = new HBaseTestSetup(suite); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCliDriver_$tname", setup)); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, setup, new TestHBaseCliDriverAddTestFromQFiles()); return setup; } http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm index 82f0584..b402585 100644 --- a/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm +++ b/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm @@ -25,6 +25,7 @@ import junit.framework.TestSuite; import java.io.*; import java.util.*; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; @@ -35,6 +36,12 @@ public class $className extends TestCase { private HBaseQTestUtil qt; private HBaseTestSetup setup; + public static class TestHBaseNegativeCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCliDriver_"+tName, (HBaseTestSetup)setup)); + } + } + public $className(String name, HBaseTestSetup setup) { super(name); qt = null; @@ -83,16 +90,12 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); HBaseTestSetup setup = new HBaseTestSetup(suite); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCliDriver_$tname", setup)); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, setup, new TestHBaseNegativeCliDriverAddTestFromQFiles()); return setup; } http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index f6081f4..d1104b3 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -55,6 +55,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import junit.framework.Assert; +import junit.framework.TestSuite; import org.apache.commons.lang.StringUtils; import org.apache.commons.io.FileUtils; @@ -158,6 +159,10 @@ public class QTestUtil { private final String initScript; private final String cleanupScript; + public interface SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName); + } + static { for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) { srcTable = srcTable.trim(); @@ -1992,4 +1997,42 @@ public class QTestUtil { (command != null ? " running " + command : "") + (debugHint != null ? debugHint : "")); } + + public static void addTestsToSuiteFromQfileNames( + String qFileNamesFile, + Set qFilesToExecute, + TestSuite suite, + Object setup, + SuiteAddTestFunctor suiteAddTestCallback) { + try { + File qFileNames = new File(qFileNamesFile); + FileReader fr = new FileReader(qFileNames.getCanonicalFile()); + BufferedReader br = new BufferedReader(fr); + String fName = null; + + while ((fName = br.readLine()) != null) { + if (fName.isEmpty() || fName.trim().equals("")) { + continue; + } + + int eIdx = fName.indexOf('.'); + + if (eIdx == -1) { + continue; + } + + String tName = fName.substring(0, eIdx); + + if (qFilesToExecute.isEmpty() || qFilesToExecute.contains(fName)) { + suiteAddTestCallback.addTestToSuite(suite, setup, tName); + } + } + br.close(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e)); + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestCliDriver.vm ---------------------------------------------------------------------- diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm index 8827f12..ae449c5 100644 --- a/ql/src/test/templates/TestCliDriver.vm +++ b/ql/src/test/templates/TestCliDriver.vm @@ -33,6 +33,12 @@ public class $className extends TestCase { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; + public static class TestCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCliDriver_"+tName)); + } + } + static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); @@ -108,15 +114,11 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCliDriver_$tname")); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, null, new TestCliDriverAddTestFromQFiles()); suite.addTest(new $className("testCliDriver_shutdown")); return suite; } http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestCompareCliDriver.vm ---------------------------------------------------------------------- diff --git a/ql/src/test/templates/TestCompareCliDriver.vm b/ql/src/test/templates/TestCompareCliDriver.vm index 6b367d6..4712708 100644 --- a/ql/src/test/templates/TestCompareCliDriver.vm +++ b/ql/src/test/templates/TestCompareCliDriver.vm @@ -34,6 +34,12 @@ public class $className extends TestCase { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; + public static class TestCompareCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCompareCliDriver_"+tName)); + } + } + static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); @@ -109,18 +115,11 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); - - - -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if (qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCompareCliDriver_$tname")); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, null, new TestCompareCliDriverAddTestFromQFiles()); suite.addTest(new $className("testCompareCliDriver_shutdown")); return suite; } http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestNegativeCliDriver.vm ---------------------------------------------------------------------- diff --git a/ql/src/test/templates/TestNegativeCliDriver.vm b/ql/src/test/templates/TestNegativeCliDriver.vm index 742044a..5f8ee8e 100644 --- a/ql/src/test/templates/TestNegativeCliDriver.vm +++ b/ql/src/test/templates/TestNegativeCliDriver.vm @@ -33,6 +33,12 @@ public class $className extends TestCase { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; + public static class TestNegativeCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testNegativeCliDriver_"+tName)); + } + } + static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); String initScript = "$initScript"; @@ -95,15 +101,11 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testNegativeCliDriver_$tname")); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, null, new TestNegativeCliDriverAddTestFromQFiles()); suite.addTest(new $className("testNegativeCliDriver_shutdown")); return suite; } http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestParseNegative.vm ---------------------------------------------------------------------- diff --git a/ql/src/test/templates/TestParseNegative.vm b/ql/src/test/templates/TestParseNegative.vm index 33b238e..c5e7bdf 100755 --- a/ql/src/test/templates/TestParseNegative.vm +++ b/ql/src/test/templates/TestParseNegative.vm @@ -32,6 +32,12 @@ public class $className extends TestCase { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; + + public static class TestParseNegativeAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testParseNegative_"+tName)); + } + } static { @@ -88,16 +94,11 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testParseNegative_$tname")); - } -#end + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, null, new TestParseNegativeAddTestFromQFiles()); suite.addTest(new $className("testParseNegative_shutdown")); return suite; }