hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject hive git commit: HIVE-15616 : Improve contents of qfile test output
Date Fri, 07 Apr 2017 01:16:11 GMT
Repository: hive
Updated Branches:
  refs/heads/master bc248642a -> a01a6a348


HIVE-15616 : Improve contents of qfile test output

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a01a6a34
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a01a6a34
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a01a6a34

Branch: refs/heads/master
Commit: a01a6a348825152c8f5495017e41016fc2ce4ac4
Parents: bc24864
Author: Barna Zsombor Klara <zsombor.klara@cloudera.com>
Authored: Fri Jan 13 06:44:00 2017 -0800
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Thu Apr 6 18:15:14 2017 -0700

----------------------------------------------------------------------
 .../hadoop/hive/ql/TestLocationQueries.java     |   8 +-
 .../control/AbstractCoreBlobstoreCliDriver.java |  11 +-
 .../hive/cli/control/CoreAccumuloCliDriver.java |  10 +-
 .../hadoop/hive/cli/control/CoreCliDriver.java  |  12 +-
 .../hive/cli/control/CoreCompareCliDriver.java  |  12 +-
 .../hive/cli/control/CoreHBaseCliDriver.java    |   9 +-
 .../cli/control/CoreHBaseNegativeCliDriver.java |   9 +-
 .../hive/cli/control/CoreNegativeCliDriver.java |  12 +-
 .../hive/cli/control/CorePerfCliDriver.java     |  12 +-
 .../hadoop/hive/ql/QTestProcessExecResult.java  |   6 +-
 .../org/apache/hadoop/hive/ql/QTestUtil.java    | 146 +++++++++++--------
 .../hadoop/hive/ql/parse/CoreParseNegative.java |  19 ++-
 12 files changed, 157 insertions(+), 109 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index 0688846..c17ca10 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -53,10 +53,11 @@ public class TestLocationQueries extends BaseTestQueries {
      * @return non-zero if it failed
      */
     @Override
-    public int checkCliDriverResults(String tname) throws Exception {
+    public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
       File logFile = new File(logDir, tname + ".out");
 
       int failedCount = 0;
+      StringBuilder fileNames = new StringBuilder("Files failing the location check:");
       FileReader fr = new FileReader(logFile);
       BufferedReader in = new BufferedReader(fr);
       try {
@@ -69,19 +70,20 @@ public class TestLocationQueries extends BaseTestQueries {
             File f = new File(m.group(1));
             if (!f.getName().equals(locationSubdir)) {
               failedCount++;
+              fileNames.append(f.getName()).append("\r\n");
             }
             locationCount++;
           }
         }
         // we always have to find at least one location, otw the test is useless
         if (locationCount == 0) {
-          return Integer.MAX_VALUE;
+          return QTestProcessExecResult.create(Integer.MAX_VALUE, "0 locations tested");
         }
       } finally {
         in.close();
       }
 
-      return failedCount;
+      return QTestProcessExecResult.create(failedCount, fileNames.toString());
     }
 
     public CheckResults(String outDir, String logDir, MiniClusterType miniMr,

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 9c97c31..02abe53 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveVariableSource;
 import org.apache.hadoop.hive.conf.VariableSubstitution;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
@@ -139,12 +140,14 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter
{
       if ((ecode == 0) ^ expectSuccess) {
         qt.failed(ecode, fname, debugHint);
       }
-      ecode = qt.checkCliDriverResults(fname);
-      if (ecode != 0) {
-        qt.failedDiff(ecode, fname, debugHint);
+      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+      if (result.getReturnCode() != 0) {
+        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+            debugHint : "\r\n" + result.getCapturedOutput();
+        qt.failedDiff(result.getReturnCode(), fname, message);
       }
     }
-    catch (Throwable e) {
+    catch (Exception e) {
       qt.failed(e, fname, debugHint);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
index 3e4b373..73e5632 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
@@ -18,8 +18,10 @@
 package org.apache.hadoop.hive.cli.control;
 
 import static org.junit.Assert.assertTrue;
+
 import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil;
 import org.apache.hadoop.hive.accumulo.AccumuloTestSetup;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -92,13 +94,13 @@ public class CoreAccumuloCliDriver extends CliAdapter {
         qt.failed(ecode, fname, null);
       }
 
-      ecode = qt.checkCliDriverResults(fname);
-      if (ecode != 0) {
-        qt.failedDiff(ecode, fname, null);
+      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+      if (result.getReturnCode() != 0) {
+        qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
       }
       qt.clearPostTestEffects();
 
-    } catch (Throwable e) {
+    } catch (Exception e) {
       qt.failed(e, fname, null);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index a735346..d59b650 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.fail;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Stopwatch;
+import com.google.common.base.Strings;
 import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.apache.hadoop.hive.util.ElapsedTimeLoggingWrapper;
@@ -175,13 +177,15 @@ public class CoreCliDriver extends CliAdapter {
         failed = true;
         qt.failed(ecode, fname, debugHint);
       }
-      ecode = qt.checkCliDriverResults(fname);
-      if (ecode != 0) {
+      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+      if (result.getReturnCode() != 0) {
         failed = true;
-        qt.failedDiff(ecode, fname, debugHint);
+        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+            debugHint : "\r\n" + result.getCapturedOutput();
+        qt.failedDiff(result.getReturnCode(), fname, message);
       }
     }
-    catch (Throwable e) {
+    catch (Exception e) {
       failed = true;
       qt.failed(e, fname, debugHint);
     } finally {

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
index 71a02bc..bff81dd 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
@@ -25,6 +25,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import com.google.common.base.Strings;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
@@ -143,12 +145,14 @@ public class CoreCompareCliDriver extends CliAdapter{
         }
       }
 
-      ecode = qt.checkCompareCliDriverResults(fname, outputs);
-      if (ecode != 0) {
-        qt.failedDiff(ecode, fname, debugHint);
+      QTestProcessExecResult result = qt.checkCompareCliDriverResults(fname, outputs);
+      if (result.getReturnCode() != 0) {
+        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+            debugHint : "\r\n" + result.getCapturedOutput();
+        qt.failedDiff(result.getReturnCode(), fname, message);
       }
     }
-    catch (Throwable e) {
+    catch (Exception e) {
       qt.failed(e, fname, debugHint);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
index 956a42d..aa0b071 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail;
 
 import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
 import org.apache.hadoop.hive.hbase.HBaseTestSetup;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -120,12 +121,12 @@ public class CoreHBaseCliDriver extends CliAdapter {
         qt.failed(ecode, fname, null);
       }
 
-      ecode = qt.checkCliDriverResults(fname);
-      if (ecode != 0) {
-        qt.failedDiff(ecode, fname, null);
+      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+      if (result.getReturnCode() != 0) {
+        qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
       }
 
-    } catch (Throwable e) {
+    } catch (Exception e) {
       qt.failed(e, fname, null);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
index 6225180..8320a80 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail;
 
 import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
 import org.apache.hadoop.hive.hbase.HBaseTestSetup;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -101,13 +102,13 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
         qt.failed(fname, null);
       }
 
-      ecode = qt.checkCliDriverResults(fname);
-      if (ecode != 0) {
-        qt.failedDiff(ecode, fname, null);
+      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+      if (result.getReturnCode() != 0) {
+        qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
       }
       qt.clearPostTestEffects();
 
-    } catch (Throwable e) {
+    } catch (Exception e) {
       qt.failed(e, fname, null);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 65b2ce7..438a61e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.cli.control;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import com.google.common.base.Strings;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
@@ -123,12 +125,14 @@ public class CoreNegativeCliDriver extends CliAdapter{
         qt.failed(fname, debugHint);
       }
 
-      ecode = qt.checkCliDriverResults(fname);
-      if (ecode != 0) {
-        qt.failedDiff(ecode, fname, debugHint);
+      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+      if (result.getReturnCode() != 0) {
+        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+            debugHint : "\r\n" + result.getCapturedOutput();
+        qt.failedDiff(result.getReturnCode(), fname, message);
       }
     }
-    catch (Throwable e) {
+    catch (Exception e) {
       qt.failed(e, fname, debugHint);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index 8620cde..26b9ce1 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -22,6 +22,8 @@ package org.apache.hadoop.hive.cli.control;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import com.google.common.base.Strings;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
@@ -121,11 +123,13 @@ public class CorePerfCliDriver extends CliAdapter{
       if (ecode != 0) {
         qt.failed(ecode, fname, debugHint);
       }
-      ecode = qt.checkCliDriverResults(fname);
-      if (ecode != 0) {
-        qt.failedDiff(ecode, fname, debugHint);
+      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+      if (result.getReturnCode() != 0) {
+        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+            debugHint : "\r\n" + result.getCapturedOutput();
+        qt.failedDiff(result.getReturnCode(), fname, message);
       }
-    } catch (Throwable e) {
+    } catch (Exception e) {
       qt.failed(e, fname, debugHint);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
index 6acc7a0..75cce14 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
@@ -18,12 +18,8 @@
 
 package org.apache.hadoop.hive.ql;
 
-
-import com.google.common.base.Strings;
-import org.apache.commons.lang3.StringUtils;
-
 /**
- * Standard output and return code of a process executed during the qtests
+ * Standard output and return code of a process executed during the qtests.
  */
 public class QTestProcessExecResult {
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 2abf252..c820dc7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -39,6 +39,7 @@ import java.io.PrintStream;
 import java.io.Serializable;
 import java.io.StringWriter;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
@@ -68,9 +69,10 @@ import com.google.common.base.Throwables;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.ByteArrayOutputStream;
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileStatus;
@@ -1468,10 +1470,6 @@ public class QTestUtil {
     // Create an instance of hive in order to create the tables
     testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
     db = Hive.get(conf);
-    // Create dest4 to replace dest4_sequencefile
-    LinkedList<String> cols = new LinkedList<String>();
-    cols.add("key");
-    cols.add("value");
 
     // Move all data from dest4_sequencefile to dest4
     drv
@@ -1482,7 +1480,7 @@ public class QTestUtil {
         true, true);
   }
 
-  public int checkNegativeResults(String tname, Exception e) throws Exception {
+  public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception
{
 
     String outFileExtension = getOutFileExtension(tname);
 
@@ -1505,16 +1503,17 @@ public class QTestUtil {
     outfd.write(e.getMessage());
     outfd.close();
 
-    int exitVal = executeDiffCommand(outf.getPath(), expf, false,
+    QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false,
                                      qSortSet.contains(qf.getName()));
-    if (exitVal != 0 && overWrite) {
-      exitVal = overwriteResults(outf.getPath(), expf);
+    if (overWrite) {
+      overwriteResults(outf.getPath(), expf);
+      return QTestProcessExecResult.createWithoutOutput(0);
     }
 
-    return exitVal;
+    return result;
   }
 
-  public int checkParseResults(String tname, ASTNode tree) throws Exception {
+  public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception
{
 
     if (tree != null) {
       String outFileExtension = getOutFileExtension(tname);
@@ -1530,10 +1529,11 @@ public class QTestUtil {
       outfd.write(tree.toStringTree());
       outfd.close();
 
-      int exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
+      QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
 
-      if (exitVal != 0 && overWrite) {
-        exitVal = overwriteResults(outf.getPath(), expf);
+      if (overWrite) {
+        overwriteResults(outf.getPath(), expf);
+        return QTestProcessExecResult.createWithoutOutput(0);
       }
 
       return exitVal;
@@ -1714,7 +1714,7 @@ public class QTestUtil {
     patternsWithMaskComments.add(toPatternPair(patternStr, maskComment));
   }
 
-  public int checkCliDriverResults(String tname) throws Exception {
+  public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
     assert(qMap.containsKey(tname));
 
     String outFileExtension = getOutFileExtension(tname);
@@ -1723,69 +1723,71 @@ public class QTestUtil {
     File f = new File(logDir, tname + outFileExtension);
 
     maskPatterns(planMask, f.getPath());
-    int exitVal = executeDiffCommand(f.getPath(),
+    QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(),
                                      outFileName, false,
                                      qSortSet.contains(tname));
 
-    if (exitVal != 0 && overWrite) {
-      exitVal = overwriteResults(f.getPath(), outFileName);
+    if (overWrite) {
+      overwriteResults(f.getPath(), outFileName);
+      return QTestProcessExecResult.createWithoutOutput(0);
     }
 
     return exitVal;
   }
 
 
-  public int checkCompareCliDriverResults(String tname, List<String> outputs) throws
Exception {
+  public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String>
outputs)
+      throws Exception {
     assert outputs.size() > 1;
     maskPatterns(planMask, outputs.get(0));
     for (int i = 1; i < outputs.size(); ++i) {
       maskPatterns(planMask, outputs.get(i));
-      int ecode = executeDiffCommand(
+      QTestProcessExecResult result = executeDiffCommand(
           outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
-      if (ecode != 0) {
+      if (result.getReturnCode() != 0) {
         System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i));
-        return ecode;
+        return result;
       }
     }
-    return 0;
+    return QTestProcessExecResult.createWithoutOutput(0);
   }
 
-  private static int overwriteResults(String inFileName, String outFileName) throws Exception
{
+  private static void overwriteResults(String inFileName, String outFileName) throws Exception
{
     // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING)
     // once Hive uses JAVA 7.
     System.out.println("Overwriting results " + inFileName + " to " + outFileName);
-    return executeCmd(new String[] {
+    int result = executeCmd(new String[]{
         "cp",
         getQuotedString(inFileName),
         getQuotedString(outFileName)
-      });
+    }).getReturnCode();
+    if (result != 0)
+      throw new IllegalStateException("Unexpected error while overwriting " +
+          inFileName + " with " + outFileName);
   }
 
-  private static int executeDiffCommand(String inFileName,
+  private static QTestProcessExecResult executeDiffCommand(String inFileName,
       String outFileName,
       boolean ignoreWhiteSpace,
       boolean sortResults
       ) throws Exception {
 
-    int result = 0;
+    QTestProcessExecResult result;
 
     if (sortResults) {
       // sort will try to open the output file in write mode on windows. We need to
       // close it first.
       SessionState ss = SessionState.get();
       if (ss != null && ss.out != null && ss.out != System.out) {
-  ss.out.close();
+        ss.out.close();
       }
 
       String inSorted = inFileName + SORT_SUFFIX;
       String outSorted = outFileName + SORT_SUFFIX;
 
-      result = sortFiles(inFileName, inSorted);
-      result |= sortFiles(outFileName, outSorted);
-      if (result != 0) {
-        System.err.println("ERROR: Could not sort files before comparing");
-        return result;
-      }
+      sortFiles(inFileName, inSorted);
+      sortFiles(outFileName, outSorted);
+
       inFileName = inSorted;
       outFileName = outSorted;
     }
@@ -1815,40 +1817,47 @@ public class QTestUtil {
     return result;
   }
 
-  private static int sortFiles(String in, String out) throws Exception {
-    return executeCmd(new String[] {
+  private static void sortFiles(String in, String out) throws Exception {
+    int result = executeCmd(new String[]{
         "sort",
         getQuotedString(in),
-      }, out, null);
+    }, out, null).getReturnCode();
+    if (result != 0)
+      throw new IllegalStateException("Unexpected error while sorting " + in);
   }
 
-  private static int executeCmd(Collection<String> args) throws Exception {
+  private static QTestProcessExecResult executeCmd(Collection<String> args) throws
Exception {
     return executeCmd(args, null, null);
   }
 
-  private static int executeCmd(String[] args) throws Exception {
+  private static QTestProcessExecResult executeCmd(String[] args) throws Exception {
     return executeCmd(args, null, null);
   }
 
-  private static int executeCmd(Collection<String> args, String outFile, String errFile)
throws Exception {
+  private static QTestProcessExecResult executeCmd(Collection<String> args, String
outFile,
+                                            String errFile) throws Exception {
     String[] cmdArray = args.toArray(new String[args.size()]);
     return executeCmd(cmdArray, outFile, errFile);
   }
 
-  private static int executeCmd(String[] args, String outFile, String errFile) throws Exception
{
+  private static QTestProcessExecResult executeCmd(String[] args, String outFile,
+                                            String errFile) throws Exception {
     System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
 
     PrintStream out = outFile == null ?
       SessionState.getConsole().getChildOutStream() :
-      new PrintStream(new FileOutputStream(outFile), true);
+      new PrintStream(new FileOutputStream(outFile), true, "UTF-8");
     PrintStream err = errFile == null ?
       SessionState.getConsole().getChildErrStream() :
-      new PrintStream(new FileOutputStream(errFile), true);
+      new PrintStream(new FileOutputStream(errFile), true, "UTF-8");
 
     Process executor = Runtime.getRuntime().exec(args);
 
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    PrintStream str = new PrintStream(bos, true, "UTF-8");
+
     StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err);
-    StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out);
+    StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str);
 
     outPrinter.start();
     errPrinter.start();
@@ -1866,7 +1875,8 @@ public class QTestUtil {
       err.close();
     }
 
-    return result;
+    return QTestProcessExecResult.
+        create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8));
   }
 
   private static String getQuotedString(String str){
@@ -2042,11 +2052,18 @@ public class QTestUtil {
       qt[i].clearTestSideEffects();
       qt[i].cliInit(qfiles[i].getName(), false);
       qt[i].executeClient(qfiles[i].getName());
-      int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
-      if (ecode != 0) {
+      QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+      if (result.getReturnCode() != 0) {
         failed = true;
-        System.err.println("Test " + qfiles[i].getName()
-            + " results check failed with error code " + ecode);
+        StringBuilder builder = new StringBuilder();
+        builder.append("Test ")
+            .append(qfiles[i].getName())
+            .append(" results check failed with error code ")
+            .append(result.getReturnCode());
+        if (Strings.isNotEmpty(result.getCapturedOutput())) {
+          builder.append(" and diff value ").append(result.getCapturedOutput());
+        }
+        System.err.println(builder.toString());
         outputTestFailureHelpMessage();
       }
       qt[i].clearPostTestEffects();
@@ -2093,11 +2110,18 @@ public class QTestUtil {
 
     for (int i = 0; i < qfiles.length; i++) {
       qtThread[i].join();
-      int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
-      if (ecode != 0) {
+      QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+      if (result.getReturnCode() != 0) {
         failed = true;
-        System.err.println("Test " + qfiles[i].getName()
-            + " results check failed with error code " + ecode);
+        StringBuilder builder = new StringBuilder();
+        builder.append("Test ")
+            .append(qfiles[i].getName())
+            .append(" results check failed with error code ")
+            .append(result.getReturnCode());
+        if (Strings.isNotEmpty(result.getCapturedOutput())) {
+          builder.append(" and diff value ").append(result.getCapturedOutput());
+        }
+        System.err.println(builder.toString());
         outputTestFailureHelpMessage();
       }
     }
@@ -2195,16 +2219,15 @@ public class QTestUtil {
 
   public void failedDiff(int ecode, String fname, String debugHint) {
     String message =
-        "Client Execution results failed with error code = " + ecode + " while executing
fname=" +
+        "Client Execution succeeded but contained differences " +
+            "(error code = " + ecode + ") after executing " +
             fname + (debugHint != null ? (" " + debugHint) : "");
     LOG.error(message);
     Assert.fail(message);
   }
 
-  public void failed(Throwable e, String fname, String debugHint) {
+  public void failed(Exception e, String fname, String debugHint) {
     String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
-    System.err.println("Exception: " + e.getMessage());
-    e.printStackTrace();
     System.err.println("Failed query: " + fname);
     System.err.flush();
     Assert.fail("Unexpected exception " +
@@ -2244,9 +2267,6 @@ public class QTestUtil {
       }
       br.close();
     } catch (Exception e) {
-      System.err.println("Exception: " + e.getMessage());
-      e.printStackTrace();
-      System.err.flush();
       Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
     }
   }
@@ -2270,7 +2290,9 @@ public class QTestUtil {
       String mdbPath =   AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/";
 
       // Setup the table column stats
-      BufferedReader br = new BufferedReader(new FileReader(new File(AbstractCliConfig.HIVE_ROOT
+ "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
+      BufferedReader br = new BufferedReader(
+          new FileReader(
+              new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
       String command;
 
       s.execute("DROP TABLE APP.TABLE_PARAMS");

http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index 8dba0bb..31f69a3 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -21,9 +21,12 @@ import static org.junit.Assert.fail;
 
 import java.io.Serializable;
 import java.util.List;
+
+import com.google.common.base.Strings;
 import org.apache.hadoop.hive.cli.control.AbstractCliConfig;
 import org.apache.hadoop.hive.cli.control.CliAdapter;
 import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.apache.hadoop.hive.ql.exec.Task;
@@ -106,18 +109,20 @@ public class CoreParseNegative extends CliAdapter{
       fail("Unexpected success for query: " + fname + debugHint);
     }
     catch (ParseException pe) {
-      int ecode = qt.checkNegativeResults(fname, pe);
-      if (ecode != 0) {
-        qt.failed(ecode, fname, debugHint);
+      QTestProcessExecResult result = qt.checkNegativeResults(fname, pe);
+      if (result.getReturnCode() != 0) {
+        qt.failed(result.getReturnCode(), fname, result.getCapturedOutput() + "\r\n" + debugHint);
       }
     }
     catch (SemanticException se) {
-      int ecode = qt.checkNegativeResults(fname, se);
-      if (ecode != 0) {
-        qt.failedDiff(ecode, fname, debugHint);
+      QTestProcessExecResult result = qt.checkNegativeResults(fname, se);
+      if (result.getReturnCode() != 0) {
+        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+            debugHint : "\r\n" + result.getCapturedOutput();
+        qt.failedDiff(result.getReturnCode(), fname, message);
       }
     }
-    catch (Throwable e) {
+    catch (Exception e) {
       qt.failed(e, fname, debugHint);
     }
 


Mime
View raw message