hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [33/51] [partial] hive git commit: Revert "HIVE-14671 : merge master into hive-14535 (Wei Zheng)"
Date Mon, 08 May 2017 20:43:23 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
index 1a8337f..e344e0f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
@@ -100,6 +100,22 @@ public class TestOperationLoggingLayout {
       String row = iter.next()[0].toString();
       Assert.assertEquals(true, row.matches("^.*(FATAL|ERROR|WARN|INFO|DEBUG|TRACE).*$"));
     }
+
+    String queryString = "set hive.server2.logging.operation.level=verbose";
+    client.executeStatement(sessionHandle, queryString, null);
+    operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null);
+    // just check for first few lines, some log lines are multi-line strings which can break format
+    // checks below
+    rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 10,
+        FetchType.LOG);
+    iter = rowSetLog.iterator();
+    // verbose pattern is "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"
+    while (iter.hasNext()) {
+      String row = iter.next()[0].toString();
+      // just check if the log line starts with date
+      Assert.assertEquals(true,
+          row.matches("^\\d{2}[/](0[1-9]|1[012])[/](0[1-9]|[12][0-9]|3[01]).*$"));
+    }
   }
 
   private SessionHandle setupSession() throws Exception {

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index 400075b..e5b54bf 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>org.apache.hive</groupId>
     <artifactId>hive</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/qtest-accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest-accumulo/pom.xml b/itests/qtest-accumulo/pom.xml
index b7ce283..31cee36 100644
--- a/itests/qtest-accumulo/pom.xml
+++ b/itests/qtest-accumulo/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.hive</groupId>
     <artifactId>hive-it</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/qtest-spark/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml
index a506f7f..f301504 100644
--- a/itests/qtest-spark/pom.xml
+++ b/itests/qtest-spark/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.hive</groupId>
     <artifactId>hive-it</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 02664f3..1c3b601 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.hive</groupId>
     <artifactId>hive-it</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
index 4dd17c1e..24eeb9d 100644
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
@@ -22,15 +22,15 @@ import java.util.List;
 
 import org.apache.hadoop.hive.cli.control.CliAdapter;
 import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.apache.hive.beeline.Parallelized;
 import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
-@RunWith(Parallelized.class)
+@RunWith(Parameterized.class)
 public class TestBeeLineDriver {
 
   static CliAdapter adapter = new CliConfigs.BeeLineConfig().getCliAdapter();

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index df947bb..dc02fd4 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -71,7 +71,6 @@ minillap.shared.query.files=insert_into1.q,\
   orc_merge3.q,\
   orc_merge4.q,\
   orc_merge_diff_fs.q,\
-  parallel_colstats.q,\
   unionDistinct_1.q,\
   union_type_chk.q,\
   cte_2.q,\
@@ -353,8 +352,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
   vector_reduce_groupby_decimal.q,\
   vector_string_concat.q,\
   vector_struct_in.q,\
-  vector_udf_character_length.q,\
-  vector_udf_octet_length.q,\
   vector_varchar_4.q,\
   vector_varchar_mapjoin1.q,\
   vector_varchar_simple.q,\
@@ -408,7 +405,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
 minillap.query.files=acid_bucket_pruning.q,\
   bucket5.q,\
   bucket6.q,\
-  dynamic_semijoin_user_level.q,\
   except_distinct.q,\
   explainuser_2.q,\
   empty_dir_in_table.q,\
@@ -503,7 +499,6 @@ minillaplocal.query.files=acid_globallimit.q,\
   groupby2.q,\
   hybridgrace_hashjoin_1.q,\
   hybridgrace_hashjoin_2.q,\
-  is_distinct_from.q,\
   infer_bucket_sort_bucketed_table.q,\
   input16_cc.q,\
   insert_dir_distcp.q,\
@@ -515,14 +510,12 @@ minillaplocal.query.files=acid_globallimit.q,\
   join_max_hashtable.q,\
   join_nulls.q,\
   join_nullsafe.q,\
-  join_is_not_distinct_from.q,\
   leftsemijoin_mr.q,\
   limit_join_transpose.q,\
   lineage2.q,\
   lineage3.q,\
   list_bucket_dml_10.q,\
   llap_partitioned.q,\
-  llap_vector_nohybridgrace.q,\
   load_dyn_part5.q,\
   lvj_mapjoin.q,\
   mapjoin_decimal.q,\
@@ -574,7 +567,6 @@ minillaplocal.query.files=acid_globallimit.q,\
   schema_evol_text_vecrow_table.q,\
   selectDistinctStar.q,\
   semijoin.q,\
-  semijoin_hint.q,\
   smb_cache.q,\
   special_character_in_tabnames_1.q,\
   sqlmerge.q,\
@@ -619,26 +611,11 @@ minillaplocal.query.files=acid_globallimit.q,\
   vector_auto_smb_mapjoin_14.q,\
   vector_decimal_2.q,\
   vector_decimal_udf.q,\
-  vector_groupby_cube1.q,\
-  vector_groupby_grouping_id1.q,\
-  vector_groupby_grouping_id2.q,\
-  vector_groupby_grouping_id3.q,\
-  vector_groupby_grouping_sets1.q,\
-  vector_groupby_grouping_sets2.q,\
-  vector_groupby_grouping_sets3.q,\
-  vector_groupby_grouping_sets4.q,\
-  vector_groupby_grouping_sets5.q,\
-  vector_groupby_grouping_sets6.q,\
-  vector_groupby_grouping_sets_grouping.q,\
-  vector_groupby_grouping_sets_limit.q,\
-  vector_groupby_grouping_window.q,\
-  vector_groupby_rollup1.q,\
   vector_join30.q,\
   vector_join_filters.q,\
   vector_leftsemi_mapjoin.q,\
   vector_number_compare_projection.q,\
   vector_partitioned_date_time.q,\
-  vector_ptf_part_simple.q,\
   vector_udf1.q,\
   vectorization_short_regress.q,\
   vectorized_dynamic_partition_pruning.q,\
@@ -748,22 +725,10 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
   encryption_drop_view.q \
   encryption_drop_partition.q \
   encryption_with_trash.q \
-  encryption_ctas.q \
-  encryption_auto_purge_tables.q \
-  encryption_drop_table_in_encrypted_db.q
+  encryption_ctas.q
 
 beeline.positive.include=drop_with_concurrency.q,\
-  escape_comments.q,\
-  smb_mapjoin_1.q,\
-  smb_mapjoin_10.q,\
-  smb_mapjoin_11.q,\
-  smb_mapjoin_12.q,\
-  smb_mapjoin_13.q,\
-  smb_mapjoin_16.q,\
-  smb_mapjoin_2.q,\
-  smb_mapjoin_3.q,\
-  smb_mapjoin_7.q,\
-  select_dummy_source.q
+  escape_comments.q
 
 minimr.query.negative.files=cluster_tasklog_retrieval.q,\
   file_with_header_footer_negative.q,\
@@ -1371,7 +1336,6 @@ spark.query.files=add_part_multiple.q, \
 spark.only.query.files=spark_combine_equivalent_work.q,\
   spark_dynamic_partition_pruning.q,\
   spark_dynamic_partition_pruning_2.q,\
-  spark_explainuser_1.q,\
   spark_vectorized_dynamic_partition_pruning.q,\
   spark_use_file_size_for_mapjoin.q,\
   spark_use_op_stats.q
@@ -1446,6 +1410,4 @@ spark.query.negative.files=groupby2_map_skew_multi_distinct.q,\
   groupby2_multi_distinct.q,\
   groupby3_map_skew_multi_distinct.q,\
   groupby3_multi_distinct.q,\
-  groupby_grouping_sets7.q,\
-  spark_job_max_tasks.q
-
+  groupby_grouping_sets7.q

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/test-serde/pom.xml
----------------------------------------------------------------------
diff --git a/itests/test-serde/pom.xml b/itests/test-serde/pom.xml
index bf5f5d2..df0ce8a 100644
--- a/itests/test-serde/pom.xml
+++ b/itests/test-serde/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>org.apache.hive</groupId>
     <artifactId>hive-it</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/pom.xml
----------------------------------------------------------------------
diff --git a/itests/util/pom.xml b/itests/util/pom.xml
index a44a55c..3740af8 100644
--- a/itests/util/pom.xml
+++ b/itests/util/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>org.apache.hive</groupId>
     <artifactId>hive-it</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 02abe53..9c97c31 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveVariableSource;
 import org.apache.hadoop.hive.conf.VariableSubstitution;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
@@ -140,14 +139,12 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
       if ((ecode == 0) ^ expectSuccess) {
         qt.failed(ecode, fname, debugHint);
       }
-      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
-      if (result.getReturnCode() != 0) {
-        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
-            debugHint : "\r\n" + result.getCapturedOutput();
-        qt.failedDiff(result.getReturnCode(), fname, message);
+      ecode = qt.checkCliDriverResults(fname);
+      if (ecode != 0) {
+        qt.failedDiff(ecode, fname, debugHint);
       }
     }
-    catch (Exception e) {
+    catch (Throwable e) {
       qt.failed(e, fname, debugHint);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 1457db0..67064b8 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -287,7 +287,6 @@ public class CliConfigs {
 
         excludesFrom(testConfigProps, "minimr.query.negative.files");
         excludeQuery("authorization_uri_import.q");
-        excludeQuery("spark_job_max_tasks.q");
 
         setResultsDir("ql/src/test/results/clientnegative");
         setLogDir("itests/qtest/target/qfile-results/clientnegative");

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
index 73e5632..3e4b373 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
@@ -18,10 +18,8 @@
 package org.apache.hadoop.hive.cli.control;
 
 import static org.junit.Assert.assertTrue;
-
 import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil;
 import org.apache.hadoop.hive.accumulo.AccumuloTestSetup;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -94,13 +92,13 @@ public class CoreAccumuloCliDriver extends CliAdapter {
         qt.failed(ecode, fname, null);
       }
 
-      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
-      if (result.getReturnCode() != 0) {
-        qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
+      ecode = qt.checkCliDriverResults(fname);
+      if (ecode != 0) {
+        qt.failedDiff(ecode, fname, null);
       }
       qt.clearPostTestEffects();
 
-    } catch (Exception e) {
+    } catch (Throwable e) {
       qt.failed(e, fname, null);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
index 8c7057c..acc02eb 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
@@ -17,23 +17,21 @@
  */
 package org.apache.hadoop.hive.cli.control;
 
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import com.google.common.base.Strings;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.hooks.PreExecutePrinter;
-import org.apache.hive.beeline.QFile;
-import org.apache.hive.beeline.QFile.QFileBuilder;
-import org.apache.hive.beeline.QFileBeeLineClient;
-import org.apache.hive.beeline.QFileBeeLineClient.QFileClientBuilder;
+import org.apache.hive.beeline.qfile.QFile;
+import org.apache.hive.beeline.qfile.QFile.QFileBuilder;
+import org.apache.hive.beeline.qfile.QFileBeeLineClient;
+import org.apache.hive.beeline.qfile.QFileBeeLineClient.QFileClientBuilder;
 import org.apache.hive.jdbc.miniHS2.MiniHS2;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.File;
 import java.io.IOException;
-import java.sql.SQLException;
 import java.util.HashMap;
 
 public class CoreBeeLineDriver extends CliAdapter {
@@ -46,7 +44,6 @@ public class CoreBeeLineDriver extends CliAdapter {
   private final File testDataDirectory;
   private final File testScriptDirectory;
   private boolean overwrite = false;
-  private boolean rewriteSourceTables = true;
   private MiniHS2 miniHS2;
   private QFileClientBuilder clientBuilder;
   private QFileBuilder fileBuilder;
@@ -71,10 +68,6 @@ public class CoreBeeLineDriver extends CliAdapter {
     if (testOutputOverwrite != null && "true".equalsIgnoreCase(testOutputOverwrite)) {
       overwrite = true;
     }
-    String testRewriteSourceTables = System.getProperty("test.rewrite.source.tables");
-    if (testRewriteSourceTables != null && "false".equalsIgnoreCase(testRewriteSourceTables)) {
-      rewriteSourceTables = false;
-    }
 
     HiveConf hiveConf = new HiveConf();
     // We do not need Zookeeper at the moment
@@ -99,17 +92,19 @@ public class CoreBeeLineDriver extends CliAdapter {
         .setPassword("password");
 
     fileBuilder = new QFileBuilder()
+        .setHiveRootDirectory(hiveRootDirectory)
         .setLogDirectory(logDirectory)
         .setQueryDirectory(queryDirectory)
         .setResultsDirectory(resultsDirectory)
-        .setRewriteSourceTables(rewriteSourceTables);
+        .setScratchDirectoryString(hiveConf.getVar(HiveConf.ConfVars.SCRATCHDIR))
+        .setWarehouseDirectoryString(hiveConf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE));
 
     runInfraScript(initScript, new File(logDirectory, "init.beeline"),
         new File(logDirectory, "init.raw"));
   }
 
   protected void runInfraScript(File script, File beeLineOutput, File log)
-      throws IOException, SQLException {
+      throws IOException {
     try (QFileBeeLineClient beeLineClient = clientBuilder.getClient(beeLineOutput)) {
       beeLineClient.execute(
           new String[]{
@@ -119,9 +114,6 @@ public class CoreBeeLineDriver extends CliAdapter {
             "!run " + script,
           },
           log);
-    } catch (Exception e) {
-      throw new SQLException("Error running infra script: " + script
-          + "\nCheck the following logs for details:\n - " + beeLineOutput + "\n - " + log, e);
     }
   }
 
@@ -142,41 +134,28 @@ public class CoreBeeLineDriver extends CliAdapter {
     try (QFileBeeLineClient beeLineClient = clientBuilder.getClient(qFile.getLogFile())) {
       long startTime = System.currentTimeMillis();
       System.err.println(">>> STARTED " + qFile.getName());
+      assertTrue("QFile execution failed, see logs for details", beeLineClient.execute(qFile));
 
-      beeLineClient.execute(qFile);
-
-      long queryEndTime = System.currentTimeMillis();
-      System.err.println(">>> EXECUTED " + qFile.getName() + ": " + (queryEndTime - startTime)
-          + "ms");
+      long endTime = System.currentTimeMillis();
+      System.err.println(">>> EXECUTED " + qFile.getName() + ":" + (endTime - startTime) / 1000
+          + "s");
 
       qFile.filterOutput();
       long filterEndTime = System.currentTimeMillis();
-      System.err.println(">>> FILTERED " + qFile.getName() + ": " + (filterEndTime - queryEndTime)
-          + "ms");
+      System.err.println(">>> FILTERED " + qFile.getName() + ":" + (filterEndTime - endTime) / 1000
+          + "s");
 
       if (!overwrite) {
-        QTestProcessExecResult result = qFile.compareResults();
-
-        long compareEndTime = System.currentTimeMillis();
-        System.err.println(">>> COMPARED " + qFile.getName() + ": "
-            + (compareEndTime - filterEndTime) + "ms");
-        if (result.getReturnCode() == 0) {
+        if (qFile.compareResults()) {
           System.err.println(">>> PASSED " + qFile.getName());
         } else {
           System.err.println(">>> FAILED " + qFile.getName());
-          String messageText = "Client result comparison failed with error code = "
-              + result.getReturnCode() + " while executing fname=" + qFile.getName() + "\n";
-          String messageBody = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
-              qFile.getDebugHint() : result.getCapturedOutput();
-          fail(messageText + messageBody);
+          fail("Failed diff");
         }
       } else {
         qFile.overwriteResults();
         System.err.println(">>> PASSED " + qFile.getName());
       }
-    } catch (Exception e) {
-      throw new Exception("Exception running or analyzing the results of the query file: " + qFile
-          + "\n" + qFile.getDebugHint(), e);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index d59b650..a735346 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -23,9 +23,7 @@ import static org.junit.Assert.fail;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Stopwatch;
-import com.google.common.base.Strings;
 import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.apache.hadoop.hive.util.ElapsedTimeLoggingWrapper;
@@ -177,15 +175,13 @@ public class CoreCliDriver extends CliAdapter {
         failed = true;
         qt.failed(ecode, fname, debugHint);
       }
-      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
-      if (result.getReturnCode() != 0) {
+      ecode = qt.checkCliDriverResults(fname);
+      if (ecode != 0) {
         failed = true;
-        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
-            debugHint : "\r\n" + result.getCapturedOutput();
-        qt.failedDiff(result.getReturnCode(), fname, message);
+        qt.failedDiff(ecode, fname, debugHint);
       }
     }
-    catch (Exception e) {
+    catch (Throwable e) {
       failed = true;
       qt.failed(e, fname, debugHint);
     } finally {

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
index bff81dd..71a02bc 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
@@ -25,8 +25,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.google.common.base.Strings;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
@@ -145,14 +143,12 @@ public class CoreCompareCliDriver extends CliAdapter{
         }
       }
 
-      QTestProcessExecResult result = qt.checkCompareCliDriverResults(fname, outputs);
-      if (result.getReturnCode() != 0) {
-        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
-            debugHint : "\r\n" + result.getCapturedOutput();
-        qt.failedDiff(result.getReturnCode(), fname, message);
+      ecode = qt.checkCompareCliDriverResults(fname, outputs);
+      if (ecode != 0) {
+        qt.failedDiff(ecode, fname, debugHint);
       }
     }
-    catch (Exception e) {
+    catch (Throwable e) {
       qt.failed(e, fname, debugHint);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
index aa0b071..956a42d 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.fail;
 
 import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
 import org.apache.hadoop.hive.hbase.HBaseTestSetup;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -121,12 +120,12 @@ public class CoreHBaseCliDriver extends CliAdapter {
         qt.failed(ecode, fname, null);
       }
 
-      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
-      if (result.getReturnCode() != 0) {
-        qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
+      ecode = qt.checkCliDriverResults(fname);
+      if (ecode != 0) {
+        qt.failedDiff(ecode, fname, null);
       }
 
-    } catch (Exception e) {
+    } catch (Throwable e) {
       qt.failed(e, fname, null);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
index 8320a80..6225180 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.fail;
 
 import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
 import org.apache.hadoop.hive.hbase.HBaseTestSetup;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -102,13 +101,13 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
         qt.failed(fname, null);
       }
 
-      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
-      if (result.getReturnCode() != 0) {
-        qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
+      ecode = qt.checkCliDriverResults(fname);
+      if (ecode != 0) {
+        qt.failedDiff(ecode, fname, null);
       }
       qt.clearPostTestEffects();
 
-    } catch (Exception e) {
+    } catch (Throwable e) {
       qt.failed(e, fname, null);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 438a61e..65b2ce7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hive.cli.control;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import com.google.common.base.Strings;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
@@ -125,14 +123,12 @@ public class CoreNegativeCliDriver extends CliAdapter{
         qt.failed(fname, debugHint);
       }
 
-      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
-      if (result.getReturnCode() != 0) {
-        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
-            debugHint : "\r\n" + result.getCapturedOutput();
-        qt.failedDiff(result.getReturnCode(), fname, message);
+      ecode = qt.checkCliDriverResults(fname);
+      if (ecode != 0) {
+        qt.failedDiff(ecode, fname, debugHint);
       }
     }
-    catch (Exception e) {
+    catch (Throwable e) {
       qt.failed(e, fname, debugHint);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index 26b9ce1..8620cde 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -22,8 +22,6 @@ package org.apache.hadoop.hive.cli.control;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import com.google.common.base.Strings;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
@@ -123,13 +121,11 @@ public class CorePerfCliDriver extends CliAdapter{
       if (ecode != 0) {
         qt.failed(ecode, fname, debugHint);
       }
-      QTestProcessExecResult result = qt.checkCliDriverResults(fname);
-      if (result.getReturnCode() != 0) {
-        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
-            debugHint : "\r\n" + result.getCapturedOutput();
-        qt.failedDiff(result.getReturnCode(), fname, message);
+      ecode = qt.checkCliDriverResults(fname);
+      if (ecode != 0) {
+        qt.failedDiff(ecode, fname, debugHint);
       }
-    } catch (Exception e) {
+    } catch (Throwable e) {
       qt.failed(e, fname, debugHint);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
index 75cce14..6acc7a0 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
@@ -18,8 +18,12 @@
 
 package org.apache.hadoop.hive.ql;
 
+
+import com.google.common.base.Strings;
+import org.apache.commons.lang3.StringUtils;
+
 /**
- * Standard output and return code of a process executed during the qtests.
+ * Standard output and return code of a process executed during the qtests
  */
 public class QTestProcessExecResult {
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index b897ffa..e32ee74 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -39,10 +39,6 @@ import java.io.PrintStream;
 import java.io.Serializable;
 import java.io.StringWriter;
 import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.StandardOpenOption;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
@@ -55,7 +51,6 @@ import java.util.Collection;
 import java.util.Comparator;
 import java.util.Deque;
 import java.util.EnumSet;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -64,22 +59,17 @@ import java.util.Properties;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Stream;
 
 import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
 
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.output.ByteArrayOutputStream;
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileStatus;
@@ -128,6 +118,7 @@ import org.apache.hadoop.hive.ql.processors.HiveCommand;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.util.Shell;
 import org.apache.hive.common.util.StreamPrinter;
 import org.apache.logging.log4j.util.Strings;
 import org.apache.tools.ant.BuildException;
@@ -176,7 +167,7 @@ public class QTestUtil {
   private final Set<String> qNoSessionReuseQuerySet;
   private final Set<String> qJavaVersionSpecificOutput;
   private static final String SORT_SUFFIX = ".sorted";
-  private final Set<String> srcTables;
+  private final HashSet<String> srcTables;
   private final Set<String> srcUDFs;
   private final MiniClusterType clusterType;
   private final FsType fsType;
@@ -209,7 +200,7 @@ public class QTestUtil {
   }
   private HBaseTestingUtility utility;
 
-  public static Set<String> getSrcTables() {
+  HashSet<String> getSrcTables() {
     HashSet<String> srcTables = new HashSet<String>();
     // FIXME: moved default value to here...for now
     // i think this features is never really used from the command line
@@ -1404,8 +1395,7 @@ public class QTestUtil {
 
         int rc = response.getResponseCode();
         if (rc != 0) {
-          SessionState.getConsole().printError(response.toString(), response.getException() != null ?
-                  Throwables.getStackTraceAsString(response.getException()) : "");
+          SessionState.get().out.println(response);
         }
 
         return rc;
@@ -1413,7 +1403,7 @@ public class QTestUtil {
         throw new RuntimeException("Could not get CommandProcessor for command: " + commandName);
       }
     } catch (Exception e) {
-      throw new RuntimeException("Could not execute test command", e);
+      throw new RuntimeException("Could not execute test command: " + e.getMessage());
     }
   }
 
@@ -1476,6 +1466,10 @@ public class QTestUtil {
     // Create an instance of hive in order to create the tables
     testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
     db = Hive.get(conf);
+    // Create dest4 to replace dest4_sequencefile
+    LinkedList<String> cols = new LinkedList<String>();
+    cols.add("key");
+    cols.add("value");
 
     // Move all data from dest4_sequencefile to dest4
     drv
@@ -1486,7 +1480,7 @@ public class QTestUtil {
         true, true);
   }
 
-  public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception {
+  public int checkNegativeResults(String tname, Exception e) throws Exception {
 
     String outFileExtension = getOutFileExtension(tname);
 
@@ -1509,17 +1503,16 @@ public class QTestUtil {
     outfd.write(e.getMessage());
     outfd.close();
 
-    QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false,
+    int exitVal = executeDiffCommand(outf.getPath(), expf, false,
                                      qSortSet.contains(qf.getName()));
-    if (overWrite) {
-      overwriteResults(outf.getPath(), expf);
-      return QTestProcessExecResult.createWithoutOutput(0);
+    if (exitVal != 0 && overWrite) {
+      exitVal = overwriteResults(outf.getPath(), expf);
     }
 
-    return result;
+    return exitVal;
   }
 
-  public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception {
+  public int checkParseResults(String tname, ASTNode tree) throws Exception {
 
     if (tree != null) {
       String outFileExtension = getOutFileExtension(tname);
@@ -1535,11 +1528,10 @@ public class QTestUtil {
       outfd.write(tree.toStringTree());
       outfd.close();
 
-      QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
+      int exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
 
-      if (overWrite) {
-        overwriteResults(outf.getPath(), expf);
-        return QTestProcessExecResult.createWithoutOutput(0);
+      if (exitVal != 0 && overWrite) {
+        exitVal = overwriteResults(outf.getPath(), expf);
       }
 
       return exitVal;
@@ -1708,7 +1700,7 @@ public class QTestUtil {
   });
 
   /* This list may be modified by specific cli drivers to mask strings that change on every test */
-  private final List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{
+  private List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{
     add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###"));
   }};
 
@@ -1720,7 +1712,7 @@ public class QTestUtil {
     patternsWithMaskComments.add(toPatternPair(patternStr, maskComment));
   }
 
-  public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
+  public int checkCliDriverResults(String tname) throws Exception {
     assert(qMap.containsKey(tname));
 
     String outFileExtension = getOutFileExtension(tname);
@@ -1729,71 +1721,69 @@ public class QTestUtil {
     File f = new File(logDir, tname + outFileExtension);
 
     maskPatterns(planMask, f.getPath());
-    QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(),
+    int exitVal = executeDiffCommand(f.getPath(),
                                      outFileName, false,
                                      qSortSet.contains(tname));
 
-    if (overWrite) {
-      overwriteResults(f.getPath(), outFileName);
-      return QTestProcessExecResult.createWithoutOutput(0);
+    if (exitVal != 0 && overWrite) {
+      exitVal = overwriteResults(f.getPath(), outFileName);
     }
 
     return exitVal;
   }
 
 
-  public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs)
-      throws Exception {
+  public int checkCompareCliDriverResults(String tname, List<String> outputs) throws Exception {
     assert outputs.size() > 1;
     maskPatterns(planMask, outputs.get(0));
     for (int i = 1; i < outputs.size(); ++i) {
       maskPatterns(planMask, outputs.get(i));
-      QTestProcessExecResult result = executeDiffCommand(
+      int ecode = executeDiffCommand(
           outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
-      if (result.getReturnCode() != 0) {
+      if (ecode != 0) {
         System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i));
-        return result;
+        return ecode;
       }
     }
-    return QTestProcessExecResult.createWithoutOutput(0);
+    return 0;
   }
 
-  private static void overwriteResults(String inFileName, String outFileName) throws Exception {
+  private static int overwriteResults(String inFileName, String outFileName) throws Exception {
     // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING)
     // once Hive uses JAVA 7.
     System.out.println("Overwriting results " + inFileName + " to " + outFileName);
-    int result = executeCmd(new String[]{
+    return executeCmd(new String[] {
         "cp",
         getQuotedString(inFileName),
         getQuotedString(outFileName)
-    }).getReturnCode();
-    if (result != 0)
-      throw new IllegalStateException("Unexpected error while overwriting " +
-          inFileName + " with " + outFileName);
+      });
   }
 
-  private static QTestProcessExecResult executeDiffCommand(String inFileName,
+  private static int executeDiffCommand(String inFileName,
       String outFileName,
       boolean ignoreWhiteSpace,
       boolean sortResults
       ) throws Exception {
 
-    QTestProcessExecResult result;
+    int result = 0;
 
     if (sortResults) {
       // sort will try to open the output file in write mode on windows. We need to
       // close it first.
       SessionState ss = SessionState.get();
       if (ss != null && ss.out != null && ss.out != System.out) {
-        ss.out.close();
+  ss.out.close();
       }
 
       String inSorted = inFileName + SORT_SUFFIX;
       String outSorted = outFileName + SORT_SUFFIX;
 
-      sortFiles(inFileName, inSorted);
-      sortFiles(outFileName, outSorted);
-
+      result = sortFiles(inFileName, inSorted);
+      result |= sortFiles(outFileName, outSorted);
+      if (result != 0) {
+        System.err.println("ERROR: Could not sort files before comparing");
+        return result;
+      }
       inFileName = inSorted;
       outFileName = outSorted;
     }
@@ -1823,47 +1813,40 @@ public class QTestUtil {
     return result;
   }
 
-  private static void sortFiles(String in, String out) throws Exception {
-    int result = executeCmd(new String[]{
+  private static int sortFiles(String in, String out) throws Exception {
+    return executeCmd(new String[] {
         "sort",
         getQuotedString(in),
-    }, out, null).getReturnCode();
-    if (result != 0)
-      throw new IllegalStateException("Unexpected error while sorting " + in);
+      }, out, null);
   }
 
-  private static QTestProcessExecResult executeCmd(Collection<String> args) throws Exception {
+  private static int executeCmd(Collection<String> args) throws Exception {
     return executeCmd(args, null, null);
   }
 
-  private static QTestProcessExecResult executeCmd(String[] args) throws Exception {
+  private static int executeCmd(String[] args) throws Exception {
     return executeCmd(args, null, null);
   }
 
-  private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile,
-                                            String errFile) throws Exception {
+  private static int executeCmd(Collection<String> args, String outFile, String errFile) throws Exception {
     String[] cmdArray = args.toArray(new String[args.size()]);
     return executeCmd(cmdArray, outFile, errFile);
   }
 
-  private static QTestProcessExecResult executeCmd(String[] args, String outFile,
-                                            String errFile) throws Exception {
+  private static int executeCmd(String[] args, String outFile, String errFile) throws Exception {
     System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
 
     PrintStream out = outFile == null ?
       SessionState.getConsole().getChildOutStream() :
-      new PrintStream(new FileOutputStream(outFile), true, "UTF-8");
+      new PrintStream(new FileOutputStream(outFile), true);
     PrintStream err = errFile == null ?
       SessionState.getConsole().getChildErrStream() :
-      new PrintStream(new FileOutputStream(errFile), true, "UTF-8");
+      new PrintStream(new FileOutputStream(errFile), true);
 
     Process executor = Runtime.getRuntime().exec(args);
 
-    ByteArrayOutputStream bos = new ByteArrayOutputStream();
-    PrintStream str = new PrintStream(bos, true, "UTF-8");
-
     StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err);
-    StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str);
+    StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out);
 
     outPrinter.start();
     errPrinter.start();
@@ -1881,8 +1864,7 @@ public class QTestUtil {
       err.close();
     }
 
-    return QTestProcessExecResult.
-        create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8));
+    return result;
   }
 
   private static String getQuotedString(String str){
@@ -2058,18 +2040,11 @@ public class QTestUtil {
       qt[i].clearTestSideEffects();
       qt[i].cliInit(qfiles[i].getName(), false);
       qt[i].executeClient(qfiles[i].getName());
-      QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
-      if (result.getReturnCode() != 0) {
+      int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
+      if (ecode != 0) {
         failed = true;
-        StringBuilder builder = new StringBuilder();
-        builder.append("Test ")
-            .append(qfiles[i].getName())
-            .append(" results check failed with error code ")
-            .append(result.getReturnCode());
-        if (Strings.isNotEmpty(result.getCapturedOutput())) {
-          builder.append(" and diff value ").append(result.getCapturedOutput());
-        }
-        System.err.println(builder.toString());
+        System.err.println("Test " + qfiles[i].getName()
+            + " results check failed with error code " + ecode);
         outputTestFailureHelpMessage();
       }
       qt[i].clearPostTestEffects();
@@ -2116,18 +2091,11 @@ public class QTestUtil {
 
     for (int i = 0; i < qfiles.length; i++) {
       qtThread[i].join();
-      QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
-      if (result.getReturnCode() != 0) {
+      int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
+      if (ecode != 0) {
         failed = true;
-        StringBuilder builder = new StringBuilder();
-        builder.append("Test ")
-            .append(qfiles[i].getName())
-            .append(" results check failed with error code ")
-            .append(result.getReturnCode());
-        if (Strings.isNotEmpty(result.getCapturedOutput())) {
-          builder.append(" and diff value ").append(result.getCapturedOutput());
-        }
-        System.err.println(builder.toString());
+        System.err.println("Test " + qfiles[i].getName()
+            + " results check failed with error code " + ecode);
         outputTestFailureHelpMessage();
       }
     }
@@ -2225,15 +2193,16 @@ public class QTestUtil {
 
   public void failedDiff(int ecode, String fname, String debugHint) {
     String message =
-        "Client Execution succeeded but contained differences " +
-            "(error code = " + ecode + ") after executing " +
+        "Client Execution results failed with error code = " + ecode + " while executing fname=" +
             fname + (debugHint != null ? (" " + debugHint) : "");
     LOG.error(message);
     Assert.fail(message);
   }
 
-  public void failed(Exception e, String fname, String debugHint) {
+  public void failed(Throwable e, String fname, String debugHint) {
     String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
+    System.err.println("Exception: " + e.getMessage());
+    e.printStackTrace();
     System.err.println("Failed query: " + fname);
     System.err.flush();
     Assert.fail("Unexpected exception " +
@@ -2273,6 +2242,9 @@ public class QTestUtil {
       }
       br.close();
     } catch (Exception e) {
+      System.err.println("Exception: " + e.getMessage());
+      e.printStackTrace();
+      System.err.flush();
       Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
     }
   }
@@ -2296,9 +2268,7 @@ public class QTestUtil {
       String mdbPath =   AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/";
 
       // Setup the table column stats
-      BufferedReader br = new BufferedReader(
-          new FileReader(
-              new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
+      BufferedReader br = new BufferedReader(new FileReader(new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
       String command;
 
       s.execute("DROP TABLE APP.TABLE_PARAMS");
@@ -2324,13 +2294,15 @@ public class QTestUtil {
       }
       br.close();
 
-      java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv" ,"TAB_COL_STATS.txt.bz2");
-      java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2");
+      File tabColStatsCsv = new File(mdbPath+"csv/TAB_COL_STATS.txt");
+      File tabParamsCsv = new File(mdbPath+"csv/TABLE_PARAMS.txt");
 
       // Set up the foreign key constraints properly in the TAB_COL_STATS data
       String tmpBaseDir =  System.getProperty(TEST_TMP_DIR_PROPERTY);
-      java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt");
-      java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt");
+      File tmpFileLoc1 = new File(tmpBaseDir+"/TAB_COL_STATS.txt");
+      File tmpFileLoc2 = new File(tmpBaseDir+"/TABLE_PARAMS.txt");
+      FileUtils.copyFile(tabColStatsCsv, tmpFileLoc1);
+      FileUtils.copyFile(tabParamsCsv, tmpFileLoc2);
 
       class MyComp implements Comparator<String> {
         @Override
@@ -2342,7 +2314,7 @@ public class QTestUtil {
         }
       }
 
-      final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
+      SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
 
      rs = s.executeQuery("SELECT * FROM APP.TBLS");
       while(rs.next()) {
@@ -2354,73 +2326,29 @@ public class QTestUtil {
           LOG.debug("Resultset : " +  tblName + " | " + tblId);
         }
       }
-
-      final Map<String, Map<String, String>> data = new HashMap<>();
-      rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from  "
-          + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a"
-          + " join APP.TBLS on  TBLS.SD_ID = a.SD_ID");
-      while (rs.next()) {
-        String tblName = rs.getString(1);
-        String colName = rs.getString(2);
-        String typeName = rs.getString(3);
-        Map<String, String> cols = data.get(tblName);
-        if (null == cols) {
-          cols = new HashMap<>();
+      for (Map.Entry<String, Integer> entry : tableNameToID.entrySet()) {
+        String toReplace1 = ",_" + entry.getKey() + "_" ;
+        String replacementString1 = ","+entry.getValue();
+        String toReplace2 = "_" + entry.getKey() + "_@" ;
+        String replacementString2 = ""+entry.getValue()+"@";
+        try {
+          String content1 = FileUtils.readFileToString(tmpFileLoc1, "UTF-8");
+          content1 = content1.replaceAll(toReplace1, replacementString1);
+          FileUtils.writeStringToFile(tmpFileLoc1, content1, "UTF-8");
+          String content2 = FileUtils.readFileToString(tmpFileLoc2, "UTF-8");
+          content2 = content2.replaceAll(toReplace2, replacementString2);
+          FileUtils.writeStringToFile(tmpFileLoc2, content2, "UTF-8");
+        } catch (IOException e) {
+          LOG.info("Generating file failed", e);
         }
-        cols.put(colName, typeName);
-        data.put(tblName, cols);
       }
 
-      BufferedReader reader = new BufferedReader(new InputStreamReader(
-        new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ))));
-
-      Stream<String> replaced = reader.lines().parallel().map(str-> {
-        String[] splits = str.split(",");
-        String tblName = splits[0];
-        String colName = splits[1];
-        Integer tblID = tableNameToID.get(tblName);
-        StringBuilder sb = new StringBuilder("default@"+tblName + "@" + colName + "@" + data.get(tblName).get(colName)+"@");
-        for (int i = 2; i < splits.length; i++) {
-          sb.append(splits[i]+"@");
-        }
-        return sb.append(tblID).toString();
-        });
-
-      Files.write(tmpFileLoc1, (Iterable<String>)replaced::iterator);
-      replaced.close();
-      reader.close();
-
-      BufferedReader reader2 = new BufferedReader(new InputStreamReader(
-          new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ))));
-      final Map<String,String> colStats = new ConcurrentHashMap<>();
-      Stream<String> replacedStream = reader2.lines().parallel().map(str-> {
-        String[] splits = str.split("_@");
-        String tblName = splits[0];
-        Integer tblId = tableNameToID.get(tblName);
-        Map<String,String> cols = data.get(tblName);
-        StringBuilder sb = new StringBuilder();
-        sb.append("{\"COLUMN_STATS\":{");
-        for (String colName : cols.keySet()) {
-          sb.append("\""+colName+"\":\"true\",");
-        }
-        sb.append("},\"BASIC_STATS\":\"true\"}");
-        colStats.put(tblId.toString(), sb.toString());
-
-        return  tblId.toString() + "@" + splits[1];
-      });
-
-      Files.write(tmpFileLoc2, (Iterable<String>)replacedStream::iterator);
-      Files.write(tmpFileLoc2, (Iterable<String>)colStats.entrySet().stream()
-        .map(map->map.getKey()+"@COLUMN_STATS_ACCURATE@"+map.getValue())::iterator, StandardOpenOption.APPEND);
-
-      replacedStream.close();
-      reader2.close();
       // Load the column stats and table params with 30 TB scale
-      String importStatement1 =  "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" +
-        "', '" + tmpFileLoc1.toAbsolutePath().toString() +
-        "', '@', null, 'UTF-8', 1)";
-      String importStatement2 =  "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" +
-        "', '" + tmpFileLoc2.toAbsolutePath().toString() +
+      String importStatement1 =  "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE(null, '" + "TAB_COL_STATS" +
+        "', '" + tmpFileLoc1.getAbsolutePath() +
+        "', ',', null, 'UTF-8', 1)";
+      String importStatement2 =  "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE(null, '" + "TABLE_PARAMS" +
+        "', '" + tmpFileLoc2.getAbsolutePath() +
         "', '@', null, 'UTF-8', 1)";
       try {
         PreparedStatement psImport1 = conn.prepareStatement(importStatement1);

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index 31f69a3..8dba0bb 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -21,12 +21,9 @@ import static org.junit.Assert.fail;
 
 import java.io.Serializable;
 import java.util.List;
-
-import com.google.common.base.Strings;
 import org.apache.hadoop.hive.cli.control.AbstractCliConfig;
 import org.apache.hadoop.hive.cli.control.CliAdapter;
 import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.apache.hadoop.hive.ql.exec.Task;
@@ -109,20 +106,18 @@ public class CoreParseNegative extends CliAdapter{
       fail("Unexpected success for query: " + fname + debugHint);
     }
     catch (ParseException pe) {
-      QTestProcessExecResult result = qt.checkNegativeResults(fname, pe);
-      if (result.getReturnCode() != 0) {
-        qt.failed(result.getReturnCode(), fname, result.getCapturedOutput() + "\r\n" + debugHint);
+      int ecode = qt.checkNegativeResults(fname, pe);
+      if (ecode != 0) {
+        qt.failed(ecode, fname, debugHint);
       }
     }
     catch (SemanticException se) {
-      QTestProcessExecResult result = qt.checkNegativeResults(fname, se);
-      if (result.getReturnCode() != 0) {
-        String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
-            debugHint : "\r\n" + result.getCapturedOutput();
-        qt.failedDiff(result.getReturnCode(), fname, message);
+      int ecode = qt.checkNegativeResults(fname, se);
+      if (ecode != 0) {
+        qt.failedDiff(ecode, fname, debugHint);
       }
     }
-    catch (Exception e) {
+    catch (Throwable e) {
       qt.failed(e, fname, debugHint);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java b/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java
deleted file mode 100644
index a819e26..0000000
--- a/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline;
-
-import org.junit.runners.Parameterized;
-import org.junit.runners.model.RunnerScheduler;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Class to run Parameterized test in parallel.
- * Source: http://hwellmann.blogspot.hu/2009/12/running-parameterized-junit-tests-in.html
- */
-public class Parallelized extends Parameterized {
-  private static class ThreadPoolScheduler implements RunnerScheduler {
-    private ExecutorService executor;
-
-    public ThreadPoolScheduler() {
-      String threads = System.getProperty("junit.parallel.threads");
-      int numThreads = Runtime.getRuntime().availableProcessors();
-      if (threads != null) {
-        numThreads = Integer.parseInt(threads);
-      }
-      executor = Executors.newFixedThreadPool(numThreads);
-    }
-
-    @Override
-    public void finished() {
-      executor.shutdown();
-      try {
-        executor.awaitTermination(10, TimeUnit.MINUTES);
-      } catch (InterruptedException exc) {
-        throw new RuntimeException(exc);
-      }
-    }
-
-    @Override
-    public void schedule(Runnable childStatement) {
-      executor.submit(childStatement);
-    }
-  }
-
-  public Parallelized(Class klass) throws Throwable {
-    super(klass);
-    setScheduler(new ThreadPoolScheduler());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
deleted file mode 100644
index 0bde529..0000000
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.beeline;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
-import org.apache.hadoop.hive.ql.QTestUtil;
-import org.apache.hadoop.util.Shell;
-import org.apache.hive.common.util.StreamPrinter;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-/**
- * Class for representing a Query and the connected files. It provides accessors for the specific
- * input and output files, and provides methods for filtering the output of the runs.
- */
-public final class QFile {
-  private static final Set<String> srcTables = QTestUtil.getSrcTables();
-  private static final String DEBUG_HINT =
-      "The following files can help you identifying the problem:%n"
-      + " - Query file: %1s%n"
-      + " - Raw output file: %2s%n"
-      + " - Filtered output file: %3s%n"
-      + " - Expected output file: %4s%n"
-      + " - Client log file: %5s%n"
-      + " - Client log files before the test: %6s%n"
-      + " - Client log files after the test: %7s%n"
-      + " - Hiveserver2 log file: %8s%n";
-  private static final String USE_COMMAND_WARNING =
-      "The query file %1s contains \"%2s\" command.%n"
-      + "The source table name rewrite is turned on, so this might cause problems when the used "
-      + "database contains tables named any of the following: " + srcTables + "%n"
-      + "To turn off the table name rewrite use -Dtest.rewrite.source.tables=false%n";
-
-  private static final Pattern USE_PATTERN =
-      Pattern.compile("^\\s*use\\s.*", Pattern.CASE_INSENSITIVE);
-
-  private static final String MASK_PATTERN = "#### A masked pattern was here ####\n";
-
-  private String name;
-  private File inputFile;
-  private File rawOutputFile;
-  private File outputFile;
-  private File expectedOutputFile;
-  private File logFile;
-  private File beforeExecuteLogFile;
-  private File afterExecuteLogFile;
-  private static RegexFilterSet staticFilterSet = getStaticFilterSet();
-  private RegexFilterSet specificFilterSet;
-  private boolean rewriteSourceTables;
-
-  private QFile() {}
-
-  public String getName() {
-    return name;
-  }
-
-  public File getInputFile() {
-    return inputFile;
-  }
-
-  public File getRawOutputFile() {
-    return rawOutputFile;
-  }
-
-  public File getOutputFile() {
-    return outputFile;
-  }
-
-  public File getExpectedOutputFile() {
-    return expectedOutputFile;
-  }
-
-  public File getLogFile() {
-    return logFile;
-  }
-
-  public File getBeforeExecuteLogFile() {
-    return beforeExecuteLogFile;
-  }
-
-  public File getAfterExecuteLogFile() {
-    return afterExecuteLogFile;
-  }
-
-  public String getDebugHint() {
-    return String.format(DEBUG_HINT, inputFile, rawOutputFile, outputFile, expectedOutputFile,
-        logFile, beforeExecuteLogFile, afterExecuteLogFile,
-        "./itests/qtest/target/tmp/log/hive.log");
-  }
-
-  /**
-   * Filters the sql commands if necessary.
-   * @param commands The array of the sql commands before filtering
-   * @return The filtered array of the sql command strings
-   * @throws IOException File read error
-   */
-  public String[] filterCommands(String[] commands) throws IOException {
-    if (rewriteSourceTables) {
-      for (int i=0; i<commands.length; i++) {
-        if (USE_PATTERN.matcher(commands[i]).matches()) {
-          System.err.println(String.format(USE_COMMAND_WARNING, inputFile, commands[i]));
-        }
-        commands[i] = replaceTableNames(commands[i]);
-      }
-    }
-    return commands;
-  }
-
-  /**
-   * Replace the default src database TABLE_NAMEs in the queries with default.TABLE_NAME, like
-   * src->default.src, srcpart->default.srcpart, so the queries could be run even if the used
-   * database is query specific. This change is only a best effort, since we do not want to parse
-   * the queries, we could not be sure that we do not replace other strings which are not
-   * tablenames. Like 'select src from othertable;'. The q files containing these commands should
-   * be excluded. Only replace the tablenames, if rewriteSourceTables is set.
-   * @param source The original query string
-   * @return The query string where the tablenames are replaced
-   */
-  private String replaceTableNames(String source) {
-    for (String table : srcTables) {
-      source = source.replaceAll("(?is)(\\s+)" + table + "([\\s;\\n\\)])", "$1default." + table
-          + "$2");
-    }
-    return source;
-  }
-
-  /**
-   * The result contains the original queries. To revert them to the original form remove the
-   * 'default' from every default.TABLE_NAME, like default.src->src, default.srcpart->srcpart.
-   * @param source The original query output
-   * @return The query output where the tablenames are replaced
-   */
-  private String revertReplaceTableNames(String source) {
-    for (String table : srcTables) {
-      source = source.replaceAll("(?is)(\\s+)default\\." + table + "([\\s;\\n\\)])", "$1" + table
-          + "$2");
-    }
-    return source;
-  }
-
-  public void filterOutput() throws IOException {
-    String rawOutput = FileUtils.readFileToString(rawOutputFile, "UTF-8");
-    if (rewriteSourceTables) {
-      rawOutput = revertReplaceTableNames(rawOutput);
-    }
-    String filteredOutput = staticFilterSet.filter(specificFilterSet.filter(rawOutput));
-    FileUtils.writeStringToFile(outputFile, filteredOutput);
-  }
-
-  public QTestProcessExecResult compareResults() throws IOException, InterruptedException {
-    if (!expectedOutputFile.exists()) {
-      throw new IOException("Expected results file does not exist: " + expectedOutputFile);
-    }
-    return executeDiff();
-  }
-
-  public void overwriteResults() throws IOException {
-    FileUtils.copyFile(outputFile, expectedOutputFile);
-  }
-
-  private QTestProcessExecResult executeDiff() throws IOException, InterruptedException {
-    List<String> diffCommandArgs = new ArrayList<String>();
-    diffCommandArgs.add("diff");
-
-    // Text file comparison
-    diffCommandArgs.add("-a");
-
-    if (Shell.WINDOWS) {
-      // Ignore changes in the amount of white space
-      diffCommandArgs.add("-b");
-
-      // Files created on Windows machines have different line endings
-      // than files created on Unix/Linux. Windows uses carriage return and line feed
-      // ("\r\n") as a line ending, whereas Unix uses just line feed ("\n").
-      // Also StringBuilder.toString(), Stream to String conversions adds extra
-      // spaces at the end of the line.
-      diffCommandArgs.add("--strip-trailing-cr"); // Strip trailing carriage return on input
-      diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank
-    }
-
-    // Add files to compare to the arguments list
-    diffCommandArgs.add(getQuotedString(expectedOutputFile));
-    diffCommandArgs.add(getQuotedString(outputFile));
-
-    System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(diffCommandArgs,
-        ' '));
-    Process executor = Runtime.getRuntime().exec(diffCommandArgs.toArray(
-        new String[diffCommandArgs.size()]));
-
-    ByteArrayOutputStream bos = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(bos, true, "UTF-8");
-
-    StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err);
-    StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out, out);
-
-    outPrinter.start();
-    errPrinter.start();
-
-    int result = executor.waitFor();
-
-    outPrinter.join();
-    errPrinter.join();
-
-    executor.waitFor();
-
-    return QTestProcessExecResult.create(result, new String(bos.toByteArray(),
-        StandardCharsets.UTF_8));
-  }
-
-  private static String getQuotedString(File file) {
-    return Shell.WINDOWS ? String.format("\"%s\"", file.getAbsolutePath()) : file.getAbsolutePath();
-  }
-
-  private static class Filter {
-    private final Pattern pattern;
-    private final String replacement;
-
-    public Filter(Pattern pattern, String replacement) {
-      this.pattern = pattern;
-      this.replacement = replacement;
-    }
-  }
-
-  private static class RegexFilterSet {
-    private final List<Filter> regexFilters = new ArrayList<Filter>();
-
-    public RegexFilterSet addFilter(String regex, String replacement) {
-      regexFilters.add(new Filter(Pattern.compile(regex), replacement));
-      return this;
-    }
-
-    public String filter(String input) {
-      for (Filter filter : regexFilters) {
-        input = filter.pattern.matcher(input).replaceAll(filter.replacement);
-      }
-      return input;
-    }
-  }
-
-  // These are the filters which are common for every QTest.
-  // Check specificFilterSet for QTest specific ones.
-  private static RegexFilterSet getStaticFilterSet() {
-    // Pattern to remove the timestamp and other infrastructural info from the out file
-    return new RegexFilterSet()
-        .addFilter("Reading log file: .*\n", "")
-        .addFilter("INFO  : ", "")
-        .addFilter(".*/tmp/.*\n", MASK_PATTERN)
-        .addFilter(".*file:.*\n", MASK_PATTERN)
-        .addFilter(".*file\\..*\n", MASK_PATTERN)
-        .addFilter(".*CreateTime.*\n", MASK_PATTERN)
-        .addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN)
-        .addFilter("(?s)(" + MASK_PATTERN + ")+", MASK_PATTERN);
-  }
-
-  /**
-   * Builder to generate QFile objects. After initializing the builder it is possible the
-   * generate the next QFile object using it's name only.
-   */
-  public static class QFileBuilder {
-    private File queryDirectory;
-    private File logDirectory;
-    private File resultsDirectory;
-    private boolean rewriteSourceTables;
-
-    public QFileBuilder() {
-    }
-
-    public QFileBuilder setQueryDirectory(File queryDirectory) {
-      this.queryDirectory = queryDirectory;
-      return this;
-    }
-
-    public QFileBuilder setLogDirectory(File logDirectory) {
-      this.logDirectory = logDirectory;
-      return this;
-    }
-
-    public QFileBuilder setResultsDirectory(File resultsDirectory) {
-      this.resultsDirectory = resultsDirectory;
-      return this;
-    }
-
-    public QFileBuilder setRewriteSourceTables(boolean rewriteSourceTables) {
-      this.rewriteSourceTables = rewriteSourceTables;
-      return this;
-    }
-
-    public QFile getQFile(String name) throws IOException {
-      QFile result = new QFile();
-      result.name = name;
-      result.inputFile = new File(queryDirectory, name + ".q");
-      result.rawOutputFile = new File(logDirectory, name + ".q.out.raw");
-      result.outputFile = new File(logDirectory, name + ".q.out");
-      result.expectedOutputFile = new File(resultsDirectory, name + ".q.out");
-      result.logFile = new File(logDirectory, name + ".q.beeline");
-      result.beforeExecuteLogFile = new File(logDirectory, name + ".q.beforeExecute.log");
-      result.afterExecuteLogFile = new File(logDirectory, name + ".q.afterExecute.log");
-      result.rewriteSourceTables = rewriteSourceTables;
-      result.specificFilterSet = new RegexFilterSet()
-          .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + name + "\n",
-              "$1: $2: database:default\n")
-          .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + name + "@", "$1: $2: default@")
-          .addFilter("name(:?) " + name + "\\.(.*)\n", "name$1 default.$2\n")
-          .addFilter("/" + name + ".db/", "/");
-      return result;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java b/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
deleted file mode 100644
index f1b53f7..0000000
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.beeline;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.sql.SQLException;
-
-/**
- * QFile test client using BeeLine. It can be used to submit a list of command strings, or a QFile.
- */
-public class QFileBeeLineClient implements AutoCloseable {
-  private BeeLine beeLine;
-  private PrintStream beelineOutputStream;
-  private File logFile;
-
-  protected QFileBeeLineClient(String jdbcUrl, String jdbcDriver, String username, String password,
-      File log) throws IOException {
-    logFile = log;
-    beeLine = new BeeLine();
-    beelineOutputStream = new PrintStream(logFile, "UTF-8");
-    beeLine.setOutputStream(beelineOutputStream);
-    beeLine.setErrorStream(beelineOutputStream);
-    beeLine.runCommands(
-        new String[] {
-          "!set verbose true",
-          "!set shownestederrs true",
-          "!set showwarnings true",
-          "!set showelapsedtime false",
-          "!set trimscripts false",
-          "!set maxwidth -1",
-          "!connect " + jdbcUrl + " " + username + " " + password + " " + jdbcDriver
-        });
-  }
-
-  public void execute(String[] commands, File resultFile) throws SQLException {
-    beeLine.runCommands(
-        new String[] {
-          "!record " + resultFile.getAbsolutePath()
-        });
-
-    int lastSuccessfulCommand = beeLine.runCommands(commands);
-    if (commands.length != lastSuccessfulCommand) {
-      throw new SQLException("Error executing SQL command: " + commands[lastSuccessfulCommand]);
-    }
-
-    beeLine.runCommands(new String[] {"!record"});
-  }
-
-  private void beforeExecute(QFile qFile) throws SQLException {
-    execute(
-        new String[] {
-          "!set outputformat tsv2",
-          "!set verbose false",
-          "!set silent true",
-          "!set showheader false",
-          "USE default;",
-          "SHOW TABLES;",
-          "DROP DATABASE IF EXISTS `" + qFile.getName() + "` CASCADE;",
-          "CREATE DATABASE `" + qFile.getName() + "`;",
-          "USE `" + qFile.getName() + "`;",
-          "set hive.in.test.short.logs=true;",
-          "set hive.in.test.remove.logs=false;",
-        },
-        qFile.getBeforeExecuteLogFile());
-    beeLine.setIsTestMode(true);
-  }
-
-  private void afterExecute(QFile qFile) throws SQLException {
-    beeLine.setIsTestMode(false);
-    execute(
-        new String[] {
-          "set hive.in.test.short.logs=false;",
-          "!set verbose true",
-          "!set silent false",
-          "!set showheader true",
-          "!set outputformat table",
-          "USE default;",
-          "DROP DATABASE IF EXISTS `" + qFile.getName() + "` CASCADE;",
-        },
-        qFile.getAfterExecuteLogFile());
-  }
-
-  public void execute(QFile qFile) throws SQLException, IOException {
-    beforeExecute(qFile);
-    String[] commands = beeLine.getCommands(qFile.getInputFile());
-    execute(qFile.filterCommands(commands), qFile.getRawOutputFile());
-    afterExecute(qFile);
-  }
-
-  public void close() {
-    if (beeLine != null) {
-      beeLine.runCommands(new String[] {
-        "!quit"
-      });
-    }
-    if (beelineOutputStream != null) {
-      beelineOutputStream.close();
-    }
-  }
-
-  /**
-   * Builder to generated QFileBeeLineClient objects. The after initializing the builder, it can be
-   * used to create new clients without any parameters.
-   */
-  public static class QFileClientBuilder {
-    private String username;
-    private String password;
-    private String jdbcUrl;
-    private String jdbcDriver;
-
-    public QFileClientBuilder() {
-    }
-
-    public QFileClientBuilder setUsername(String username) {
-      this.username = username;
-      return this;
-    }
-
-    public QFileClientBuilder setPassword(String password) {
-      this.password = password;
-      return this;
-    }
-
-    public QFileClientBuilder setJdbcUrl(String jdbcUrl) {
-      this.jdbcUrl = jdbcUrl;
-      return this;
-    }
-
-    public QFileClientBuilder setJdbcDriver(String jdbcDriver) {
-      this.jdbcDriver = jdbcDriver;
-      return this;
-    }
-
-    public QFileBeeLineClient getClient(File logFile) throws IOException {
-      return new QFileBeeLineClient(jdbcUrl, jdbcDriver, username, password, logFile);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/package-info.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/package-info.java b/itests/util/src/main/java/org/apache/hive/beeline/package-info.java
deleted file mode 100644
index e05ac0a..0000000
--- a/itests/util/src/main/java/org/apache/hive/beeline/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package for the BeeLine specific QTest classes.
- */
-package org.apache.hive.beeline;


Mime
View raw message