hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ych...@apache.org
Subject hive git commit: HIVE-16643: BeeLine tests output should keep the PREHOOK/POSTHOOK Input/Output orderdering (Peter Vary, reviewed by Yongzhi Chen)
Date Wed, 24 May 2017 14:30:28 GMT
Repository: hive
Updated Branches:
  refs/heads/master 1bdc5413d -> dec96ca6f


HIVE-16643: BeeLine tests output should keep the PREHOOK/POSTHOOK Input/Output orderdering
(Peter Vary, reviewed by Yongzhi Chen)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dec96ca6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dec96ca6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dec96ca6

Branch: refs/heads/master
Commit: dec96ca6f6fb8de04b2847be2285792275ea9423
Parents: 1bdc541
Author: Yongzhi Chen <ychena@apache.org>
Authored: Wed May 24 10:30:13 2017 -0400
Committer: Yongzhi Chen <ychena@apache.org>
Committed: Wed May 24 10:30:13 2017 -0400

----------------------------------------------------------------------
 .../test/resources/testconfiguration.properties |   3 +-
 .../java/org/apache/hive/beeline/QFile.java     |  32 ++++-
 .../beeline/create_merge_compressed.q.out       | 138 +++++++++++++++++++
 .../beeline/escape_comments.q.out               |   2 -
 4 files changed, 168 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/dec96ca6/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index e23ef63..8e99b85 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -756,7 +756,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
   encryption_auto_purge_tables.q \
   encryption_drop_table_in_encrypted_db.q
 
-beeline.positive.include=drop_with_concurrency.q,\
+beeline.positive.include=create_merge_compressed.q,\
+  drop_with_concurrency.q,\
   escape_comments.q,\
   mapjoin2.q,\
   materialized_view_create_rewrite.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/dec96ca6/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
index af464b9..0d00a9b 100644
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
+++ b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
@@ -31,8 +31,11 @@ import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
@@ -59,6 +62,8 @@ public final class QFile {
 
   private static final Pattern USE_PATTERN =
       Pattern.compile("^\\s*use\\s.*", Pattern.CASE_INSENSITIVE);
+  private static final Pattern ENTITYLIST_PATTERN =
+      Pattern.compile("(((PREHOOK|POSTHOOK): (Input|Output): \\S+\n)+)", Pattern.MULTILINE);
 
   private static final String MASK_PATTERN = "#### A masked pattern was here ####\n";
 
@@ -174,13 +179,29 @@ public final class QFile {
     return source;
   }
 
+  /**
+   * The PREHOOK/POSTHOOK Input/Output lists should be sorted again after reverting the database
+   * name in those strings to match the original Cli output.
+   * @param source The original query output
+   * @return The query output where the input/output list are alphabetically ordered
+   */
+  private String sortInputOutput(String source) {
+    Matcher matcher = ENTITYLIST_PATTERN.matcher(source);
+    while(matcher.find()) {
+      List<String> lines = Arrays.asList(matcher.group(1).split("\n"));
+      Collections.sort(lines);
+      source = source.replaceAll(matcher.group(1), String.join("\n", lines) + "\n");
+    }
+    return source;
+  }
+
   public void filterOutput() throws IOException {
-    String rawOutput = FileUtils.readFileToString(rawOutputFile, "UTF-8");
+    String output = FileUtils.readFileToString(rawOutputFile, "UTF-8");
+    output = staticFilterSet.filter(specificFilterSet.filter(output));
     if (rewriteSourceTables) {
-      rawOutput = revertReplaceTableNames(rawOutput);
+      output = sortInputOutput(revertReplaceTableNames(output));
     }
-    String filteredOutput = staticFilterSet.filter(specificFilterSet.filter(rawOutput));
-    FileUtils.writeStringToFile(outputFile, filteredOutput);
+    FileUtils.writeStringToFile(outputFile, output);
   }
 
   public QTestProcessExecResult compareResults() throws IOException, InterruptedException
{
@@ -285,6 +306,9 @@ public final class QFile {
         .addFilter(".*file\\..*\n", MASK_PATTERN)
         .addFilter(".*CreateTime.*\n", MASK_PATTERN)
         .addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN)
+        .addFilter(".*lastUpdateTime.*\n", MASK_PATTERN)
+        .addFilter(".*lastAccessTime.*\n", MASK_PATTERN)
+        .addFilter(".*[Oo]wner.*\n", MASK_PATTERN)
         .addFilter("(?s)(" + MASK_PATTERN + ")+", MASK_PATTERN);
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/dec96ca6/ql/src/test/results/clientpositive/beeline/create_merge_compressed.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/beeline/create_merge_compressed.q.out b/ql/src/test/results/clientpositive/beeline/create_merge_compressed.q.out
new file mode 100644
index 0000000..58f517c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/beeline/create_merge_compressed.q.out
@@ -0,0 +1,138 @@
+PREHOOK: query: create table src_rc_merge_test(key int, value string) stored as rcfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_rc_merge_test
+POSTHOOK: query: create table src_rc_merge_test(key int, value string) stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_rc_merge_test
+PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' into table src_rc_merge_test
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_rc_merge_test
+POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' into table src_rc_merge_test
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_rc_merge_test
+PREHOOK: query: create table tgt_rc_merge_test(key int, value string) stored as rcfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: create table tgt_rc_merge_test(key int, value string) stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tgt_rc_merge_test
+PREHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_rc_merge_test
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_rc_merge_test
+POSTHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: Lineage: tgt_rc_merge_test.key SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:key,
type:int, comment:null), ]
+POSTHOOK: Lineage: tgt_rc_merge_test.value SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:value,
type:string, comment:null), ]
+PREHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_rc_merge_test
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_rc_merge_test
+POSTHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: Lineage: tgt_rc_merge_test.key SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:key,
type:int, comment:null), ]
+POSTHOOK: Lineage: tgt_rc_merge_test.value SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:value,
type:string, comment:null), ]
+PREHOOK: query: show table extended like `tgt_rc_merge_test`
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like `tgt_rc_merge_test`
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:tgt_rc_merge_test
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+columns:struct columns { i32 key, string value}
+partitioned:false
+partitionColumns:
+totalNumberFiles:2
+totalFileSize:342
+maxFileSize:171
+minFileSize:171
+#### A masked pattern was here ####
+
+PREHOOK: query: select count(1) from tgt_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from tgt_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+10
+PREHOOK: query: select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+46	-751895388
+PREHOOK: query: alter table tgt_rc_merge_test concatenate
+PREHOOK: type: ALTER_TABLE_MERGE
+PREHOOK: Input: default@tgt_rc_merge_test
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: alter table tgt_rc_merge_test concatenate
+POSTHOOK: type: ALTER_TABLE_MERGE
+POSTHOOK: Input: default@tgt_rc_merge_test
+POSTHOOK: Output: default@tgt_rc_merge_test
+PREHOOK: query: show table extended like `tgt_rc_merge_test`
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like `tgt_rc_merge_test`
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:tgt_rc_merge_test
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+columns:struct columns { i32 key, string value}
+partitioned:false
+partitionColumns:
+totalNumberFiles:1
+totalFileSize:243
+maxFileSize:243
+minFileSize:243
+#### A masked pattern was here ####
+
+PREHOOK: query: select count(1) from tgt_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from tgt_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+10
+PREHOOK: query: select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+46	-751895388
+PREHOOK: query: drop table src_rc_merge_test
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@src_rc_merge_test
+PREHOOK: Output: default@src_rc_merge_test
+POSTHOOK: query: drop table src_rc_merge_test
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@src_rc_merge_test
+POSTHOOK: Output: default@src_rc_merge_test
+PREHOOK: query: drop table tgt_rc_merge_test
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tgt_rc_merge_test
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: drop table tgt_rc_merge_test
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tgt_rc_merge_test
+POSTHOOK: Output: default@tgt_rc_merge_test

http://git-wip-us.apache.org/repos/asf/hive/blob/dec96ca6/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/beeline/escape_comments.q.out b/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
index 07fef57..aa26c57 100644
--- a/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
+++ b/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
@@ -95,7 +95,6 @@ p1	string	a\nb
 	NULL	NULL
 # Detailed Table Information	NULL	NULL
 Database:           	escape_comments_db  	NULL
-Owner:              	user                	NULL
 #### A masked pattern was here ####
 LastAccessTime:     	UNKNOWN             	NULL
 Retention:          	0                   	NULL
@@ -171,7 +170,6 @@ col1	string	a\nb
 	NULL	NULL
 # Detailed Table Information	NULL	NULL
 Database:           	escape_comments_db  	NULL
-Owner:              	user                	NULL
 #### A masked pattern was here ####
 LastAccessTime:     	UNKNOWN             	NULL
 Retention:          	0                   	NULL


Mime
View raw message