hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject hive git commit: HIVE-15226: Add a different masking comment to qtests blobstore output (Sergio Pena, reviewed by Mohit Sabharwal and Sahil Takiar)
Date Thu, 17 Nov 2016 18:44:43 GMT
Repository: hive
Updated Branches:
  refs/heads/master 2de8b3192 -> c5d147b68


HIVE-15226: Add a different masking comment to qtests blobstore output (Sergio Pena, reviewed
by Mohit Sabharwal and Sahil Takiar)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c5d147b6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c5d147b6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c5d147b6

Branch: refs/heads/master
Commit: c5d147b68a922a69aef163a2287bc3761b3a7186
Parents: 2de8b31
Author: Sergio Pena <sergio.pena@cloudera.com>
Authored: Thu Nov 17 12:43:26 2016 -0600
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Thu Nov 17 12:44:33 2016 -0600

----------------------------------------------------------------------
 .../results/clientpositive/insert_into.q.out    |  4 ++--
 .../control/AbstractCoreBlobstoreCliDriver.java |  2 ++
 .../org/apache/hadoop/hive/ql/QTestUtil.java    | 24 ++++++++++++++++++--
 3 files changed, 26 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c5d147b6/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
index 919f3e7..93f545d 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
@@ -4,12 +4,12 @@ POSTHOOK: query: DROP TABLE qtest
 POSTHOOK: type: DROPTABLE
 #### A masked pattern was here ####
 PREHOOK: type: CREATETABLE
-PREHOOK: Input: #### A masked pattern was here ####
+PREHOOK: Input: ### test.blobstore.path ###/qtest
 PREHOOK: Output: database:default
 PREHOOK: Output: default@qtest
 #### A masked pattern was here ####
 POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: #### A masked pattern was here ####
+POSTHOOK: Input: ### test.blobstore.path ###/qtest
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@qtest
 PREHOOK: query: INSERT INTO qtest VALUES (1), (10), (100), (1000)

http://git-wip-us.apache.org/repos/asf/hive/blob/c5d147b6/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index e797eb5..2f57683 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -163,5 +163,7 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter
{
     String uid = new SimpleDateFormat("yyyyMMdd.HHmmss.SSS").format(Calendar.getInstance().getTime())
         + "-" + String.format("%03d", (int)(Math.random() * 999));
     testBlobstorePathUnique = testBlobstorePath + uid;
+
+    qt.addPatternWithMaskComment(testBlobstorePathUnique, String.format("### %s ###", HCONF_TEST_BLOBSTORE_PATH));
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/c5d147b6/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index ab21589..d08ecc7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -67,6 +67,8 @@ import com.google.common.base.Preconditions;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileStatus;
@@ -1643,6 +1645,18 @@ public class QTestUtil {
       }
 
       if (!partialMaskWasMatched) {
+        for (Pair<Pattern, String> pair : patternsWithMaskComments) {
+          Pattern pattern = pair.getLeft();
+          String maskComment = pair.getRight();
+
+          matcher = pattern.matcher(line);
+          if (matcher.find()) {
+            line = matcher.replaceAll(maskComment);
+            partialMaskWasMatched = true;
+            break;
+          }
+        }
+
         for (Pattern pattern : patterns) {
           line = pattern.matcher(line).replaceAll(maskPattern);
         }
@@ -1707,8 +1721,7 @@ public class QTestUtil {
       ".*at com\\.sun\\.proxy.*",
       ".*at com\\.jolbox.*",
       ".*at com\\.zaxxer.*",
-      "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*",
-      "(s3.?|swift|wasb.?):\\/\\/[\\w\\.\\/-]*"
+      "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*"
   });
 
   private final Pattern[] partialReservedPlanMask = toPattern(new String[] {
@@ -1716,6 +1729,13 @@ public class QTestUtil {
       //TODO: add more expected test result here
   });
 
+  /* This list may be modified by specific cli drivers to mask strings that change on every
test */
+  private final List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<>();
+
+  public void addPatternWithMaskComment(String patternStr, String maskComment) {
+    patternsWithMaskComments.add(ImmutablePair.of(Pattern.compile(patternStr), maskComment));
+  }
+
   public int checkCliDriverResults(String tname) throws Exception {
     assert(qMap.containsKey(tname));
 


Mime
View raw message