hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject hive git commit: HIVE-15363: Execute hive-blobstore tests using ProxyLocalFileSystem (Sergio Pena, reviewed by Mohit Sabharwal)
Date Tue, 06 Dec 2016 21:00:11 GMT
Repository: hive
Updated Branches:
  refs/heads/master 2f1015365 -> f317d738f


HIVE-15363: Execute hive-blobstore tests using ProxyLocalFileSystem (Sergio Pena, reviewed
by Mohit Sabharwal)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f317d738
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f317d738
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f317d738

Branch: refs/heads/master
Commit: f317d738f526cf3a43955358c48612fab2cfa681
Parents: 2f10153
Author: Sergio Pena <sergio.pena@cloudera.com>
Authored: Tue Dec 6 14:59:42 2016 -0600
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Tue Dec 6 14:59:42 2016 -0600

----------------------------------------------------------------------
 itests/hive-blobstore/pom.xml                   | 25 --------------------
 .../src/test/resources/hive-site.xml            | 19 +++++++++++++++
 .../control/AbstractCoreBlobstoreCliDriver.java | 13 +++++++++-
 .../org/apache/hadoop/hive/ql/QTestUtil.java    |  2 +-
 4 files changed, 32 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/hive-blobstore/pom.xml
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/pom.xml b/itests/hive-blobstore/pom.xml
index a62885f..edc0872 100644
--- a/itests/hive-blobstore/pom.xml
+++ b/itests/hive-blobstore/pom.xml
@@ -39,31 +39,6 @@
     <test.dfs.mkdir>-mkdir -p</test.dfs.mkdir>
   </properties>
 
-  <profiles>
-    <profile>
-      <id>tests-off</id>
-      <activation>
-        <file>
-          <missing>src/test/resources/blobstore-conf.xml</missing>
-        </file>
-      </activation>
-      <properties>
-        <maven.test.skip>true</maven.test.skip>
-      </properties>
-    </profile>
-    <profile>
-      <id>tests-on</id>
-      <activation>
-        <file>
-          <exists>src/test/resources/blobstore-conf.xml</exists>
-        </file>
-      </activation>
-      <properties>
-        <maven.test.skip>false</maven.test.skip>
-      </properties>
-    </profile>
-  </profiles>
-
   <dependencies>
     <!-- dependencies are always listed in sorted order by groupId, artifactId -->
     <dependency>

http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/hive-blobstore/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/resources/hive-site.xml b/itests/hive-blobstore/src/test/resources/hive-site.xml
index ffac40a..406b3b8 100644
--- a/itests/hive-blobstore/src/test/resources/hive-site.xml
+++ b/itests/hive-blobstore/src/test/resources/hive-site.xml
@@ -261,6 +261,25 @@
   </property>
 
   <!--
+    The following 3 properties are used for testing blobstore changes on the local filesystem.
+    If tests are required to run on an real Blobstore system, then create the blobstore-conf.xml.
+  -->
+  <property>
+    <name>fs.pblob.impl</name>
+    <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
+  </property>
+
+  <property>
+    <name>hive.blobstore.supported.schemes</name>
+    <value>pblob,s3a,s3c,s3</value>
+  </property>
+
+  <property>
+    <name>test.blobstore.path</name>
+    <value>pblob:${system:test.tmp.dir}/bucket</value>
+  </property>
+
+  <!--
   To run these tests:
   # Create a file blobstore-conf.xml  - DO NOT ADD TO REVISION CONTROL
   # Add the credentials for the service you are testing against

http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 2f57683..9c97c31 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -23,7 +23,12 @@ import static org.junit.Assert.fail;
 import com.google.common.base.Strings;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
+import java.util.Map;
+
 import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.junit.After;
@@ -157,7 +162,13 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter
{
    *  the same ${test.blobstore.path} (possible if test runs are controlled by an automated
system)
    */
   private void setupUniqueTestPath() {
-    String testBlobstorePath = qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH);
+    String testBlobstorePath = new VariableSubstitution(new HiveVariableSource() {
+      @Override
+      public Map<String, String> getHiveVariable() {
+        return null;
+      }
+    }).substitute(new HiveConf(), qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH));
+
     testBlobstorePath = QTestUtil.ensurePathEndsInSlash(testBlobstorePath);
     testBlobstorePath += QTestUtil.ensurePathEndsInSlash(this.getClass().getSimpleName());
// name of child class
     String uid = new SimpleDateFormat("yyyyMMdd.HHmmss.SSS").format(Calendar.getInstance().getTime())

http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 5bf23e7..8b383c7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1731,7 +1731,7 @@ public class QTestUtil {
 
   /* This list may be modified by specific cli drivers to mask strings that change on every
test */
   private List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern,
String>>() {{
-    add(toPatternPair("(s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###"));
+    add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH
###"));
   }};
 
   private Pair<Pattern, String> toPatternPair(String patternStr, String maskComment)
{


Mime
View raw message