hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [36/51] [partial] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)
Date Fri, 05 May 2017 17:32:23 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java b/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java
new file mode 100644
index 0000000..6cbcf8c
--- /dev/null
+++ b/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hadoop.hive.jdbc;
+
+import java.io.File;
+import java.net.URLEncoder;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Map;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+
+public class SSLTestUtils {
+
+  private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks";
+  private static final String TRUST_STORE_NAME = "truststore.jks";
+  private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc";
+  private static final String HS2_BINARY_MODE = "binary";
+  private static final String HS2_HTTP_MODE = "http";
+  private static final String HS2_HTTP_ENDPOINT = "cliservice";
+  private static final String HS2_BINARY_AUTH_MODE = "NONE";
+
+  private static final HiveConf conf = new HiveConf();
+  private static final String dataFileDir = !System.getProperty("test.data.files", "").isEmpty() ? System.getProperty(
+          "test.data.files") : conf.get("test.data.files").replace('\\', '/').replace("c:", "");
+
+  public static final String SSL_CONN_PARAMS = "ssl=true;sslTrustStore="
+      + URLEncoder.encode(dataFileDir + File.separator + TRUST_STORE_NAME) + ";trustStorePassword="
+      + KEY_STORE_TRUST_STORE_PASSWORD;
+
+  public static void setSslConfOverlay(Map<String, String> confOverlay) {
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_USE_SSL.varname, "true");
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname,
+            dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME);
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname,
+            KEY_STORE_TRUST_STORE_PASSWORD);
+  }
+
+  public static void setMetastoreSslConf(HiveConf conf) {
+    conf.setBoolVar(HiveConf.ConfVars.HIVE_METASTORE_USE_SSL, true);
+    conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PATH,
+            dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME);
+    conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PASSWORD,
+            KEY_STORE_TRUST_STORE_PASSWORD);
+    conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PATH,
+            dataFileDir + File.separator + TRUST_STORE_NAME);
+    conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PASSWORD,
+            KEY_STORE_TRUST_STORE_PASSWORD);
+  }
+
+  public static void clearSslConfOverlay(Map<String, String> confOverlay) {
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_USE_SSL.varname, "false");
+  }
+
+  public static void setHttpConfOverlay(Map<String, String> confOverlay) {
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE);
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT);
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true");
+  }
+
+  public static void setBinaryConfOverlay(Map<String, String> confOverlay) {
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_BINARY_MODE);
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION.varname, HS2_BINARY_AUTH_MODE);
+    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true");
+  }
+
+  public static void setupTestTableWithData(String tableName, Path dataFilePath,
+      Connection hs2Conn) throws Exception {
+    Statement stmt = hs2Conn.createStatement();
+    stmt.execute("set hive.support.concurrency = false");
+
+    stmt.execute("drop table if exists " + tableName);
+    stmt.execute("create table " + tableName
+        + " (under_col int comment 'the under column', value string)");
+
+    // load data
+    stmt.execute("load data local inpath '"
+        + dataFilePath.toString() + "' into table " + tableName);
+    stmt.close();
+  }
+
+  public static String getDataFileDir() {
+    return dataFileDir;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java
index c6a906a..bc00d11 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java
@@ -26,8 +26,6 @@ public class TestEmbeddedHiveMetaStore extends TestHiveMetaStore {
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-    hiveConf.setBoolean(
-        HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, true);
     warehouse = new Warehouse(hiveConf);
     try {
       client = new HiveMetaStoreClient(hiveConf);

http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index af125c3..b95c25c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -171,14 +171,6 @@ public abstract class TestHiveMetaStore extends TestCase {
       db = client.getDatabase(dbName);
       Path dbPath = new Path(db.getLocationUri());
       FileSystem fs = FileSystem.get(dbPath.toUri(), hiveConf);
-      boolean inheritPerms = hiveConf.getBoolVar(
-          HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
-      FsPermission dbPermission = fs.getFileStatus(dbPath).getPermission();
-      if (inheritPerms) {
-         //Set different perms for the database dir for further tests
-         dbPermission = new FsPermission((short)488);
-         fs.setPermission(dbPath, dbPermission);
-      }
 
       client.dropType(typeName);
       Type typ1 = new Type();
@@ -239,9 +231,6 @@ public abstract class TestHiveMetaStore extends TestCase {
         tbl = client.getTable(dbName, tblName);
       }
 
-      assertEquals(dbPermission, fs.getFileStatus(new Path(tbl.getSd().getLocation()))
-          .getPermission());
-
       Partition part = makePartitionObject(dbName, tblName, vals, tbl, "/part1");
       Partition part2 = makePartitionObject(dbName, tblName, vals2, tbl, "/part2");
       Partition part3 = makePartitionObject(dbName, tblName, vals3, tbl, "/part3");
@@ -259,20 +248,12 @@ public abstract class TestHiveMetaStore extends TestCase {
       assertTrue("getPartition() should have thrown NoSuchObjectException", exceptionThrown);
       Partition retp = client.add_partition(part);
       assertNotNull("Unable to create partition " + part, retp);
-      assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation()))
-          .getPermission());
       Partition retp2 = client.add_partition(part2);
       assertNotNull("Unable to create partition " + part2, retp2);
-      assertEquals(dbPermission, fs.getFileStatus(new Path(retp2.getSd().getLocation()))
-          .getPermission());
       Partition retp3 = client.add_partition(part3);
       assertNotNull("Unable to create partition " + part3, retp3);
-      assertEquals(dbPermission, fs.getFileStatus(new Path(retp3.getSd().getLocation()))
-          .getPermission());
       Partition retp4 = client.add_partition(part4);
       assertNotNull("Unable to create partition " + part4, retp4);
-      assertEquals(dbPermission, fs.getFileStatus(new Path(retp4.getSd().getLocation()))
-          .getPermission());
 
       Partition part_get = client.getPartition(dbName, tblName, part.getValues());
       if(isThriftClient) {
@@ -394,8 +375,6 @@ public abstract class TestHiveMetaStore extends TestCase {
       // tested
       retp = client.add_partition(part);
       assertNotNull("Unable to create partition " + part, retp);
-      assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation()))
-          .getPermission());
 
       // test add_partitions
 
@@ -431,9 +410,8 @@ public abstract class TestHiveMetaStore extends TestCase {
 
       // create dir for /mpart5
       Path mp5Path = new Path(mpart5.getSd().getLocation());
-      warehouse.mkdirs(mp5Path, true);
+      warehouse.mkdirs(mp5Path);
       assertTrue(fs.exists(mp5Path));
-      assertEquals(dbPermission, fs.getFileStatus(mp5Path).getPermission());
 
       // add_partitions(5,4) : err = duplicate keyvals on mpart4
       savedException = null;
@@ -2502,7 +2480,7 @@ public abstract class TestHiveMetaStore extends TestCase {
       //test params
       //test_param_2 = "50"
       filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS +
-          "test_param_2 = \"50\"";
+          "test_param_2 LIKE \"50\"";
 
       tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1);
       assertEquals(2, tableNames.size());
@@ -2511,30 +2489,31 @@ public abstract class TestHiveMetaStore extends TestCase {
 
       //test_param_2 = "75"
       filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS +
-          "test_param_2 = \"75\"";
+          "test_param_2 LIKE \"75\"";
 
       tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1);
       assertEquals(0, tableNames.size());
 
       //key_dne = "50"
       filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS +
-          "key_dne = \"50\"";
+          "key_dne LIKE \"50\"";
 
       tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1);
       assertEquals(0, tableNames.size());
 
       //test_param_1 != "yellow"
       filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS +
-          "test_param_1 <> \"yellow\"";
+          "test_param_1 NOT LIKE \"yellow\"";
 
-      tableNames = client.listTableNamesByFilter(dbName, filter, (short) 2);
-      assertEquals(2, tableNames.size());
+      // Commenting as part of HIVE-12274 != and <> are not supported for CLOBs
+      // tableNames = client.listTableNamesByFilter(dbName, filter, (short) 2);
+      // assertEquals(2, tableNames.size());
 
       filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS +
-          "test_param_1 != \"yellow\"";
+          "test_param_1 NOT LIKE \"yellow\"";
 
-      tableNames = client.listTableNamesByFilter(dbName, filter, (short) 2);
-      assertEquals(2, tableNames.size());
+      // tableNames = client.listTableNamesByFilter(dbName, filter, (short) 2);
+      // assertEquals(2, tableNames.size());
 
       //owner = "testOwner1" and (lastAccessTime = 30 or test_param_1 = "hi")
       filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_OWNER +
@@ -2542,7 +2521,7 @@ public abstract class TestHiveMetaStore extends TestCase {
         org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_LAST_ACCESS +
         " = 30 or " +
         org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS +
-        "test_param_1 = \"hi\")";
+        "test_param_1 LIKE \"hi\")";
       tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1);
 
       assertEquals(2, tableNames.size());

http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
index a0f18c6..1002be7 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
@@ -128,7 +128,7 @@ public class TestHiveMetaStoreTxns {
     Assert.assertEquals(ValidTxnList.RangeResponse.NONE,
         validTxns.isTxnRangeValid(5L, 10L));
 
-    validTxns = new ValidReadTxnList("10:5:4:5:6");
+    validTxns = new ValidReadTxnList("10:5:4,5,6:");
     Assert.assertEquals(ValidTxnList.RangeResponse.NONE,
         validTxns.isTxnRangeValid(4,6));
     Assert.assertEquals(ValidTxnList.RangeResponse.ALL,
@@ -223,15 +223,15 @@ public class TestHiveMetaStoreTxns {
   @Test
   public void stringifyValidTxns() throws Exception {
     // Test with just high water mark
-    ValidTxnList validTxns = new ValidReadTxnList("1:" + Long.MAX_VALUE + ":");
+    ValidTxnList validTxns = new ValidReadTxnList("1:" + Long.MAX_VALUE + "::");
     String asString = validTxns.toString();
-    Assert.assertEquals("1:" + Long.MAX_VALUE + ":", asString);
+    Assert.assertEquals("1:" + Long.MAX_VALUE + "::", asString);
     validTxns = new ValidReadTxnList(asString);
     Assert.assertEquals(1, validTxns.getHighWatermark());
     Assert.assertNotNull(validTxns.getInvalidTransactions());
     Assert.assertEquals(0, validTxns.getInvalidTransactions().length);
     asString = validTxns.toString();
-    Assert.assertEquals("1:" + Long.MAX_VALUE + ":", asString);
+    Assert.assertEquals("1:" + Long.MAX_VALUE + "::", asString);
     validTxns = new ValidReadTxnList(asString);
     Assert.assertEquals(1, validTxns.getHighWatermark());
     Assert.assertNotNull(validTxns.getInvalidTransactions());

http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
index 6d1673d..7188af6 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
@@ -189,13 +189,13 @@ public class TestMetastoreVersion extends TestCase {
 
   //  write the given version to metastore
   private String getVersion(HiveConf conf) throws HiveMetaException {
-    MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, conf, "derby");
+    MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, "derby");
     return getMetaStoreVersion();
   }
 
   //  write the given version to metastore
   private void setVersion(HiveConf conf, String version) throws HiveMetaException {
-    MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, conf, "derby");
+    MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, "derby");
     setMetaStoreVersion(version, "setVersion test");
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
index 1ac4d01..3f9eec3 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
@@ -151,15 +151,15 @@ public class TestReplChangeManager {
     Partition part3 = createPartition(dbName, tblName, columns, values, serdeInfo);
     client.add_partition(part3);
 
-    Path part1Path = new Path(warehouse.getPartitionPath(db, tblName, ImmutableMap.of("dt", "20160101")), "part");
+    Path part1Path = new Path(warehouse.getDefaultPartitionPath(db, tblName, ImmutableMap.of("dt", "20160101")), "part");
     createFile(part1Path, "p1");
     String path1Chksum = ReplChangeManager.getChksumString(part1Path, fs);
 
-    Path part2Path = new Path(warehouse.getPartitionPath(db, tblName, ImmutableMap.of("dt", "20160102")), "part");
+    Path part2Path = new Path(warehouse.getDefaultPartitionPath(db, tblName, ImmutableMap.of("dt", "20160102")), "part");
     createFile(part2Path, "p2");
     String path2Chksum = ReplChangeManager.getChksumString(part2Path, fs);
 
-    Path part3Path = new Path(warehouse.getPartitionPath(db, tblName, ImmutableMap.of("dt", "20160103")), "part");
+    Path part3Path = new Path(warehouse.getDefaultPartitionPath(db, tblName, ImmutableMap.of("dt", "20160103")), "part");
     createFile(part3Path, "p3");
     String path3Chksum = ReplChangeManager.getChksumString(part3Path, fs);
 
@@ -221,15 +221,15 @@ public class TestReplChangeManager {
 
     client.createTable(tbl);
 
-    Path filePath1 = new Path(warehouse.getTablePath(db, tblName), "part1");
+    Path filePath1 = new Path(warehouse.getDefaultTablePath(db, tblName), "part1");
     createFile(filePath1, "f1");
     String fileChksum1 = ReplChangeManager.getChksumString(filePath1, fs);
 
-    Path filePath2 = new Path(warehouse.getTablePath(db, tblName), "part2");
+    Path filePath2 = new Path(warehouse.getDefaultTablePath(db, tblName), "part2");
     createFile(filePath2, "f2");
     String fileChksum2 = ReplChangeManager.getChksumString(filePath2, fs);
 
-    Path filePath3 = new Path(warehouse.getTablePath(db, tblName), "part3");
+    Path filePath3 = new Path(warehouse.getDefaultTablePath(db, tblName), "part3");
     createFile(filePath3, "f3");
     String fileChksum3 = ReplChangeManager.getChksumString(filePath3, fs);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
new file mode 100644
index 0000000..abf9769
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
@@ -0,0 +1,436 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestAutoPurgeTables {
+  private static final String driverName = "org.apache.hive.jdbc.HiveDriver";
+  private static final String testDbName = "auto_purge_test_db";
+  //private static final String testTableName = "auto_purge_test_table";
+  private static final String INSERT_OVERWRITE_COMMAND_FORMAT =
+      "insert overwrite table " + testDbName + ".%s select 1, \"test\"";
+  private static final String TRUNCATE_TABLE_COMMAND_FORMAT =
+      "truncate table " + testDbName + ".%s";
+  private static final String partitionedColumnName = "partCol";
+  private static final String partitionedColumnValue1 = "20090619";
+  private static final String INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT =
+      "insert overwrite table " + testDbName + ".%s PARTITION ("
+          + partitionedColumnName + "=" + partitionedColumnValue1 + ")" + " select 1, \"test\"";
+  private static final String partitionedColumnValue2 = "20100720";
+  private static HiveConf conf;
+  private static Connection con;
+  private static MiniHS2 miniHS2;
+  static final private Logger LOG = LoggerFactory.getLogger("TestAutoPurgeTables");
+
+  @Rule
+  public TestName name = new TestName();
+
+  private static Connection getConnection(String url) throws SQLException {
+    Connection con1;
+    con1 = DriverManager.getConnection(url, "", "");
+    Assert.assertNotNull("Connection is null", con1);
+    Assert.assertFalse("Connection should not be closed", con1.isClosed());
+    return con1;
+  }
+
+  private static void createTestTable(Statement stmt, String isAutopurge, boolean isExternal,
+      boolean isPartitioned, String testTableName) throws SQLException {
+    String createTablePrefix;
+    if (isExternal) {
+      createTablePrefix = "create external table ";
+    } else {
+      createTablePrefix = "create table ";
+    }
+    if (isPartitioned) {
+      // create a partitioned table
+      stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value string) "
+          + " partitioned by (" + partitionedColumnName + " STRING)");
+      // load data
+      stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION ("
+          + partitionedColumnName + "=" + partitionedColumnValue1
+          + ") values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")");
+      stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION ("
+          + partitionedColumnName + "=" + partitionedColumnValue2
+          + ") values (4, \"dummy4\"), (5, \"dummy5\"), (6, \"dummy6\")");
+    } else {
+      // create a table
+      stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value string)");
+      // load data
+      stmt.execute("insert into " + testDbName + "." + testTableName
+          + " values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")");
+    }
+    if (isAutopurge != null) {
+      stmt.execute("alter table " + testDbName + "." + testTableName
+          + " set tblproperties (\"auto.purge\"=\"" + isAutopurge + "\")");
+    }
+  }
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    conf = new HiveConf(TestAutoPurgeTables.class);
+    // enable trash so it can be tested
+    conf.setFloat("fs.trash.checkpoint.interval", 30);
+    conf.setFloat("fs.trash.interval", 30);
+    // Create test database and base tables once for all the test
+    miniHS2 = new MiniHS2.Builder().withConf(conf).build();
+    miniHS2.start(new HashMap<String, String>());
+    Class.forName(driverName);
+    con = getConnection(miniHS2.getBaseJdbcURL() + ";create=true");
+    try (Statement stmt = con.createStatement()) {
+      Assert.assertNotNull("Statement is null", stmt);
+      stmt.execute("set hive.support.concurrency = false");
+      stmt.execute("drop database if exists " + testDbName + " cascade");
+      stmt.execute("create database " + testDbName);
+    }
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() {
+    Statement stmt = null;
+    try {
+      stmt = con.createStatement();
+      // drop test db and its tables and views
+      stmt.execute("set hive.support.concurrency = false");
+      stmt.execute("drop database if exists " + testDbName + " cascade");
+      FileSystem fs = FileSystem.get(conf);
+      fs.deleteOnExit(ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs));
+    } catch (SQLException | IOException e) {
+      e.printStackTrace();
+    } finally {
+      if (stmt != null) {
+        try {
+          stmt.close();
+        } catch (SQLException e) {
+          //
+        }
+      }
+      if (con != null) {
+        try {
+          con.close();
+        } catch (SQLException e) {
+          //
+        }
+      }
+      if (miniHS2 != null) {
+        miniHS2.cleanup();
+        miniHS2.stop();
+        miniHS2 = null;
+      }
+    }
+  }
+
+  @Before
+  public void afterTest() throws Exception {
+    FileSystem fs = FileSystem.get(conf);
+    Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs);
+    fs.delete(trashDir, true);
+  }
+
+  /**
+   * Tests if previous table data skips trash when insert overwrite table .. is run against a table
+   * which has auto.purge property set
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("true", false, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to a invalid string, trash should be used for insert overwrite
+   * queries
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testAutoPurgeInvalid() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("invalid", false, false, false, name.getMethodName());
+  }
+
+  /**
+   * Test when auto.purge property is not set. Data should be moved to trash for insert overwrite
+   * queries
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testAutoPurgeUnset() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(null, false, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests if the auto.purge property works correctly for external tables. Old data should skip
+   * trash when insert overwrite table .. is run when auto.purge is set to true
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testExternalTable() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("true", true, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests auto.purge when managed table is partitioned. Old data should skip trash when insert
+   * overwrite table .. is run and auto.purge property is set to true
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testPartitionedTable() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("true", false, true, false, name.getMethodName());
+  }
+
+  /**
+   * Tests auto.purge for an external, partitioned table. Old partition data should skip trash when
+   * auto.purge is set to true
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testExternalPartitionedTable() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("true", true, true, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to false, older data is moved to Trash when insert overwrite table
+   * .. is run
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("false", false, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to false on a external table, older data is moved to Trash when
+   * insert overwrite table .. is run
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testExternalNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("false", true, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to false on a partitioned table, older data is moved to Trash when
+   * insert overwrite table .. is run
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testPartitionedNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("false", false, true, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to false on a partitioned external table, older data is moved to
+   * Trash when insert overwrite table .. is run
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testPartitionedExternalNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("false", true, true, false, name.getMethodName());
+  }
+
+  //truncate on external table is not allowed
+  @Test(expected = SQLException.class)
+  public void testTruncatePartitionedExternalNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(false), true, true, true, name.getMethodName());
+  }
+
+  //truncate on external table is not allowed
+  @Test(expected = SQLException.class)
+  public void testTruncateExternalNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(false), true, false, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncatePartitionedNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(false), false, true, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncateNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(false), false, false, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncateInvalidAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("invalid", false, false, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncateUnsetAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(null, false, false, true, name.getMethodName());
+  }
+
+  //truncate on external table is not allowed
+  @Test(expected = SQLException.class)
+  public void testTruncatePartitionedExternalAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(true), true, true, true, name.getMethodName());
+  }
+
+  //truncate on external table is not allowed
+  @Test(expected = SQLException.class)
+  public void testTruncateExternalAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(true), true, false, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncatePartitionedAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(true), false, true, true, name.getMethodName());
+  }
+
+  /**
+   * Test util method to run the insert overwrite table or truncate table test on a table
+   * 
+   * @param autoPurgePropValue - string value of the auto.purge property for the test table. Ignored
+   *          if null
+   * @param isExternal - if set creates a external table for the test
+   * @param isPartitioned - if set creates a partitioned table for the test
+   * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses Insert
+   *          overwrite table command for the test
+   * @param testTableName - test table name
+   * @throws Exception
+   */
+  private void testUtil(String autoPurgePropValue, boolean isExternal, boolean isPartitioned,
+      boolean isTruncateTest, String testTableName) throws Exception {
+    testUtil(autoPurgePropValue, isExternal, isPartitioned,
+        !"true".equalsIgnoreCase(autoPurgePropValue), isTruncateTest, testTableName);
+  }
+  /**
+   * Test util method to run the insert overwrite table or truncate table test on a table
+   * 
+   * @param isAutoPurge - If set, creates a table with auto.purge with the given value
+   * @param isExternal - if set creates a external table for the test
+   * @param isPartitioned - if set creates a partitioned table for the test
+   * @param purgeExpected - if set the assert condition for the test is such that it expectes old
+   *          table data to be moved to trash. If not creates a assert condition to make sure that
+   *          data is not moved to trash
+   * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses Insert
+   *          overwrite table command for the test
+   * @param testTableName - table name for the test table
+   * @throws Exception
+   */
+  private void testUtil(String isAutoPurge, boolean isExternal, boolean isPartitioned,
+      boolean purgeExpected, boolean isTruncateTest, String testTableName) throws Exception {
+    try (Statement stmt = con.createStatement()) {
+      // create a test table with auto.purge = true
+      createTestTable(stmt, isAutoPurge, isExternal, isPartitioned, testTableName);
+      int numFilesInTrashBefore = getTrashFileCount();
+      String command = getCommand(isTruncateTest, isPartitioned, testTableName);
+      stmt.execute(command);
+      int numFilesInTrashAfter = getTrashFileCount();
+      if (purgeExpected) {
+        Assert.assertTrue(
+            String.format(
+                "Data should have been moved to trash. Number of files in trash: before : %d after %d",
+                numFilesInTrashBefore, numFilesInTrashAfter),
+            numFilesInTrashBefore < numFilesInTrashAfter);
+      } else {
+        Assert.assertEquals(
+            String.format(
+                "Data should not have been moved to trash. Number of files in trash: before : %d after %d",
+                numFilesInTrashBefore, numFilesInTrashAfter),
+            numFilesInTrashBefore, numFilesInTrashAfter);
+      }
+    }
+  }
+
+  private static String getCommand(boolean isTruncateTest, boolean isPartitioned, String testTableName) {
+    if (isTruncateTest) {
+      return String.format(TRUNCATE_TABLE_COMMAND_FORMAT, testTableName);
+    } else if (isPartitioned) {
+      return String.format(INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT, testTableName);
+    } else {
+      return String.format(INSERT_OVERWRITE_COMMAND_FORMAT, testTableName);
+    }
+  }
+
+  private int getTrashFileCount() throws Exception {
+    FileSystem fs = FileSystem.get(conf);
+    Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs);
+    return getFileCount(fs, trashDir);
+  }
+
+  private int getFileCount(FileSystem fs, Path path) throws Exception {
+    try {
+      int count = 0;
+      if (!fs.exists(path)) {
+        return count;
+      }
+      RemoteIterator<LocatedFileStatus> lfs = fs.listFiles(path, true);
+      while (lfs.hasNext()) {
+        LocatedFileStatus lf = lfs.next();
+        LOG.info(lf.getPath().toString());
+        if (lf.isFile()) {
+          count++;
+        }
+      }
+      return count;
+    } catch (IOException e) {
+      throw new Exception("Exception while list files on " + path, e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
index bfb25aa..ce8fe60 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
@@ -52,6 +52,7 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase {
   private static final String Table4Name = "table4_nondefault_nn";
   private static final String Table5Name = "table5_nondefault_nn";
   private static final String Table6Name = "table6_nondefault_nn";
+  private static final String Table7Name = "table7_nondefault_nn";
   private static final String Index1Name = "index1_table1_nondefault_nn";
   private static final String Index2Name = "index2_table1_nondefault_nn";
   private static final String tmpdir = System.getProperty("test.tmp.dir");
@@ -197,6 +198,27 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase {
     }
   }
 
+  private void alterPartitionAndCheck(Table table, String column,
+      String value, String location) throws CommandNeedRetryException, HiveException {
+    assertNotNull(location);
+    executeQuery("ALTER TABLE " + table.getTableName() +
+        " PARTITION (" + column + "='" + value + "')" +
+        " SET LOCATION '" + location + "'");
+    HashMap<String, String> partitions = new HashMap<String, String>();
+    partitions.put(column, value);
+    Partition partition = db.getPartition(table, partitions, false);
+    assertNotNull("Partition object is expected for " + table.getTableName() , partition);
+    String locationActual = partition.getLocation();
+    if (new Path(location).toUri().getScheme() != null) {
+      assertEquals("Partition should be located in the first filesystem",
+          fs.makeQualified(new Path(location)).toString(), locationActual);
+    }
+    else {
+      assertEquals("Partition should be located in the second filesystem",
+          fs2.makeQualified(new Path(location)).toString(), locationActual);
+    }
+  }
+
   private Table createTableAndCheck(String tableName, String tableLocation)
           throws CommandNeedRetryException, HiveException, URISyntaxException {
     return createTableAndCheck(null, tableName, tableLocation);
@@ -294,6 +316,15 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase {
     createTableAndCheck(table1, Table6Name, null);
   }
 
+  public void testAlterPartitionSetLocationNonDefaultNameNode() throws Exception {
+    assertTrue("Test suite should have been initialized", isInitialized);
+    String tableLocation = tmppathFs2 + "/" + "test_set_part_loc";
+    Table table = createTableAndCheck(Table7Name, tableLocation);
+
+    addPartitionAndCheck(table, "p", "p1", "/tmp/test/1");
+    alterPartitionAndCheck(table, "p", "p1", "/tmp/test/2");
+  }
+
   public void testCreateDatabaseWithTableNonDefaultNameNode() throws Exception {
     assertTrue("Test suite should be initialied", isInitialized );
     final String tableLocation = tmppathFs2 + "/" + Table3Name;

http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index 0688846..c17ca10 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -53,10 +53,11 @@ public class TestLocationQueries extends BaseTestQueries {
      * @return non-zero if it failed
      */
     @Override
-    public int checkCliDriverResults(String tname) throws Exception {
+    public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
       File logFile = new File(logDir, tname + ".out");
 
       int failedCount = 0;
+      StringBuilder fileNames = new StringBuilder("Files failing the location check:");
       FileReader fr = new FileReader(logFile);
       BufferedReader in = new BufferedReader(fr);
       try {
@@ -69,19 +70,20 @@ public class TestLocationQueries extends BaseTestQueries {
             File f = new File(m.group(1));
             if (!f.getName().equals(locationSubdir)) {
               failedCount++;
+              fileNames.append(f.getName()).append("\r\n");
             }
             locationCount++;
           }
         }
         // we always have to find at least one location, otw the test is useless
         if (locationCount == 0) {
-          return Integer.MAX_VALUE;
+          return QTestProcessExecResult.create(Integer.MAX_VALUE, "0 locations tested");
         }
       } finally {
         in.close();
       }
 
-      return failedCount;
+      return QTestProcessExecResult.create(failedCount, fileNames.toString());
     }
 
     public CheckResults(String outDir, String logDir, MiniClusterType miniMr,

http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
new file mode 100644
index 0000000..191d4a3
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
@@ -0,0 +1,319 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * This unit test is for testing HIVE-13884 with more complex queries and
+ * hive.metastore.limit.partition.request enabled.
+ * It covers cases when the query predicates can be pushed down and the
+ * number of partitions can be retrieved via directSQL.
+ * It also covers cases when the number of partitions cannot be retrieved
+ * via directSQL, so it falls back to ORM.
+ */
+public class TestMetaStoreLimitPartitionRequest {
+
+  private static final String DB_NAME = "max_partition_test_db";
+  private static final String TABLE_NAME = "max_partition_test_table";
+  private static int PARTITION_REQUEST_LIMIT = 4;
+  private static MiniHS2 miniHS2 = null;
+  private static HiveConf conf;
+  private Connection hs2Conn = null;
+  private Statement stmt;
+
+  @BeforeClass
+  public static void beforeTest() throws Exception {
+    Class.forName(MiniHS2.getJdbcDriverName());
+    conf = new HiveConf();
+    DriverManager.setLoginTimeout(0);
+
+    conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+    conf.setIntVar(HiveConf.ConfVars.METASTORE_LIMIT_PARTITION_REQUEST, PARTITION_REQUEST_LIMIT);
+    conf.setBoolVar(HiveConf.ConfVars.METASTORE_INTEGER_JDO_PUSHDOWN, true);
+    conf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, true);
+    conf.setBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING, true);
+    conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+    conf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_ENABLED, false);
+
+    miniHS2 = new MiniHS2.Builder().withConf(conf).build();
+    Map<String, String> overlayProps = new HashMap<String, String>();
+    miniHS2.start(overlayProps);
+    createDb();
+  }
+
+  private static void createDb() throws Exception {
+    Connection conn =
+        DriverManager.getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
+    Statement stmt2 = conn.createStatement();
+    stmt2.execute("DROP DATABASE IF EXISTS " + DB_NAME + " CASCADE");
+    stmt2.execute("CREATE DATABASE " + DB_NAME);
+    stmt2.close();
+    conn.close();
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(DB_NAME),
+        System.getProperty("user.name"), "bar");
+    stmt = hs2Conn.createStatement();
+    stmt.execute("USE " + DB_NAME);
+    createTable();
+  }
+
+  private void createTable() throws Exception {
+    String tmpTableName = TABLE_NAME + "_tmp";
+    stmt.execute("CREATE TABLE " + tmpTableName
+        + " (id string, value string, num string, ds date) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' STORED AS TEXTFILE");
+    stmt.execute("INSERT OVERWRITE TABLE " + tmpTableName
+        + " VALUES ('1', 'value1', '25', '2008-04-09'), ('2', 'value2', '30', '2008-04-09'), "
+        + "('3', 'value3', '35', '2008-04-09'), ('4', 'value4', '40', '2008-04-09'), "
+        + "('5', 'value5', '25', '2008-05-09'), ('6', 'value6', '30', '2008-05-09'), "
+        + "('7', 'value7', '35', '2008-05-09'), ('8', 'value8', '40', '2008-05-09'), "
+        + "('9', 'value9', '25', '2009-04-09'), ('10', 'value10', '30', '2009-04-09'), "
+        + "('11', 'value11', '35', '2009-04-09'), ('12', 'value12', '40', '2009-04-09')");
+
+    stmt.execute("CREATE TABLE " + TABLE_NAME + " (id string, value string) PARTITIONED BY (num string, ds date)");
+    stmt.execute("INSERT OVERWRITE TABLE " + TABLE_NAME + " PARTITION (num, ds) SELECT id, value, num, ds FROM " + tmpTableName);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    String tmpTableName = TABLE_NAME + "_tmp";
+    stmt.execute("DROP TABLE IF EXISTS " + TABLE_NAME);
+    stmt.execute("DROP TABLE IF EXISTS " + tmpTableName);
+    stmt.execute("DROP TABLE IF EXISTS " + TABLE_NAME + "_num_tmp");
+
+    if (hs2Conn != null) {
+      hs2Conn.close();
+    }
+  }
+
+  @AfterClass
+  public static void afterTest() throws Exception {
+    if (miniHS2 != null && miniHS2.isStarted()) {
+      miniHS2.stop();
+    }
+  }
+
+  /* Tests with queries which can be pushed down and executed with directSQL */
+
+  @Test
+  public void testSimpleQueryWithDirectSql() throws Exception {
+    String queryString = "select value from %s where num='25' and ds='2008-04-09'";
+    executeQuery(queryString, "value1");
+  }
+
+  @Test
+  public void testMoreComplexQueryWithDirectSql() throws Exception {
+    String queryString = "select value from %s where (ds between '2009-01-01' and '2009-12-31' and num='25') or (ds between '2008-01-01' and '2008-12-31' and num='30')";
+    executeQuery(queryString, "value2", "value6", "value9");
+  }
+
+  /*
+   * Tests with queries which can be pushed down and executed with directSQL, but the number of
+   * partitions which should be fetched is bigger than the maximum set by the
+   * hive.metastore.limit.partition.request parameter.
+   */
+
+  @Test
+  public void testSimpleQueryWithDirectSqlTooManyPartitions() throws Exception {
+    String queryString = "select value from %s where ds>'2008-04-20'";
+    executeQueryExceedPartitionLimit(queryString, 8);
+  }
+
+  @Test
+  public void testMoreComplexQueryWithDirectSqlTooManyPartitions() throws Exception {
+    String queryString = "select value from %s where num='25' or (num='30' and ds between '2008-01-01' and '2008-12-31')";
+    executeQueryExceedPartitionLimit(queryString, 5);
+  }
+
+  /*
+   * Tests with queries which cannot be executed with directSQL, because of type mismatch. The type
+   * of the num column is string, but the parameters used in the where clause are numbers. After
+   * falling back to ORM, the number of partitions can be fetched by the
+   * ObjectStore.getNumPartitionsViaOrmFilter method.
+   */
+
+  @Test
+  public void testQueryWithFallbackToORM1() throws Exception {
+    String queryString = "select value from %s where num!=25 and num!=35 and num!=40";
+    executeQuery(queryString, "value2", "value6", "value10");
+  }
+
+  @Test
+  public void testQueryWithFallbackToORMTooManyPartitions1() throws Exception {
+    String queryString = "select value from %s where num=30 or num=25";
+    executeQueryExceedPartitionLimit(queryString, 6);
+  }
+
+  /*
+   * Tests with queries which cannot be executed with directSQL, because of type mismatch. The type
+   * of the num column is string, but the parameters used in the where clause are numbers. After
+   * falling back to ORM the number of partitions cannot be fetched by the
+   * ObjectStore.getNumPartitionsViaOrmFilter method. They are fetched by the
+   * ObjectStore.getPartitionNamesPrunedByExprNoTxn method.
+   */
+
+  @Test
+  public void testQueryWithFallbackToORM2() throws Exception {
+    String queryString = "select value from %s where num!=25 and ds='2008-04-09'";
+    executeQuery(queryString, "value2", "value3", "value4");
+  }
+
+  @Test
+  public void testQueryWithFallbackToORM3() throws Exception {
+    String queryString = "select value from %s where num between 26 and 31";
+    executeQuery(queryString, "value2", "value6", "value10");
+  }
+
+  @Test
+  public void testQueryWithFallbackToORMTooManyPartitions2() throws Exception {
+    String queryString = "select value from %s where num!=25 and (ds='2008-04-09' or ds='2008-05-09')";
+    executeQueryExceedPartitionLimit(queryString, 6);
+  }
+
+  @Test
+  public void testQueryWithFallbackToORMTooManyPartitions3() throws Exception {
+    String queryString = "select value from %s where num>=30";
+    executeQueryExceedPartitionLimit(queryString, 9);
+  }
+
+  @Test
+  public void testQueryWithFallbackToORMTooManyPartitions4() throws Exception {
+    String queryString = "select value from %s where num between 20 and 50";
+    executeQueryExceedPartitionLimit(queryString, 12);
+  }
+
+  /*
+   * Tests with queries which cannot be executed with directSQL, because the contain like or in.
+   * After falling back to ORM the number of partitions cannot be fetched by the
+   * ObjectStore.getNumPartitionsViaOrmFilter method. They are fetched by the
+   * ObjectStore.getPartitionNamesPrunedByExprNoTxn method.
+   */
+
+  @Test
+  public void testQueryWithInWithFallbackToORM() throws Exception {
+    setupNumTmpTable();
+    String queryString = "select value from %s a where ds='2008-04-09' and a.num in (select value from " + TABLE_NAME + "_num_tmp)";
+    executeQuery(queryString, "value1", "value2");
+  }
+
+  @Test
+  public void testQueryWithInWithFallbackToORMTooManyPartitions() throws Exception {
+    setupNumTmpTable();
+    String queryString = "select value from %s a where a.num in (select value from " + TABLE_NAME + "_num_tmp)";
+    executeQueryExceedPartitionLimit(queryString, 12);
+  }
+
+  @Test
+  public void testQueryWithInWithFallbackToORMTooManyPartitions2() throws Exception {
+    setupNumTmpTable();
+    String queryString = "select value from %s a where a.num in (select value from " + TABLE_NAME + "_num_tmp where value='25')";
+    executeQueryExceedPartitionLimit(queryString, 12);
+  }
+
+  @Test
+  public void testQueryWithLikeWithFallbackToORMTooManyPartitions() throws Exception {
+    String queryString = "select value from %s where num like '3%%'";
+    executeQueryExceedPartitionLimit(queryString, 6);
+  }
+
+  private void setupNumTmpTable() throws SQLException {
+    stmt.execute("CREATE TABLE " + TABLE_NAME + "_num_tmp (value string)");
+    stmt.execute("INSERT INTO " + TABLE_NAME + "_num_tmp VALUES ('25')");
+    stmt.execute("INSERT INTO " + TABLE_NAME + "_num_tmp VALUES ('30')");
+  }
+
+  private void executeQuery(String query, String... expectedValues) throws SQLException {
+    String queryStr = String.format(query, TABLE_NAME);
+    ResultSet result = stmt.executeQuery(queryStr);
+    assertTrue(result != null);
+    Set<String> expectedValueSet = new HashSet<>(Arrays.asList(expectedValues));
+    Set<String> resultValues = getResultValues(result);
+    String errorMsg = getWrongResultErrorMsg(queryStr, expectedValueSet.toString(), resultValues.toString());
+    assertTrue(errorMsg, resultValues.equals(expectedValueSet));
+  }
+
+  private Set<String> getResultValues(ResultSet result) throws SQLException {
+    Set<String> resultValues = new HashSet<>();
+    while(result.next()) {
+      resultValues.add(result.getString(1));
+    }
+    return resultValues;
+  }
+
+  private void executeQueryExceedPartitionLimit(String query, int expectedPartitionNumber) throws Exception {
+    try {
+      String queryStr = String.format(query, TABLE_NAME);
+      stmt.executeQuery(queryStr);
+      fail("The query should have failed, because the number of requested partitions are bigger than "
+              + PARTITION_REQUEST_LIMIT);
+    } catch (HiveSQLException e) {
+      String exceedLimitMsg = String.format(HiveMetaStore.PARTITION_NUMBER_EXCEED_LIMIT_MSG, expectedPartitionNumber,
+          TABLE_NAME, PARTITION_REQUEST_LIMIT, ConfVars.METASTORE_LIMIT_PARTITION_REQUEST.varname);
+      assertTrue(getWrongExceptionMessage(exceedLimitMsg, e.getMessage()),
+          e.getMessage().contains(exceedLimitMsg.toString()));
+    }
+  }
+
+  private String getWrongResultErrorMsg(String query, String expectedValues, String resultValues) {
+    StringBuilder errorMsg = new StringBuilder();
+    errorMsg.append("The query '");
+    errorMsg.append(query);
+    errorMsg.append("' returned wrong values. It returned the values ");
+    errorMsg.append(resultValues);
+    errorMsg.append(" instead of the expected ");
+    errorMsg.append(expectedValues);
+    return errorMsg.toString();
+  }
+
+  private String getWrongExceptionMessage(String exceedLimitMsg, String exceptionMessage) {
+    StringBuilder errorMsg = new StringBuilder();
+    errorMsg.append("The message of the exception doesn't contain the expected '");
+    errorMsg.append(exceedLimitMsg.toString());
+    errorMsg.append("'. It is: ");
+    errorMsg.append(exceptionMessage);
+    return errorMsg.toString();
+  }
+
+}


Mime
View raw message