hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject [41/51] [abbrv] hive git commit: HIVE-13243: Hive drop table on encyption zone fails for external tables (Chaoyu Tang, reviewed by Sergio Pena)
Date Thu, 17 Mar 2016 22:47:45 GMT
HIVE-13243: Hive drop table on encyption zone fails for external tables (Chaoyu Tang, reviewed
by Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5b1adbb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5b1adbb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5b1adbb

Branch: refs/heads/llap
Commit: d5b1adbb6638acd74fd4162a6ed408ebd72abeae
Parents: f87b2b6
Author: ctang <ctang@cloudera.com>
Authored: Tue Mar 15 11:41:41 2016 -0400
Committer: ctang <ctang@cloudera.com>
Committed: Tue Mar 15 11:41:41 2016 -0400

----------------------------------------------------------------------
 .../hadoop/hive/metastore/HiveMetaStore.java    | 34 ++++++---
 .../clientpositive/encryption_drop_partition.q  | 10 +++
 .../clientpositive/encryption_drop_table.q      |  9 ++-
 .../encrypted/encryption_drop_partition.q.out   | 76 ++++++++++++++++++++
 .../encrypted/encryption_drop_table.q.out       | 27 +++++++
 5 files changed, 144 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index f0bc560..2fa0e9a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -1508,8 +1508,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
           }
         }
 
-        // tblPath will be null when tbl is a view. We skip the following if block in that
case.
-        checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge);
+        checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge, deleteData &&
!isExternal);
         // Drop the partitions and get a list of locations which need to be deleted
         partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath,
             tbl.getPartitionKeys(), deleteData && !isExternal);
@@ -1546,15 +1545,20 @@ public class HiveMetaStore extends ThriftHiveMetastore {
      * @param objectName db.table, or db.table.part
      * @param ifPurge if PURGE options is specified
      */
-    private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge)
-      throws MetaException {
-      if (!(pathToData != null && !ifPurge)) {//pathToData may be NULL for a view
+    private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge,
+        boolean deleteData) throws MetaException {
+      // There is no need to check TrashPurgeCombination in following cases since Purge/Trash
+      // is not applicable:
+      // a) deleteData is false -- drop an external table
+      // b) pathToData is null -- a view
+      // c) ifPurge is true -- force delete without Trash
+      if (!deleteData || pathToData == null || ifPurge) {
         return;
       }
 
       boolean trashEnabled = false;
       try {
-  trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1);
+        trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1);
       } catch(NumberFormatException ex) {
   // nothing to do
       }
@@ -2644,11 +2648,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
       boolean isArchived = false;
       Path archiveParentDir = null;
       boolean mustPurge = false;
+      boolean isExternalTbl = false;
 
       try {
         ms.openTransaction();
         part = ms.getPartition(db_name, tbl_name, part_vals);
         tbl = get_table_core(db_name, tbl_name);
+        isExternalTbl = isExternal(tbl);
         firePreEvent(new PreDropPartitionEvent(tbl, part, deleteData, this));
         mustPurge = isMustPurge(envContext, tbl);
 
@@ -2661,7 +2667,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
         if (isArchived) {
           archiveParentDir = MetaStoreUtils.getOriginalLocation(part);
           verifyIsWritablePath(archiveParentDir);
-          checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals,
mustPurge);
+          checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals,
+              mustPurge, deleteData && !isExternalTbl);
         }
         if (!ms.dropPartition(db_name, tbl_name, part_vals)) {
           throw new MetaException("Unable to drop partition");
@@ -2670,13 +2677,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
         if ((part.getSd() != null) && (part.getSd().getLocation() != null)) {
           partPath = new Path(part.getSd().getLocation());
           verifyIsWritablePath(partPath);
-          checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals,
mustPurge);
+          checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals,
+              mustPurge, deleteData && !isExternalTbl);
         }
       } finally {
         if (!success) {
           ms.rollbackTransaction();
         } else if (deleteData && ((partPath != null) || (archiveParentDir != null)))
{
-          if (tbl != null && !isExternal(tbl)) {
+          if (!isExternalTbl) {
             if (mustPurge) {
               LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash.");
             }
@@ -2761,10 +2769,12 @@ public class HiveMetaStore extends ThriftHiveMetastore {
       Table tbl = null;
       List<Partition> parts = null;
       boolean mustPurge = false;
+      boolean isExternalTbl = false;
       try {
         // We need Partition-s for firing events and for result; DN needs MPartition-s to
drop.
         // Great... Maybe we could bypass fetching MPartitions by issuing direct SQL deletes.
         tbl = get_table_core(dbName, tblName);
+        isExternalTbl = isExternal(tbl);
         mustPurge = isMustPurge(envContext, tbl);
         int minCount = 0;
         RequestPartsSpec spec = request.getParts();
@@ -2829,13 +2839,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
           if (MetaStoreUtils.isArchived(part)) {
             Path archiveParentDir = MetaStoreUtils.getOriginalLocation(part);
             verifyIsWritablePath(archiveParentDir);
-            checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." + part.getValues(),
mustPurge);
+            checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." +
+                part.getValues(), mustPurge, deleteData && !isExternalTbl);
             archToDelete.add(archiveParentDir);
           }
           if ((part.getSd() != null) && (part.getSd().getLocation() != null)) {
             Path partPath = new Path(part.getSd().getLocation());
             verifyIsWritablePath(partPath);
-            checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(),
mustPurge);
+            checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(),
+                mustPurge, deleteData && !isExternalTbl);
             dirsToDelete.add(new PathAndPartValSize(partPath, part.getValues().size()));
           }
         }

http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/queries/clientpositive/encryption_drop_partition.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_drop_partition.q b/ql/src/test/queries/clientpositive/encryption_drop_partition.q
index a26aa19..57dfabd 100644
--- a/ql/src/test/queries/clientpositive/encryption_drop_partition.q
+++ b/ql/src/test/queries/clientpositive/encryption_drop_partition.q
@@ -12,7 +12,17 @@ CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.
 
 INSERT INTO encrypted_table_dp PARTITION(p)(p,key,value) values('2014-09-23', 1, 'foo'),('2014-09-24',
2, 'bar');
 SELECT * FROM encrypted_table_dp;
+
+CREATE EXTERNAL TABLE encrypted_ext_table_dp (key INT, value STRING) partitioned by (p STRING)
LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp';
+ALTER TABLE encrypted_ext_table_dp ADD PARTITION (p='2014-09-23') LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp/p=2014-09-23';
+SELECT * FROM encrypted_ext_table_dp;
+ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23');
+SELECT * FROM encrypted_ext_table_dp;
+DROP TABLE encrypted_ext_table_dp;
+
+SELECT * FROM encrypted_table_dp;
 ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23');
 SELECT * FROM encrypted_table_dp;
 ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PURGE;
 SELECT * FROM encrypted_table_dp;
+DROP TABLE encrypted_table_dp PURGE;

http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/queries/clientpositive/encryption_drop_table.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table.q b/ql/src/test/queries/clientpositive/encryption_drop_table.q
index 193beea..2ae3c69 100644
--- a/ql/src/test/queries/clientpositive/encryption_drop_table.q
+++ b/ql/src/test/queries/clientpositive/encryption_drop_table.q
@@ -10,9 +10,16 @@ CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
 CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
 
 INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+
+CREATE EXTERNAL TABLE encrypted_ext_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+SHOW TABLES;
+
+DROP TABLE default.encrypted_ext_table;
 SHOW TABLES;
+
 DROP TABLE default.encrypted_table;
 SHOW TABLES;
+
 DROP TABLE default.encrypted_table PURGE;
 SHOW TABLES;
-CRYPTO DELETE_KEY --keyName key_128;
\ No newline at end of file
+CRYPTO DELETE_KEY --keyName key_128;

http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
index 067bf82..2643006 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
@@ -41,6 +41,74 @@ POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
 #### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
 1	foo	2014-09-23
 2	bar	2014-09-24
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_ext_table_dp
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_ext_table_dp
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@encrypted_ext_table_dp
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@encrypted_ext_table_dp
+POSTHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23
+PREHOOK: query: SELECT * FROM encrypted_ext_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_ext_table_dp
+PREHOOK: Input: default@encrypted_ext_table_dp@p=2014-09-23
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_ext_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_ext_table_dp
+POSTHOOK: Input: default@encrypted_ext_table_dp@p=2014-09-23
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+1	foo	2014-09-23
+PREHOOK: query: ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@encrypted_ext_table_dp
+PREHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23
+POSTHOOK: query: ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@encrypted_ext_table_dp
+POSTHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23
+PREHOOK: query: SELECT * FROM encrypted_ext_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_ext_table_dp
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_ext_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_ext_table_dp
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+PREHOOK: query: DROP TABLE encrypted_ext_table_dp
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_ext_table_dp
+PREHOOK: Output: default@encrypted_ext_table_dp
+POSTHOOK: query: DROP TABLE encrypted_ext_table_dp
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_ext_table_dp
+POSTHOOK: Output: default@encrypted_ext_table_dp
+PREHOOK: query: SELECT * FROM encrypted_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table_dp
+PREHOOK: Input: default@encrypted_table_dp@p=2014-09-23
+PREHOOK: Input: default@encrypted_table_dp@p=2014-09-24
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table_dp
+POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-23
+POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+1	foo	2014-09-23
+2	bar	2014-09-24
 PREHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23')
 PREHOOK: type: ALTERTABLE_DROPPARTS
 PREHOOK: Input: default@encrypted_table_dp
@@ -79,3 +147,11 @@ POSTHOOK: Input: default@encrypted_table_dp
 POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
 #### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
 2	bar	2014-09-24
+PREHOOK: query: DROP TABLE encrypted_table_dp PURGE
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table_dp
+PREHOOK: Output: default@encrypted_table_dp
+POSTHOOK: query: DROP TABLE encrypted_table_dp PURGE
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_table_dp
+POSTHOOK: Output: default@encrypted_table_dp

http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
index 55eefa0..c5007ee 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
@@ -24,6 +24,33 @@ POSTHOOK: Input: default@src
 POSTHOOK: Output: default@encrypted_table
 POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string,
comment:default), ]
 POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string,
comment:default), ]
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_ext_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_ext_table
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+encrypted_ext_table
+encrypted_table
+src
+PREHOOK: query: DROP TABLE default.encrypted_ext_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_ext_table
+PREHOOK: Output: default@encrypted_ext_table
+POSTHOOK: query: DROP TABLE default.encrypted_ext_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_ext_table
+POSTHOOK: Output: default@encrypted_ext_table
 PREHOOK: query: SHOW TABLES
 PREHOOK: type: SHOWTABLES
 PREHOOK: Input: database:default


Mime
View raw message