hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1663759 - in /hive/trunk: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
Date Tue, 03 Mar 2015 18:06:26 GMT
Author: hashutosh
Date: Tue Mar  3 18:06:26 2015
New Revision: 1663759

URL: http://svn.apache.org/r1663759
Log:
HIVE-9118 : Support auto-purge for tables, when dropping tables/partitions. (Mithun Radhakrishnan
via Ashutosh Chauhan)

Modified:
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1663759&r1=1663758&r2=1663759&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Tue
Mar  3 18:06:26 2015
@@ -1535,8 +1535,16 @@ public class HiveMetaStore extends Thrif
         if (!success) {
           ms.rollbackTransaction();
         } else if (deleteData && !isExternal) {
-          boolean ifPurge = envContext != null &&
-              Boolean.parseBoolean(envContext.getProperties().get("ifPurge"));
+          // Data needs deletion. Check if trash may be skipped.
+          // Trash may be skipped iff:
+          //  1. deleteData == true, obviously.
+          //  2. tbl is external.
+          //  3. Either
+          //    3.1. User has specified PURGE from the commandline, and if not,
+          //    3.2. User has set the table to auto-purge.
+          boolean ifPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")))
+                            ||
+                             (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge")));
           // Delete the data in the partitions which have other locations
           deletePartitionData(partPaths, ifPurge);
           // Delete the data in the table
@@ -2579,8 +2587,15 @@ public class HiveMetaStore extends Thrif
         } else if (deleteData && ((partPath != null) || (archiveParentDir != null)))
{
           if (tbl != null && !isExternal(tbl)) {
             // Data needs deletion. Check if trash may be skipped.
-            boolean mustPurge = (envContext != null)
-                                && Boolean.parseBoolean(envContext.getProperties().get("ifPurge"));
+            // Trash may be skipped iff:
+            //  1. deleteData == true, obviously.
+            //  2. tbl is external.
+            //  3. Either
+            //    3.1. User has specified PURGE from the commandline, and if not,
+            //    3.2. User has set the table to auto-purge.
+            boolean mustPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")))
+                                ||
+                                 (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge")));
             if (mustPurge) {
               LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash.");
             }
@@ -2741,8 +2756,15 @@ public class HiveMetaStore extends Thrif
           ms.rollbackTransaction();
         } else if (deleteData && !isExternal(tbl)) {
           // Data needs deletion. Check if trash may be skipped.
-          boolean mustPurge = (envContext != null)
-                              && Boolean.parseBoolean(envContext.getProperties().get("ifPurge"));
+          // Trash may be skipped iff:
+          //  1. deleteData == true, obviously.
+          //  2. tbl is external.
+          //  3. Either
+          //    3.1. User has specified PURGE from the commandline, and if not,
+          //    3.2. User has set the table to auto-purge.
+          boolean mustPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")))
+                              ||
+                              (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge")));
           LOG.info( mustPurge?
                       "dropPartition() will purge partition-directories directly, skipping
trash."
                     :  "dropPartition() will move partition-directories to trash-directory.");

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java?rev=1663759&r1=1663758&r2=1663759&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java Tue Mar  3 18:06:26
2015
@@ -447,7 +447,7 @@ public class TestHive extends TestCase {
     }
     catch (Exception exception) {
       fail("Unable to drop and create table " + dbName + "." + tableName
-           + " because " + StringUtils.stringifyException(exception));
+          + " because " + StringUtils.stringifyException(exception));
       throw exception;
     }
   }
@@ -522,6 +522,62 @@ public class TestHive extends TestCase {
     }
     finally {
       cleanUpTableQuietly(dbName, tableName);
+    }
+  }
+
+  /**
+   * Test that tables set up with auto-purge skip trash-directory when tables/partitions
are dropped.
+   * @throws Throwable
+   */
+  public void testAutoPurgeTablesAndPartitions() throws Throwable {
+
+    String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
+    String tableName = "table_for_testAutoPurgeTablesAndPartitions";
+    try {
+
+      Table table = createPartitionedTable(dbName, tableName);
+      table.getParameters().put("auto.purge", "true");
+      hm.alterTable(tableName, table);
+
+      Map<String, String> partitionSpec =  new ImmutableMap.Builder<String, String>()
+          .put("ds", "20141216")
+          .put("hr", "12")
+          .build();
+
+      int trashSizeBeforeDrop = getTrashContents().length;
+
+      hm.createPartition(table, partitionSpec);
+
+      Partition partition = hm.getPartition(table, partitionSpec, false);
+      assertNotNull("Newly created partition shouldn't be null!", partition);
+
+      hm.dropPartition(dbName, tableName,
+          partition.getValues(),
+          PartitionDropOptions.instance()
+                              .deleteData(true)
+                              .purgeData(false)
+      );
+
+      int trashSizeAfterDrop = getTrashContents().length;
+
+      assertEquals("After dropPartition(noPurge), data should still have skipped trash.",
+                 trashSizeBeforeDrop, trashSizeAfterDrop);
+
+      // Repeat the same check for dropTable.
+
+      trashSizeBeforeDrop = trashSizeAfterDrop;
+      hm.dropTable(dbName, tableName);
+      trashSizeAfterDrop = getTrashContents().length;
+
+      assertEquals("After dropTable(noPurge), data should still have skipped trash.",
+                 trashSizeBeforeDrop, trashSizeAfterDrop);
+
+    }
+    catch(Exception e) {
+      fail("Unexpected failure: " + StringUtils.stringifyException(e));
+    }
+    finally {
+      cleanUpTableQuietly(dbName, tableName);
     }
   }
 



Mime
View raw message