hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ct...@apache.org
Subject [1/2] hive git commit: HIVE-15880: Allow insert overwrite and truncate table query to use auto.purge table property (Vihang Karajgaonkar via Chaoyu Tang)
Date Sat, 01 Apr 2017 17:03:58 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2 7df55c932 -> 4cdef08f7


HIVE-15880: Allow insert overwrite and truncate table query to use auto.purge table property
(Vihang Karajgaonkar via Chaoyu Tang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e122de82
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e122de82
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e122de82

Branch: refs/heads/branch-2
Commit: e122de829d5780290454adb7fb571785814a44db
Parents: 7df55c9
Author: Chaoyu Tang <ctang@cloudera.com>
Authored: Sat Apr 1 10:49:43 2017 -0400
Committer: Chaoyu Tang <ctang@cloudera.com>
Committed: Sat Apr 1 12:58:10 2017 -0400

----------------------------------------------------------------------
 .../apache/hadoop/hive/common/FileUtils.java    |  15 +-
 .../hadoop/hive/ql/TestAutoPurgeTables.java     | 436 +++++++++++++++++++
 .../test/resources/testconfiguration.properties |   3 +-
 .../hive/metastore/HiveMetaStoreFsImpl.java     |  21 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   6 +-
 .../apache/hadoop/hive/ql/metadata/Hive.java    |  19 +-
 .../encryption_auto_purge_tables.q              |  38 ++
 .../encryption_auto_purge_tables.q.out          | 157 +++++++
 8 files changed, 661 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e122de82/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index 9a0521c..e586015 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -616,15 +616,19 @@ public final class FileUtils {
    * @return true if move successful
    * @throws IOException
    */
-  public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf)
+  public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf, boolean purge)
       throws IOException {
     LOG.debug("deleting  " + f);
     boolean result = false;
     try {
-      result = Trash.moveToAppropriateTrash(fs, f, conf);
-      if (result) {
-        LOG.trace("Moved to trash: " + f);
-        return true;
+      if(purge) {
+        LOG.debug("purge is set to true. Not moving to Trash " + f);
+      } else {
+        result = Trash.moveToAppropriateTrash(fs, f, conf);
+        if (result) {
+          LOG.trace("Moved to trash: " + f);
+          return true;
+        }
       }
     } catch (IOException ioe) {
       // for whatever failure reason including that trash has lower encryption zone
@@ -636,7 +640,6 @@ public final class FileUtils {
     if (!result) {
       LOG.error("Failed to delete " + f);
     }
-
     return result;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/e122de82/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
new file mode 100644
index 0000000..abf9769
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
@@ -0,0 +1,436 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestAutoPurgeTables {
+  private static final String driverName = "org.apache.hive.jdbc.HiveDriver";
+  private static final String testDbName = "auto_purge_test_db";
+  //private static final String testTableName = "auto_purge_test_table";
+  private static final String INSERT_OVERWRITE_COMMAND_FORMAT =
+      "insert overwrite table " + testDbName + ".%s select 1, \"test\"";
+  private static final String TRUNCATE_TABLE_COMMAND_FORMAT =
+      "truncate table " + testDbName + ".%s";
+  private static final String partitionedColumnName = "partCol";
+  private static final String partitionedColumnValue1 = "20090619";
+  private static final String INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT =
+      "insert overwrite table " + testDbName + ".%s PARTITION ("
+          + partitionedColumnName + "=" + partitionedColumnValue1 + ")" + " select 1, \"test\"";
+  private static final String partitionedColumnValue2 = "20100720";
+  private static HiveConf conf;
+  private static Connection con;
+  private static MiniHS2 miniHS2;
+  static final private Logger LOG = LoggerFactory.getLogger("TestAutoPurgeTables");
+
+  @Rule
+  public TestName name = new TestName();
+
+  private static Connection getConnection(String url) throws SQLException {
+    Connection con1;
+    con1 = DriverManager.getConnection(url, "", "");
+    Assert.assertNotNull("Connection is null", con1);
+    Assert.assertFalse("Connection should not be closed", con1.isClosed());
+    return con1;
+  }
+
+  private static void createTestTable(Statement stmt, String isAutopurge, boolean isExternal,
+      boolean isPartitioned, String testTableName) throws SQLException {
+    String createTablePrefix;
+    if (isExternal) {
+      createTablePrefix = "create external table ";
+    } else {
+      createTablePrefix = "create table ";
+    }
+    if (isPartitioned) {
+      // create a partitioned table
+      stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value
string) "
+          + " partitioned by (" + partitionedColumnName + " STRING)");
+      // load data
+      stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION ("
+          + partitionedColumnName + "=" + partitionedColumnValue1
+          + ") values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")");
+      stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION ("
+          + partitionedColumnName + "=" + partitionedColumnValue2
+          + ") values (4, \"dummy4\"), (5, \"dummy5\"), (6, \"dummy6\")");
+    } else {
+      // create a table
+      stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value
string)");
+      // load data
+      stmt.execute("insert into " + testDbName + "." + testTableName
+          + " values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")");
+    }
+    if (isAutopurge != null) {
+      stmt.execute("alter table " + testDbName + "." + testTableName
+          + " set tblproperties (\"auto.purge\"=\"" + isAutopurge + "\")");
+    }
+  }
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    conf = new HiveConf(TestAutoPurgeTables.class);
+    // enable trash so it can be tested
+    conf.setFloat("fs.trash.checkpoint.interval", 30);
+    conf.setFloat("fs.trash.interval", 30);
+    // Create test database and base tables once for all the test
+    miniHS2 = new MiniHS2.Builder().withConf(conf).build();
+    miniHS2.start(new HashMap<String, String>());
+    Class.forName(driverName);
+    con = getConnection(miniHS2.getBaseJdbcURL() + ";create=true");
+    try (Statement stmt = con.createStatement()) {
+      Assert.assertNotNull("Statement is null", stmt);
+      stmt.execute("set hive.support.concurrency = false");
+      stmt.execute("drop database if exists " + testDbName + " cascade");
+      stmt.execute("create database " + testDbName);
+    }
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() {
+    Statement stmt = null;
+    try {
+      stmt = con.createStatement();
+      // drop test db and its tables and views
+      stmt.execute("set hive.support.concurrency = false");
+      stmt.execute("drop database if exists " + testDbName + " cascade");
+      FileSystem fs = FileSystem.get(conf);
+      fs.deleteOnExit(ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs));
+    } catch (SQLException | IOException e) {
+      e.printStackTrace();
+    } finally {
+      if (stmt != null) {
+        try {
+          stmt.close();
+        } catch (SQLException e) {
+          //
+        }
+      }
+      if (con != null) {
+        try {
+          con.close();
+        } catch (SQLException e) {
+          //
+        }
+      }
+      if (miniHS2 != null) {
+        miniHS2.cleanup();
+        miniHS2.stop();
+        miniHS2 = null;
+      }
+    }
+  }
+
+  @Before
+  public void afterTest() throws Exception {
+    FileSystem fs = FileSystem.get(conf);
+    Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs);
+    fs.delete(trashDir, true);
+  }
+
+  /**
+   * Tests if previous table data skips trash when insert overwrite table .. is run against
a table
+   * which has auto.purge property set
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("true", false, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to a invalid string, trash should be used for insert overwrite
+   * queries
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testAutoPurgeInvalid() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("invalid", false, false, false, name.getMethodName());
+  }
+
+  /**
+   * Test when auto.purge property is not set. Data should be moved to trash for insert overwrite
+   * queries
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testAutoPurgeUnset() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(null, false, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests if the auto.purge property works correctly for external tables. Old data should
skip
+   * trash when insert overwrite table .. is run when auto.purge is set to true
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testExternalTable() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("true", true, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests auto.purge when managed table is partitioned. Old data should skip trash when
insert
+   * overwrite table .. is run and auto.purge property is set to true
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testPartitionedTable() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("true", false, true, false, name.getMethodName());
+  }
+
+  /**
+   * Tests auto.purge for an external, partitioned table. Old partition data should skip
trash when
+   * auto.purge is set to true
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testExternalPartitionedTable() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("true", true, true, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to false, older data is moved to Trash when insert overwrite
table
+   * .. is run
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("false", false, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to false on a external table, older data is moved to Trash
when
+   * insert overwrite table .. is run
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testExternalNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("false", true, false, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to false on a partitioned table, older data is moved to
Trash when
+   * insert overwrite table .. is run
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testPartitionedNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("false", false, true, false, name.getMethodName());
+  }
+
+  /**
+   * Tests when auto.purge is set to false on a partitioned external table, older data is
moved to
+   * Trash when insert overwrite table .. is run
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testPartitionedExternalNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("false", true, true, false, name.getMethodName());
+  }
+
+  //truncate on external table is not allowed
+  @Test(expected = SQLException.class)
+  public void testTruncatePartitionedExternalNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(false), true, true, true, name.getMethodName());
+  }
+
+  //truncate on external table is not allowed
+  @Test(expected = SQLException.class)
+  public void testTruncateExternalNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(false), true, false, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncatePartitionedNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(false), false, true, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncateNoAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(false), false, false, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncateInvalidAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil("invalid", false, false, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncateUnsetAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(null, false, false, true, name.getMethodName());
+  }
+
+  //truncate on external table is not allowed
+  @Test(expected = SQLException.class)
+  public void testTruncatePartitionedExternalAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(true), true, true, true, name.getMethodName());
+  }
+
+  //truncate on external table is not allowed
+  @Test(expected = SQLException.class)
+  public void testTruncateExternalAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(true), true, false, true, name.getMethodName());
+  }
+
+  @Test
+  public void testTruncatePartitionedAutoPurge() throws Exception {
+    LOG.info("Running " + name.getMethodName());
+    testUtil(String.valueOf(true), false, true, true, name.getMethodName());
+  }
+
+  /**
+   * Test util method to run the insert overwrite table or truncate table test on a table
+   * 
+   * @param autoPurgePropValue - string value of the auto.purge property for the test table.
Ignored
+   *          if null
+   * @param isExternal - if set creates a external table for the test
+   * @param isPartitioned - if set creates a partitioned table for the test
+   * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses
Insert
+   *          overwrite table command for the test
+   * @param testTableName - test table name
+   * @throws Exception
+   */
+  private void testUtil(String autoPurgePropValue, boolean isExternal, boolean isPartitioned,
+      boolean isTruncateTest, String testTableName) throws Exception {
+    testUtil(autoPurgePropValue, isExternal, isPartitioned,
+        !"true".equalsIgnoreCase(autoPurgePropValue), isTruncateTest, testTableName);
+  }
+  /**
+   * Test util method to run the insert overwrite table or truncate table test on a table
+   * 
+   * @param isAutoPurge - If set, creates a table with auto.purge with the given value
+   * @param isExternal - if set creates a external table for the test
+   * @param isPartitioned - if set creates a partitioned table for the test
+   * @param purgeExpected - if set the assert condition for the test is such that it expectes
old
+   *          table data to be moved to trash. If not creates a assert condition to make
sure that
+   *          data is not moved to trash
+   * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses
Insert
+   *          overwrite table command for the test
+   * @param testTableName - table name for the test table
+   * @throws Exception
+   */
+  private void testUtil(String isAutoPurge, boolean isExternal, boolean isPartitioned,
+      boolean purgeExpected, boolean isTruncateTest, String testTableName) throws Exception
{
+    try (Statement stmt = con.createStatement()) {
+      // create a test table with auto.purge = true
+      createTestTable(stmt, isAutoPurge, isExternal, isPartitioned, testTableName);
+      int numFilesInTrashBefore = getTrashFileCount();
+      String command = getCommand(isTruncateTest, isPartitioned, testTableName);
+      stmt.execute(command);
+      int numFilesInTrashAfter = getTrashFileCount();
+      if (purgeExpected) {
+        Assert.assertTrue(
+            String.format(
+                "Data should have been moved to trash. Number of files in trash: before :
%d after %d",
+                numFilesInTrashBefore, numFilesInTrashAfter),
+            numFilesInTrashBefore < numFilesInTrashAfter);
+      } else {
+        Assert.assertEquals(
+            String.format(
+                "Data should not have been moved to trash. Number of files in trash: before
: %d after %d",
+                numFilesInTrashBefore, numFilesInTrashAfter),
+            numFilesInTrashBefore, numFilesInTrashAfter);
+      }
+    }
+  }
+
+  private static String getCommand(boolean isTruncateTest, boolean isPartitioned, String
testTableName) {
+    if (isTruncateTest) {
+      return String.format(TRUNCATE_TABLE_COMMAND_FORMAT, testTableName);
+    } else if (isPartitioned) {
+      return String.format(INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT, testTableName);
+    } else {
+      return String.format(INSERT_OVERWRITE_COMMAND_FORMAT, testTableName);
+    }
+  }
+
+  private int getTrashFileCount() throws Exception {
+    FileSystem fs = FileSystem.get(conf);
+    Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs);
+    return getFileCount(fs, trashDir);
+  }
+
+  private int getFileCount(FileSystem fs, Path path) throws Exception {
+    try {
+      int count = 0;
+      if (!fs.exists(path)) {
+        return count;
+      }
+      RemoteIterator<LocatedFileStatus> lfs = fs.listFiles(path, true);
+      while (lfs.hasNext()) {
+        LocatedFileStatus lf = lfs.next();
+        LOG.info(lf.getPath().toString());
+        if (lf.isFile()) {
+          count++;
+        }
+      }
+      return count;
+    } catch (IOException e) {
+      throw new Exception("Exception while list files on " + path, e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/e122de82/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 3c98a57..7a70c9c 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -741,7 +741,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
   encryption_drop_view.q \
   encryption_drop_partition.q \
   encryption_with_trash.q \
-  encryption_ctas.q
+  encryption_ctas.q \
+  encryption_auto_purge_tables.q
 
 beeline.positive.include=drop_with_concurrency.q,\
   escape_comments.q

http://git-wip-us.apache.org/repos/asf/hive/blob/e122de82/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
index df698c8..b7d7b50 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
@@ -25,35 +25,23 @@ import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 
 public class HiveMetaStoreFsImpl implements MetaStoreFS {
 
   public static final Logger LOG = LoggerFactory
-      .getLogger("hive.metastore.hivemetastoressimpl");
+      .getLogger("hive.metastore.hivemetastoreFsimpl");
 
   @Override
   public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
       boolean ifPurge, Configuration conf) throws MetaException {
-    LOG.debug("deleting  " + f);
-
     try {
-      if (ifPurge) {
-        LOG.info("Not moving "+ f +" to trash");
-      } else if (Trash.moveToAppropriateTrash(fs, f, conf)) {
-        LOG.info("Moved to trash: " + f);
-        return true;
-      }
-
-      if (fs.delete(f, true)) {
-        LOG.debug("Deleted the diretory " + f);
-        return true;
-      }
-
+      FileUtils.moveToTrash(fs, f, conf, ifPurge);
       if (fs.exists(f)) {
         throw new MetaException("Unable to delete directory: " + f);
       }
+      return true;
     } catch (FileNotFoundException e) {
       return true; // ok even if there is not data
     } catch (Exception e) {
@@ -61,5 +49,4 @@ public class HiveMetaStoreFsImpl implements MetaStoreFS {
     }
     return false;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/e122de82/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 0769dad..917e565 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4662,7 +4662,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
     Map<String, String> partSpec = truncateTableDesc.getPartSpec();
 
     Table table = db.getTable(tableName, true);
-
+    boolean isAutopurge = "true".equalsIgnoreCase(table.getProperty("auto.purge"));
     try {
       // this is not transactional
       for (Path location : getLocations(db, table, partSpec)) {
@@ -4673,7 +4673,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
           HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location);
           FileStatus targetStatus = fs.getFileStatus(location);
           String targetGroup = targetStatus == null ? null : targetStatus.getGroup();
-          FileUtils.moveToTrash(fs, location, conf);
+          FileUtils.moveToTrash(fs, location, conf, isAutopurge);
           fs.mkdirs(location);
           HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false);
         } else {
@@ -4681,7 +4681,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
           if (statuses == null || statuses.length == 0) {
             continue;
           }
-          boolean success = Hive.trashFiles(fs, statuses, conf);
+          boolean success = Hive.trashFiles(fs, statuses, conf, isAutopurge);
           if (!success) {
             throw new HiveException("Error in deleting the contents of " + location.toString());
           }

http://git-wip-us.apache.org/repos/asf/hive/blob/e122de82/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ed854bf..f64cfda 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1646,8 +1646,9 @@ public class Hive {
       PerfLogger perfLogger = SessionState.getPerfLogger();
       perfLogger.PerfLogBegin("MoveTask", "FileMoves");
       if (replace || (oldPart == null && !isAcid)) {
+        boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
         replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(),
-            isSrcLocal);
+            isSrcLocal, isAutoPurge);
       } else {
         if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() &&
oldPart != null) {
           newFiles = Collections.synchronizedList(new ArrayList<Path>());
@@ -2012,7 +2013,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
     }
     if (replace) {
       Path tableDest = tbl.getPath();
-      replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal);
+      boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
+      replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal, isAutopurge);
     } else {
       FileSystem fs;
       try {
@@ -3398,11 +3400,13 @@ private void constructOneLBLocationMap(FileStatus fSta,
    * @param oldPath
    *          The directory where the old data location, need to be cleaned up.  Most of
time, will be the same
    *          as destf, unless its across FileSystem boundaries.
+   * @param purge
+   *          When set to true files which needs to be deleted are not moved to Trash
    * @param isSrcLocal
    *          If the source directory is LOCAL
    */
   protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, HiveConf
conf,
-          boolean isSrcLocal) throws HiveException {
+          boolean isSrcLocal, boolean purge) throws HiveException {
     try {
 
       FileSystem destFs = destf.getFileSystem(conf);
@@ -3435,7 +3439,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
             // existing content might result in incorrect (extra) data.
             // But not sure why we changed not to delete the oldPath in HIVE-8750 if it is
             // not the destf or its subdir?
-            oldPathDeleted = trashFiles(oldFs, statuses, conf);
+            oldPathDeleted = trashFiles(oldFs, statuses, conf, purge);
           }
         } catch (IOException e) {
           if (isOldPathUnderDestf) {
@@ -3495,7 +3499,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
    * @return true if deletion successful
    * @throws IOException
    */
-  public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final
Configuration conf)
+  public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses,
+      final Configuration conf, final boolean purge)
       throws IOException {
     boolean result = true;
 
@@ -3509,13 +3514,13 @@ private void constructOneLBLocationMap(FileStatus fSta,
     final SessionState parentSession = SessionState.get();
     for (final FileStatus status : statuses) {
       if (null == pool) {
-        result &= FileUtils.moveToTrash(fs, status.getPath(), conf);
+        result &= FileUtils.moveToTrash(fs, status.getPath(), conf, purge);
       } else {
         futures.add(pool.submit(new Callable<Boolean>() {
           @Override
           public Boolean call() throws Exception {
             SessionState.setCurrentSessionState(parentSession);
-            return FileUtils.moveToTrash(fs, status.getPath(), conf);
+            return FileUtils.moveToTrash(fs, status.getPath(), conf, purge);
           }
         }));
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/e122de82/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q b/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q
new file mode 100644
index 0000000..b96a0a0
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q
@@ -0,0 +1,38 @@
+-- SORT_QUERY_RESULTS;
+
+-- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after DROP
TABLE fails;
+
+DROP TABLE IF EXISTS encrypted_table PURGE;
+DROP TABLE IF EXISTS encrypted_ext_table PURGE;
+
+CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
+CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
+
+SHOW TABLES;
+
+ALTER TABLE encrypted_table SET TBLPROPERTIES("auto.purge"="true");
+
+INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+SELECT COUNT(*) from encrypted_table;
+
+TRUNCATE TABLE encrypted_table;
+SELECT COUNT(*) FROM encrypted_table;
+
+INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+SELECT COUNT(*) FROM encrypted_table;
+
+CREATE EXTERNAL TABLE encrypted_ext_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+ALTER TABLE encrypted_ext_table SET TBLPROPERTIES("auto.purge"="true");
+
+INSERT OVERWRITE TABLE encrypted_ext_table SELECT * FROM src;
+SELECT COUNT(*) from encrypted_ext_table;
+
+DROP TABLE encrypted_table;
+DROP TABLE encrypted_ext_table;
+SHOW TABLES;
+
+-- cleanup
+DROP TABLE IF EXISTS encrypted_table PURGE;
+DROP TABLE IF EXISTS encrypted_ext_table PURGE;
+CRYPTO DELETE_KEY --keyName key_128;

http://git-wip-us.apache.org/repos/asf/hive/blob/e122de82/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out
b/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out
new file mode 100644
index 0000000..a2d7dd2
--- /dev/null
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out
@@ -0,0 +1,157 @@
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_table
+Encryption key created: 'key_128'
+Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key:
'key_128'
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+encrypted_table
+src
+PREHOOK: query: ALTER TABLE encrypted_table SET TBLPROPERTIES("auto.purge"="true")
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: ALTER TABLE encrypted_table SET TBLPROPERTIES("auto.purge"="true")
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@encrypted_table
+POSTHOOK: Output: default@encrypted_table
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table
+POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string,
comment:default), ]
+POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string,
comment:default), ]
+PREHOOK: query: SELECT COUNT(*) from encrypted_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) from encrypted_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+500
+PREHOOK: query: TRUNCATE TABLE encrypted_table
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: TRUNCATE TABLE encrypted_table
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@encrypted_table
+PREHOOK: query: SELECT COUNT(*) FROM encrypted_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+0
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table
+POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string,
comment:default), ]
+POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string,
comment:default), ]
+PREHOOK: query: SELECT COUNT(*) FROM encrypted_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+500
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_ext_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_ext_table
+PREHOOK: query: ALTER TABLE encrypted_ext_table SET TBLPROPERTIES("auto.purge"="true")
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@encrypted_ext_table
+PREHOOK: Output: default@encrypted_ext_table
+POSTHOOK: query: ALTER TABLE encrypted_ext_table SET TBLPROPERTIES("auto.purge"="true")
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@encrypted_ext_table
+POSTHOOK: Output: default@encrypted_ext_table
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_ext_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_ext_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_ext_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_ext_table
+POSTHOOK: Lineage: encrypted_ext_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string,
comment:default), ]
+POSTHOOK: Lineage: encrypted_ext_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string,
comment:default), ]
+PREHOOK: query: SELECT COUNT(*) from encrypted_ext_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_ext_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) from encrypted_ext_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_ext_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+500
+PREHOOK: query: DROP TABLE encrypted_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: DROP TABLE encrypted_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_table
+POSTHOOK: Output: default@encrypted_table
+PREHOOK: query: DROP TABLE encrypted_ext_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_ext_table
+PREHOOK: Output: default@encrypted_ext_table
+POSTHOOK: query: DROP TABLE encrypted_ext_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_ext_table
+POSTHOOK: Output: default@encrypted_ext_table
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+src
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+POSTHOOK: type: DROPTABLE


Mime
View raw message