hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ekoif...@apache.org
Subject hive git commit: HIVE-10630:Renaming tables across encryption zones renames table even though the operation throws error (Eugene Koifman, reviewed by Sergio Pena and Ashutosh Chauhan)
Date Thu, 14 May 2015 17:29:14 GMT
Repository: hive
Updated Branches:
  refs/heads/master 05a3d2ab7 -> bc0138c43


HIVE-10630:Renaming tables across encryption zones renames table even though the operation
throws error (Eugene Koifman, reviewed by Sergio Pena and Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bc0138c4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bc0138c4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bc0138c4

Branch: refs/heads/master
Commit: bc0138c436add2335d2045b6c7bf86bc6a15cc27
Parents: 05a3d2a
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Thu May 14 10:29:00 2015 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Thu May 14 10:29:00 2015 -0700

----------------------------------------------------------------------
 .../test/resources/testconfiguration.properties |  3 +-
 .../org/apache/hadoop/hive/ql/QTestUtil.java    | 10 ++++-
 .../hadoop/hive/metastore/HiveAlterHandler.java | 28 ++++++++++---
 .../hadoop/hive/metastore/ObjectStore.java      |  2 +-
 .../clientpositive/encryption_move_tbl.q        | 18 ++++++++
 .../encrypted/encryption_move_tbl.q.out         | 43 ++++++++++++++++++++
 6 files changed, 96 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index eeb46cc..f9c9351 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -337,7 +337,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
   encryption_select_read_only_encrypted_tbl.q,\
   encryption_select_read_only_unencrypted_tbl.q,\
   encryption_load_data_to_encrypted_tables.q, \
-  encryption_unencrypted_nonhdfs_external_tables.q
+  encryption_unencrypted_nonhdfs_external_tables.q \
+  encryption_move_tbl.q
 
 beeline.positive.exclude=add_part_exist.q,\
   alter1.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 3e29d3c..f357c14 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1029,7 +1029,7 @@ public class QTestUtil {
         rc = cliDriver.processLine(command);
       }
 
-      if (rc != 0) {
+      if (rc != 0 && !ignoreErrors()) {
         break;
       }
       command = "";
@@ -1040,6 +1040,14 @@ public class QTestUtil {
     return rc;
   }
 
+  /**
+   * This allows a .q file to continue executing after a statement runs into an error which
is convenient
+   * if you want to use another hive cmd after the failure to sanity check the state of the
system.
+   */
+  private boolean ignoreErrors() {
+    return conf.getBoolVar(HiveConf.ConfVars.CLIIGNOREERRORS);
+  }
+
   private boolean isHiveCommand(String command) {
     String[] cmd = command.trim().split("\\s+");
     if (HiveCommand.find(cmd) != null) {

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 50ec1e8..a3f2359 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hive.common.util.HiveStringUtils;
 
 import com.google.common.collect.Lists;
@@ -260,17 +261,18 @@ public class HiveAlterHandler implements AlterHandler {
         // rename the src to destination
         try {
           if (srcFs.exists(srcPath) && !srcFs.rename(srcPath, destPath)) {
-            throw new IOException("Renaming " + srcPath + " to " + destPath + " is failed");
+            throw new IOException("Renaming " + srcPath + " to " + destPath + " failed");
           }
         } catch (IOException e) {
+          LOG.error("Alter Table operation for " + dbname + "." + name + " failed.", e);
           boolean revertMetaDataTransaction = false;
           try {
             msdb.openTransaction();
-            msdb.alterTable(dbname, newt.getTableName(), oldt);
+            msdb.alterTable(newt.getDbName(), newt.getTableName(), oldt);
             for (ObjectPair<Partition, String> pair : altps) {
               Partition part = pair.getFirst();
               part.getSd().setLocation(pair.getSecond());
-              msdb.alterPartition(dbname, name, part.getValues(), part);
+              msdb.alterPartition(newt.getDbName(), name, part.getValues(), part);
             }
             revertMetaDataTransaction = msdb.commitTransaction();
           } catch (Exception e1) {
@@ -288,8 +290,8 @@ public class HiveAlterHandler implements AlterHandler {
               msdb.rollbackTransaction();
             }
           }
-          throw new InvalidOperationException("Unable to access old location "
-              + srcPath + " for table " + dbname + "." + name);
+          throw new InvalidOperationException("Alter Table operation for " + dbname + "."
+ name +
+            " failed to move data due to: '" + getSimpleMessage(e) + "' See hive log file
for details.");
         }
       }
     }
@@ -298,6 +300,22 @@ public class HiveAlterHandler implements AlterHandler {
     }
   }
 
+  /**
+   * RemoteExceptionS from hadoop RPC wrap the stack trace into e.getMessage() which makes
+   * logs/stack traces confusing.
+   * @param ex
+   * @return
+   */
+  String getSimpleMessage(IOException ex) {
+    if(ex instanceof RemoteException) {
+      String msg = ex.getMessage();
+      if(msg == null || !msg.contains("\n")) {
+        return msg;
+      }
+      return msg.substring(0, msg.indexOf('\n'));
+    }
+    return ex.getMessage();
+  }
   public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
       final String name, final List<String> part_vals, final Partition new_part)
       throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 583e23f..fd61333 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -2758,7 +2758,7 @@ public class ObjectStore implements RawStore, Configurable {
 
       MTable oldt = getMTable(dbname, name);
       if (oldt == null) {
-        throw new MetaException("table " + name + " doesn't exist");
+        throw new MetaException("table " + dbname + "." + name + " doesn't exist");
       }
 
       // For now only alter name, owner, parameters, cols, bucketcols are allowed

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/ql/src/test/queries/clientpositive/encryption_move_tbl.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_move_tbl.q b/ql/src/test/queries/clientpositive/encryption_move_tbl.q
new file mode 100644
index 0000000..1168c6c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/encryption_move_tbl.q
@@ -0,0 +1,18 @@
+-- SORT_QUERY_RESULTS;
+
+-- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after ALTER
TABLE fails;
+
+set hive.cli.errors.ignore=true;
+
+DROP TABLE IF EXISTS encrypted_table;
+CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
+CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
+
+INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+SHOW TABLES;
+ALTER TABLE default.encrypted_table RENAME TO default.plain_table;
+SHOW TABLES;
+
+CRYPTO DELETE_KEY --keyName key_128;
+

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
new file mode 100644
index 0000000..d18827e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
@@ -0,0 +1,43 @@
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_table
+Encryption key created: 'key_128'
+Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key:
'key_128'
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table
+POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string,
comment:default), ]
+POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string,
comment:default), ]
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+encrypted_table
+src
+PREHOOK: query: ALTER TABLE default.encrypted_table RENAME TO default.plain_table
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable
to alter table. Alter Table operation for default.encrypted_table failed to move data due
to: '/build/ql/test/data/warehouse/default/encrypted_table can't be moved from an encryption
zone.' See hive log file for details.
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+encrypted_table
+src


Mime
View raw message