hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stak...@apache.org
Subject [11/14] hive git commit: HIVE-17189 : Fix backwards incompatibility in HiveMetaStoreClient (Vihang Karajgaonkar, reviewed by Alan Gates)
Date Thu, 09 Nov 2017 01:32:32 GMT
HIVE-17189 : Fix backwards incompatibility in HiveMetaStoreClient (Vihang Karajgaonkar, reviewed
by Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b8bd4594
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b8bd4594
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b8bd4594

Branch: refs/heads/branch-2.3
Commit: b8bd4594bef718b1eeac9fceb437d7df7b480ed1
Parents: 0ff4d5b
Author: Vihang Karajgaonkar <vihang@cloudera.com>
Authored: Thu Jul 27 16:23:47 2017 -0700
Committer: Sahil Takiar <stakiar@cloudera.com>
Committed: Tue Nov 7 08:15:47 2017 -0800

----------------------------------------------------------------------
 .../hive/metastore/TestHiveMetaStore.java       | 49 ++++++++++++++++++
 .../hive/metastore/HiveMetaStoreClient.java     | 23 +++++++++
 .../hadoop/hive/metastore/IMetaStoreClient.java | 52 +++++++++++++++++++-
 .../ql/metadata/SessionHiveMetaStoreClient.java | 13 +++++
 4 files changed, 136 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/b8bd4594/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 1b0b537..4b1df8e 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -608,6 +608,55 @@ public abstract class TestHiveMetaStore extends TestCase {
     assertEquals(" should have returned 50 partitions", maxParts, partitions.size());
   }
 
+  public void testAlterTableCascade() throws Throwable {
+    // create a table with multiple partitions
+    String dbName = "compdb";
+    String tblName = "comptbl";
+    String typeName = "Person";
+
+    cleanUp(dbName, tblName, typeName);
+
+    List<List<String>> values = new ArrayList<List<String>>();
+    values.add(makeVals("2008-07-01 14:13:12", "14"));
+    values.add(makeVals("2008-07-01 14:13:12", "15"));
+    values.add(makeVals("2008-07-02 14:13:12", "15"));
+    values.add(makeVals("2008-07-03 14:13:12", "151"));
+
+    createMultiPartitionTableSchema(dbName, tblName, typeName, values);
+    Table tbl = client.getTable(dbName, tblName);
+    List<FieldSchema> cols = tbl.getSd().getCols();
+    cols.add(new FieldSchema("new_col", serdeConstants.STRING_TYPE_NAME, ""));
+    tbl.getSd().setCols(cols);
+    //add new column with cascade option
+    client.alter_table(dbName, tblName, tbl, true);
+    //
+    Table tbl2 = client.getTable(dbName, tblName);
+    Assert.assertEquals("Unexpected number of cols", 3, tbl2.getSd().getCols().size());
+    Assert.assertEquals("Unexpected column name", "new_col", tbl2.getSd().getCols().get(2).getName());
+    //get a partition
+    List<String> pvalues = new ArrayList<>(2);
+    pvalues.add("2008-07-01 14:13:12");
+    pvalues.add("14");
+    Partition partition = client.getPartition(dbName, tblName, pvalues);
+    Assert.assertEquals("Unexpected number of cols", 3, partition.getSd().getCols().size());
+    Assert.assertEquals("Unexpected column name", "new_col", partition.getSd().getCols().get(2).getName());
+
+    //add another column
+    cols = tbl.getSd().getCols();
+    cols.add(new FieldSchema("new_col2", serdeConstants.STRING_TYPE_NAME, ""));
+    tbl.getSd().setCols(cols);
+    //add new column with no cascade option
+    client.alter_table(dbName, tblName, tbl, false);
+    tbl2 = client.getTable(dbName, tblName);
+    Assert.assertEquals("Unexpected number of cols", 4, tbl2.getSd().getCols().size());
+    Assert.assertEquals("Unexpected column name", "new_col2", tbl2.getSd().getCols().get(3).getName());
+    //get partition, this partition should not have the newly added column since cascade
option
+    //was false
+    partition = client.getPartition(dbName, tblName, pvalues);
+    Assert.assertEquals("Unexpected number of cols", 3, partition.getSd().getCols().size());
 
+  }
+
+
   public void testListPartitionNames() throws Throwable {
     // create a table with multiple partitions
     String dbName = "compdb";

http://git-wip-us.apache.org/repos/asf/hive/blob/b8bd4594/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index c5db77f..b1a9782 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -50,6 +50,7 @@ import java.security.PrivilegedExceptionAction;
 import javax.security.auth.login.LoginException;
 
 import org.apache.hadoop.hive.common.ObjectPair;
+import org.apache.hadoop.hive.common.StatsSetupConst;
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.common.auth.HiveAuthUtils;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience;
@@ -359,6 +360,16 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
   }
 
   @Override
+  public void alter_table(String defaultDatabaseName, String tblName, Table table,
+      boolean cascade) throws InvalidOperationException, MetaException, TException {
+    EnvironmentContext environmentContext = new EnvironmentContext();
+    if (cascade) {
+      environmentContext.putToProperties(StatsSetupConst.CASCADE, StatsSetupConst.TRUE);
+    }
+    alter_table_with_environmentContext(defaultDatabaseName, tblName, table, environmentContext);
+  }
+
+  @Override
   public void alter_table_with_environmentContext(String dbname, String tbl_name, Table new_tbl,
       EnvironmentContext envContext) throws InvalidOperationException, MetaException, TException
{
     client.alter_table_with_environment_context(dbname, tbl_name, new_tbl, envContext);
@@ -1493,12 +1504,24 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
   }
 
   @Override
+  public void alter_partition(String dbName, String tblName, Partition newPart)
+      throws InvalidOperationException, MetaException, TException {
+    client.alter_partition_with_environment_context(dbName, tblName, newPart, null);
+  }
+
+  @Override
   public void alter_partition(String dbName, String tblName, Partition newPart, EnvironmentContext
environmentContext)
       throws InvalidOperationException, MetaException, TException {
     client.alter_partition_with_environment_context(dbName, tblName, newPart, environmentContext);
   }
 
   @Override
+  public void alter_partitions(String dbName, String tblName, List<Partition> newParts)
+      throws InvalidOperationException, MetaException, TException {
+    client.alter_partitions_with_environment_context(dbName, tblName, newParts, null);
+  }
+
+  @Override
   public void alter_partitions(String dbName, String tblName, List<Partition> newParts,
EnvironmentContext environmentContext)
   throws InvalidOperationException, MetaException, TException {
     client.alter_partitions_with_environment_context(dbName, tblName, newParts, environmentContext);

http://git-wip-us.apache.org/repos/asf/hive/blob/b8bd4594/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index e9df1e1..e7ead6b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 
 import org.apache.hadoop.hive.common.ObjectPair;
+import org.apache.hadoop.hive.common.StatsSetupConst;
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
@@ -707,6 +708,14 @@ public interface IMetaStoreClient {
   void alter_table(String defaultDatabaseName, String tblName,
       Table table) throws InvalidOperationException, MetaException, TException;
 
+  /**
+   * Use alter_table_with_environmentContext instead of alter_table with cascade option
+   * passed in EnvironmentContext using {@code StatsSetupConst.CASCADE}
+   */
+  @Deprecated
+  void alter_table(String defaultDatabaseName, String tblName, Table table,
+      boolean cascade) throws InvalidOperationException, MetaException, TException;
+
   //wrapper of alter_table_with_cascade
   void alter_table_with_environmentContext(String defaultDatabaseName, String tblName, Table
table,
       EnvironmentContext environmentContext) throws InvalidOperationException, MetaException,
@@ -780,6 +789,26 @@ public interface IMetaStoreClient {
   boolean dropPartition(String db_name, String tbl_name,
       String name, boolean deleteData) throws NoSuchObjectException,
       MetaException, TException;
+
+  /**
+   * updates a partition to new partition
+   *
+   * @param dbName
+   *          database of the old partition
+   * @param tblName
+   *          table name of the old partition
+   * @param newPart
+   *          new partition
+   * @throws InvalidOperationException
+   *           if the old partition does not exist
+   * @throws MetaException
+   *           if error in updating metadata
+   * @throws TException
+   *           if error in communicating with metastore server
+   */
+  void alter_partition(String dbName, String tblName, Partition newPart)
+      throws InvalidOperationException, MetaException, TException;
+
   /**
    * updates a partition to new partition
    *
@@ -815,7 +844,28 @@ public interface IMetaStoreClient {
    * @throws TException
    *           if error in communicating with metastore server
    */
-  void alter_partitions(String dbName, String tblName, List<Partition> newParts, EnvironmentContext
environmentContext)
+  void alter_partitions(String dbName, String tblName, List<Partition> newParts)
+      throws InvalidOperationException, MetaException, TException;
+
+  /**
+   * updates a list of partitions
+   *
+   * @param dbName
+   *          database of the old partition
+   * @param tblName
+   *          table name of the old partition
+   * @param newParts
+   *          list of partitions
+   * @param environmentContext 
+   * @throws InvalidOperationException
+   *           if the old partition does not exist
+   * @throws MetaException
+   *           if error in updating metadata
+   * @throws TException
+   *           if error in communicating with metastore server
+   */
+  void alter_partitions(String dbName, String tblName, List<Partition> newParts,
+      EnvironmentContext environmentContext)
       throws InvalidOperationException, MetaException, TException;
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/b8bd4594/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 8eb011e..109bc3a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -280,6 +280,19 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements
I
     return super.getSchema(dbName, tableName);
   }
 
+  @Deprecated
+  @Override
+  public void alter_table(String dbname, String tbl_name, org.apache.hadoop.hive.metastore.api.Table
new_tbl,
+      boolean cascade) throws InvalidOperationException, MetaException, TException {
+    org.apache.hadoop.hive.metastore.api.Table old_tbl = getTempTable(dbname, tbl_name);
+    if (old_tbl != null) {
+      //actually temp table does not support partitions, cascade is not applicable here
+      alterTempTable(dbname, tbl_name, old_tbl, new_tbl, null);
+      return;
+    }
+    super.alter_table(dbname, tbl_name, new_tbl, cascade);
+  }
+
   @Override
   public void alter_table(String dbname, String tbl_name,
       org.apache.hadoop.hive.metastore.api.Table new_tbl) throws InvalidOperationException,


Mime
View raw message