hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ct...@apache.org
Subject [3/4] hive git commit: HIVE-16147: Rename a partitioned table should not drop its partition columns stats (Chaoyu Tang, reviewed by Pengcheng Xiong)
Date Mon, 01 May 2017 13:58:40 GMT
HIVE-16147: Rename a partitioned table should not drop its partition columns stats (Chaoyu
Tang, reviewed by Pengcheng Xiong)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/57de93a8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/57de93a8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/57de93a8

Branch: refs/heads/branch-2
Commit: 57de93a882ecb34d269ed1448229604a6abbb684
Parents: e29528c
Author: Chaoyu Tang <ctang@cloudera.com>
Authored: Mon May 1 09:39:38 2017 -0400
Committer: Chaoyu Tang <ctang@cloudera.com>
Committed: Mon May 1 09:54:58 2017 -0400

----------------------------------------------------------------------
 .../hadoop/hive/metastore/HiveAlterHandler.java |  306 +-
 .../hadoop/hive/metastore/MetaStoreUtils.java   |    8 -
 .../hadoop/hive/metastore/ObjectStore.java      |    7 +-
 .../clientpositive/alter_table_column_stats.q   |  241 ++
 .../alter_table_invalidate_column_stats.q       |  153 -
 .../alter_table_column_stats.q.out              | 2706 ++++++++++++++++++
 .../alter_table_invalidate_column_stats.q.out   |  932 ------
 7 files changed, 3120 insertions(+), 1233 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/57de93a8/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 15f2597..d281c03 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -54,8 +54,11 @@ import org.apache.hive.common.util.HiveStringUtils;
 import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 
 /**
  * Hive specific implementation of alter
@@ -88,6 +91,9 @@ public class HiveAlterHandler implements AlterHandler {
   public void alterTable(RawStore msdb, Warehouse wh, String dbname,
       String name, Table newt, EnvironmentContext environmentContext,
       HMSHandler handler) throws InvalidOperationException, MetaException {
+    name = name.toLowerCase();
+    dbname = dbname.toLowerCase();
+
     final boolean cascade = environmentContext != null
         && environmentContext.isSetProperties()
         && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get(
@@ -96,9 +102,11 @@ public class HiveAlterHandler implements AlterHandler {
       throw new InvalidOperationException("New table is invalid: " + newt);
     }
 
-    if (!MetaStoreUtils.validateName(newt.getTableName(), hiveConf)) {
-      throw new InvalidOperationException(newt.getTableName()
-          + " is not a valid object name");
+    String newTblName = newt.getTableName().toLowerCase();
+    String newDbName = newt.getDbName().toLowerCase();
+
+    if (!MetaStoreUtils.validateName(newTblName, hiveConf)) {
+      throw new InvalidOperationException(newTblName + " is not a valid object name");
     }
     String validate = MetaStoreUtils.validateTblColumns(newt.getSd().getCols());
     if (validate != null) {
@@ -112,7 +120,6 @@ public class HiveAlterHandler implements AlterHandler {
 
     boolean success = false;
     boolean dataWasMoved = false;
-    boolean rename = false;
     Table oldt = null;
     List<MetaStoreEventListener> transactionalListeners = null;
     if (handler != null) {
@@ -120,26 +127,30 @@ public class HiveAlterHandler implements AlterHandler {
     }
 
     try {
-      msdb.openTransaction();
-      name = name.toLowerCase();
-      dbname = dbname.toLowerCase();
+      boolean rename = false;
+      boolean isPartitionedTable = false;
+      List<Partition> parts = null;
 
       // check if table with the new name already exists
-      if (!newt.getTableName().equalsIgnoreCase(name)
-          || !newt.getDbName().equalsIgnoreCase(dbname)) {
-        if (msdb.getTable(newt.getDbName(), newt.getTableName()) != null) {
-          throw new InvalidOperationException("new table " + newt.getDbName()
-              + "." + newt.getTableName() + " already exists");
+      if (!newTblName.equals(name) || !newDbName.equals(dbname)) {
+        if (msdb.getTable(newDbName, newTblName) != null) {
+          throw new InvalidOperationException("new table " + newDbName
+              + "." + newTblName + " already exists");
         }
         rename = true;
       }
 
+      msdb.openTransaction();
       // get old table
       oldt = msdb.getTable(dbname, name);
       if (oldt == null) {
         throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist");
       }
 
+      if (oldt.getPartitionKeysSize() != 0) {
+        isPartitionedTable = true;
+      }
+
       if (HiveConf.getBoolVar(hiveConf,
             HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES,
             false)) {
@@ -149,32 +160,14 @@ public class HiveAlterHandler implements AlterHandler {
             oldt.getSd().getCols(), newt.getSd().getCols());
       }
 
-      if (cascade) {
-        //Currently only column related changes can be cascaded in alter table
-        if(MetaStoreUtils.isCascadeNeededInAlterTable(oldt, newt)) {
-          List<Partition> parts = msdb.getPartitions(dbname, name, -1);
-          for (Partition part : parts) {
-            List<FieldSchema> oldCols = part.getSd().getCols();
-            part.getSd().setCols(newt.getSd().getCols());
-            String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
-            updatePartColumnStatsForAlterColumns(msdb, part, oldPartName, part.getValues(),
oldCols, part);
-            msdb.alterPartition(dbname, name, part.getValues(), part);
-          }
-        } else {
-          LOG.warn("Alter table does not cascade changes to its partitions.");
-        }
-      }
-
       //check that partition keys have not changed, except for virtual views
       //however, allow the partition comments to change
       boolean partKeysPartiallyEqual = checkPartialPartKeysEqual(oldt.getPartitionKeys(),
           newt.getPartitionKeys());
 
       if(!oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())){
-        if (oldt.getPartitionKeys().size() != newt.getPartitionKeys().size()
-            || !partKeysPartiallyEqual) {
-          throw new InvalidOperationException(
-              "partition keys can not be changed.");
+        if (!partKeysPartiallyEqual) {
+          throw new InvalidOperationException("partition keys can not be changed.");
         }
       }
 
@@ -205,9 +198,9 @@ public class HiveAlterHandler implements AlterHandler {
           srcFs = wh.getFs(srcPath);
 
           // get new location
-          Database db = msdb.getDatabase(newt.getDbName());
+          Database db = msdb.getDatabase(newDbName);
           Path databasePath = constructRenamedPath(wh.getDatabasePath(db), srcPath);
-          destPath = new Path(databasePath, newt.getTableName().toLowerCase());
+          destPath = new Path(databasePath, newTblName);
           destFs = wh.getFs(destPath);
 
           newt.getSd().setLocation(destPath.toString());
@@ -224,8 +217,7 @@ public class HiveAlterHandler implements AlterHandler {
           try {
             if (destFs.exists(destPath)) {
               throw new InvalidOperationException("New location for this table "
-                  + newt.getDbName() + "." + newt.getTableName()
-                  + " already exists : " + destPath);
+                  + newDbName + "." + newTblName + " already exists : " + destPath);
             }
             // check that src exists and also checks permissions necessary, rename src to
dest
             if (srcFs.exists(srcPath) && srcFs.rename(srcPath, destPath)) {
@@ -237,38 +229,78 @@ public class HiveAlterHandler implements AlterHandler {
                 " failed to move data due to: '" + getSimpleMessage(e)
                 + "' See hive log file for details.");
           }
+        }
+
+        if (isPartitionedTable) {
           String oldTblLocPath = srcPath.toUri().getPath();
-          String newTblLocPath = destPath.toUri().getPath();
+          String newTblLocPath = dataWasMoved ? destPath.toUri().getPath() : null;
 
           // also the location field in partition
-          List<Partition> parts = msdb.getPartitions(dbname, name, -1);
+          parts = msdb.getPartitions(dbname, name, -1);
+          Map<Partition, ColumnStatistics> columnStatsNeedUpdated = new HashMap<Partition,
ColumnStatistics>();
           for (Partition part : parts) {
             String oldPartLoc = part.getSd().getLocation();
-            if (oldPartLoc.contains(oldTblLocPath)) {
+            if (dataWasMoved && oldPartLoc.contains(oldTblLocPath)) {
               URI oldUri = new Path(oldPartLoc).toUri();
               String newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath);
               Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
               part.getSd().setLocation(newPartLocPath.toString());
-              String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
-              try {
-                //existing partition column stats is no longer valid, remove them
-                msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, part.getValues(),
null);
-              } catch (InvalidInputException iie) {
-                throw new InvalidOperationException("Unable to update partition stats in
table rename." + iie);
-              }
-              msdb.alterPartition(dbname, name, part.getValues(), part);
             }
+            part.setDbName(newDbName);
+            part.setTableName(newTblName);
+            ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name,
+                part.getValues(), part.getSd().getCols(), oldt, part);
+            if (colStats != null) {
+              columnStatsNeedUpdated.put(part, colStats);
+            }
+          }
+          msdb.alterTable(dbname, name, newt);
+          // alterPartition is only for changing the partition location in the table rename
+          if (dataWasMoved) {
+            for (Partition part : parts) {
+              msdb.alterPartition(newDbName, newTblName, part.getValues(), part);
+            }
+          }
+
+          for (Entry<Partition, ColumnStatistics> partColStats : columnStatsNeedUpdated.entrySet())
{
+            ColumnStatistics newPartColStats = partColStats.getValue();
+            newPartColStats.getStatsDesc().setDbName(newDbName);
+            newPartColStats.getStatsDesc().setTableName(newTblName);
+            msdb.updatePartitionColumnStatistics(newPartColStats, partColStats.getKey().getValues());
           }
+        } else {
+          alterTableUpdateTableColumnStats(msdb, oldt, newt);
         }
-      } else if (MetaStoreUtils.requireCalStats(hiveConf, null, null, newt, environmentContext)
&&
-        (newt.getPartitionKeysSize() == 0)) {
-          Database db = msdb.getDatabase(newt.getDbName());
-          // Update table stats. For partitioned table, we update stats in
-          // alterPartition()
+      } else {
+        // operations other than table rename
+        if (MetaStoreUtils.requireCalStats(hiveConf, null, null, newt, environmentContext)
&&
+            !isPartitionedTable) {
+          Database db = msdb.getDatabase(newDbName);
+          // Update table stats. For partitioned table, we update stats in alterPartition()
           MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true, environmentContext);
+        }
+
+        if (cascade && isPartitionedTable) {
+          //Currently only column related changes can be cascaded in alter table
+          if(!MetaStoreUtils.areSameColumns(oldt.getSd().getCols(), newt.getSd().getCols()))
{
+            parts = msdb.getPartitions(dbname, name, -1);
+            for (Partition part : parts) {
+              List<FieldSchema> oldCols = part.getSd().getCols();
+              part.getSd().setCols(newt.getSd().getCols());
+              ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name,
+                  part.getValues(), oldCols, oldt, part);
+              assert(colStats == null);
+              msdb.alterPartition(dbname, name, part.getValues(), part);
+            }
+            msdb.alterTable(dbname, name, newt);
+          } else {
+            LOG.warn("Alter table does not cascade changes to its partitions.");
+          }
+        } else {
+          alterTableUpdateTableColumnStats(msdb, oldt, newt);
+        }
       }
 
-      alterTableUpdateTableColumnStats(msdb, oldt, newt);
       if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
         MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
                                               EventMessage.EventType.ALTER_TABLE,
@@ -282,6 +314,11 @@ public class HiveAlterHandler implements AlterHandler {
       throw new InvalidOperationException(
           "Unable to change partition or table."
               + " Check metastore logs for detailed stack." + e.getMessage());
+    } catch (InvalidInputException e) {
+        LOG.debug("Accessing Metastore failed due to invalid input ", e);
+        throw new InvalidOperationException(
+            "Unable to change partition or table."
+                + " Check metastore logs for detailed stack." + e.getMessage());
     } catch (NoSuchObjectException e) {
       LOG.debug("Object not found in metastore ", e);
       throw new InvalidOperationException(
@@ -379,7 +416,11 @@ public class HiveAlterHandler implements AlterHandler {
           }
         }
 
-        updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part);
+        // PartitionView does not have SD. We do not need update its column stats
+        if (oldPart.getSd() != null) {
+          updateOrGetPartitionColumnStats(msdb, dbname, name, new_part.getValues(),
+              oldPart.getSd().getCols(), tbl, new_part);
+        }
         msdb.alterPartition(dbname, name, new_part.getValues(), new_part);
         if (transactionalListeners != null && !transactionalListeners.isEmpty())
{
           MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
@@ -608,7 +649,12 @@ public class HiveAlterHandler implements AlterHandler {
             MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true, environmentContext);
           }
         }
-        updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart);
+
+        // PartitionView does not have SD and we do not need to update its column stats
+        if (oldTmpPart.getSd() != null) {
+          updateOrGetPartitionColumnStats(msdb, dbname, name, oldTmpPart.getValues(),
+              oldTmpPart.getSd().getCols(), tbl, tmpPart);
+        }
       }
 
       msdb.alterPartitions(dbname, name, partValsList, new_parts);
@@ -678,91 +724,8 @@ public class HiveAlterHandler implements AlterHandler {
         defaultNewPath.toUri().getPath());
   }
 
-  private void updatePartColumnStatsForAlterColumns(RawStore msdb, Partition oldPartition,
-      String oldPartName, List<String> partVals, List<FieldSchema> oldCols, Partition
newPart)
-          throws MetaException, InvalidObjectException {
-    String dbName = oldPartition.getDbName();
-    String tableName = oldPartition.getTableName();
-    try {
-      List<String> oldPartNames = Lists.newArrayList(oldPartName);
-      List<String> oldColNames = new ArrayList<String>(oldCols.size());
-      for (FieldSchema oldCol : oldCols) {
-        oldColNames.add(oldCol.getName());
-      }
-      List<FieldSchema> newCols = newPart.getSd().getCols();
-      List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(dbName,
tableName,
-          oldPartNames, oldColNames);
-      assert (partsColStats.size() <= 1);
-      for (ColumnStatistics partColStats : partsColStats) { //actually only at most one loop
-        List<ColumnStatisticsObj> statsObjs = partColStats.getStatsObj();
-        List<String> deletedCols = new ArrayList<String>();
-        for (ColumnStatisticsObj statsObj : statsObjs) {
-          boolean found =false;
-          for (FieldSchema newCol : newCols) {
-            if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
-                && statsObj.getColType().equalsIgnoreCase(newCol.getType())) {
-              found = true;
-              break;
-            }
-          }
-          if (!found) {
-            msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals,
-                statsObj.getColName());
-            deletedCols.add(statsObj.getColName());
-          }
-        }
-        StatsSetupConst.removeColumnStatsState(newPart.getParameters(), deletedCols);
-      }
-    } catch (NoSuchObjectException nsoe) {
-      LOG.debug("Could not find db entry." + nsoe);
-      //ignore
-    } catch (InvalidInputException iie) {
-      throw new InvalidObjectException
-      ("Invalid input to update partition column stats in alter table change columns" + iie);
-    }
-  }
-
-  private void updatePartColumnStats(RawStore msdb, String dbName, String tableName,
-      List<String> partVals, Partition newPart) throws MetaException, InvalidObjectException
{
-    dbName = HiveStringUtils.normalizeIdentifier(dbName);
-    tableName = HiveStringUtils.normalizeIdentifier(tableName);
-    String newDbName = HiveStringUtils.normalizeIdentifier(newPart.getDbName());
-    String newTableName = HiveStringUtils.normalizeIdentifier(newPart.getTableName());
-
-    Table oldTable = msdb.getTable(dbName, tableName);
-    if (oldTable == null) {
-      return;
-    }
-
-    try {
-      String oldPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), partVals);
-      String newPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), newPart.getValues());
-      if (!dbName.equals(newDbName) || !tableName.equals(newTableName)
-          || !oldPartName.equals(newPartName)) {
-        msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals, null);
-      } else {
-        Partition oldPartition = msdb.getPartition(dbName, tableName, partVals);
-        if (oldPartition == null) {
-          return;
-        }
-        if (oldPartition.getSd() != null && newPart.getSd() != null) {
-        List<FieldSchema> oldCols = oldPartition.getSd().getCols();
-          if (!MetaStoreUtils.columnsIncluded(oldCols, newPart.getSd().getCols())) {
-            updatePartColumnStatsForAlterColumns(msdb, oldPartition, oldPartName, partVals,
oldCols, newPart);
-          }
-        }
-      }
-    } catch (NoSuchObjectException nsoe) {
-      LOG.debug("Could not find db entry." + nsoe);
-      //ignore
-    } catch (InvalidInputException iie) {
-      throw new InvalidObjectException("Invalid input to update partition column stats."
+ iie);
-    }
-  }
-
   @VisibleForTesting
-  void alterTableUpdateTableColumnStats(RawStore msdb,
-      Table oldTable, Table newTable)
+  void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable)
       throws MetaException, InvalidObjectException {
     String dbName = oldTable.getDbName().toLowerCase();
     String tableName = HiveStringUtils.normalizeIdentifier(oldTable.getTableName());
@@ -839,4 +802,69 @@ public class HiveAlterHandler implements AlterHandler {
       throw new InvalidObjectException("Invalid inputs to update table column stats: " +
e);
     }
   }
+
+  private ColumnStatistics updateOrGetPartitionColumnStats(
+      RawStore msdb, String dbname, String tblname, List<String> partVals,
+      List<FieldSchema> oldCols, Table table, Partition part)
+          throws MetaException, InvalidObjectException {
+    ColumnStatistics newPartsColStats = null;
+    try {
+      List<FieldSchema> newCols = part.getSd() == null ?
+          new ArrayList<FieldSchema>() : part.getSd().getCols();
+      String oldPartName = Warehouse.makePartName(table.getPartitionKeys(), partVals);
+      String newPartName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
+      boolean rename = !part.getDbName().equals(dbname) || !part.getTableName().equals(tblname)
+          || !oldPartName.equals(newPartName);
+
+      // do not need to update column stats if alter partition is not for rename or changing
existing columns
+      if (!rename && MetaStoreUtils.columnsIncluded(oldCols, newCols)) {
+        return newPartsColStats;
+      }
+      List<String> oldColNames = new ArrayList<String>(oldCols.size());
+      for (FieldSchema oldCol : oldCols) {
+        oldColNames.add(oldCol.getName());
+      }
+      List<String> oldPartNames = Lists.newArrayList(oldPartName);
+      List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(dbname,
tblname,
+          oldPartNames, oldColNames);
+      assert (partsColStats.size() <= 1);
+      for (ColumnStatistics partColStats : partsColStats) { //actually only at most one loop
+        List<ColumnStatisticsObj> newStatsObjs = new ArrayList<ColumnStatisticsObj>();
+        List<ColumnStatisticsObj> statsObjs = partColStats.getStatsObj();
+        List<String> deletedCols = new ArrayList<String>();
+        for (ColumnStatisticsObj statsObj : statsObjs) {
+          boolean found =false;
+          for (FieldSchema newCol : newCols) {
+            if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
+                && statsObj.getColType().equalsIgnoreCase(newCol.getType())) {
+              found = true;
+              break;
+            }
+          }
+          if (found) {
+            if (rename) {
+              msdb.deletePartitionColumnStatistics(dbname, tblname, partColStats.getStatsDesc().getPartName(),
+                  partVals, statsObj.getColName());
+              newStatsObjs.add(statsObj);
+            }
+          } else {
+            msdb.deletePartitionColumnStatistics(dbname, tblname, partColStats.getStatsDesc().getPartName(),
+                partVals, statsObj.getColName());
+            deletedCols.add(statsObj.getColName());
+          }
+        }
+        StatsSetupConst.removeColumnStatsState(part.getParameters(), deletedCols);
+        if (!newStatsObjs.isEmpty()) {
+          partColStats.setStatsObj(newStatsObjs);
+          newPartsColStats = partColStats;
+        }
+      }
+    } catch (NoSuchObjectException nsoe) {
+      // ignore this exception, actually this exception won't be thrown from getPartitionColumnStatistics
+    } catch (InvalidInputException iie) {
+      throw new InvalidObjectException("Invalid input to delete partition column stats."
+ iie);
+    }
+
+    return newPartsColStats;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/57de93a8/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 79f6d7f..5418dd6 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -635,14 +635,6 @@ public class MetaStoreUtils {
     }
   }
 
-  static boolean isCascadeNeededInAlterTable(Table oldTable, Table newTable) {
-    //currently cascade only supports add/replace columns and
-    //changing column type/position/name/comments
-    List<FieldSchema> oldCols = oldTable.getSd().getCols();
-    List<FieldSchema> newCols = newTable.getSd().getCols();
-    return !areSameColumns(oldCols, newCols);
-  }
-
   static boolean areSameColumns(List<FieldSchema> oldCols, List<FieldSchema>
newCols) {
     if (oldCols.size() != newCols.size()) {
       return false;

http://git-wip-us.apache.org/repos/asf/hive/blob/57de93a8/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 358cf17..ae187cf 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -7379,7 +7379,12 @@ public class ObjectStore implements RawStore, Configurable {
     try {
       openTransaction();
       // We are not going to verify SD for each partition. Just verify for the table.
-      validateTableCols(table, colNames);
+      // ToDo: we need verify the partition column instead
+      try {
+        validateTableCols(table, colNames);
+      } catch (MetaException me) {
+        LOG.warn("The table does not have the same column definition as its partition.");
+      }
       Query query = queryWrapper.query = pm.newQuery(MPartitionColumnStatistics.class);
       String paramStr = "java.lang.String t1, java.lang.String t2";
       String filter = "tableName == t1 && dbName == t2 && (";

http://git-wip-us.apache.org/repos/asf/hive/blob/57de93a8/ql/src/test/queries/clientpositive/alter_table_column_stats.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/alter_table_column_stats.q b/ql/src/test/queries/clientpositive/alter_table_column_stats.q
new file mode 100644
index 0000000..39dfb0c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alter_table_column_stats.q
@@ -0,0 +1,241 @@
+set hive.mapred.mode=nonstrict;
+
+set hive.metastore.try.direct.sql=true;
+
+drop database if exists statsdb1;
+create database statsdb1;
+drop database if exists statsdb2;
+create database statsdb2;
+
+create table statsdb1.testtable0 (col1 int, col2 string, col3 string);
+insert into statsdb1.testtable0 select key, value, 'val3' from src limit 10;
+
+create table statsdb1.testpart0 (col1 int, col2 string, col3 string) partitioned by (part
string);
+insert into statsdb1.testpart0 partition (part = 'part1') select key, value, 'val3' from
src limit 10;
+insert into statsdb1.testpart0 partition (part = 'part2') select key, value, 'val3' from
src limit 20;
+
+use statsdb1;
+-- test non-partitioned table
+analyze table testtable0 compute statistics for columns;
+describe formatted statsdb1.testtable0;
+describe formatted statsdb1.testtable0 col1;
+describe formatted statsdb1.testtable0 col2;
+describe formatted statsdb1.testtable0 col3;
+
+-- rename non-partitioned table should not change its table and columns stats
+alter table statsdb1.testtable0 rename to statsdb1.testtable1;
+describe formatted statsdb1.testtable1;
+describe formatted statsdb1.testtable1 col1;
+describe formatted statsdb1.testtable1 col2;
+describe formatted statsdb1.testtable1 col3;
+
+-- when replacing columns in a non-partitioned table, the table stats should not change,
+-- but the stats of the changed columns are removed
+alter table testtable1 replace columns (col1 int, col2 string, col4 string);
+describe formatted statsdb1.testtable1;
+describe formatted statsdb1.testtable1 col1;
+describe formatted statsdb1.testtable1 col2;
+describe formatted statsdb1.testtable1 col4;
+
+-- when changing the column type in a non-partitioned table, the table stats should not change,
+-- but the stats of the type-changed columns are removed
+alter table testtable1 change col1 col1 string;
+describe formatted statsdb1.testtable1;
+describe formatted statsdb1.testtable1 col1;
+describe formatted statsdb1.testtable1 col2;
+describe formatted statsdb1.testtable1 col4;
+
+-- rename the db of a non-partitoned table, the table and columns stats should not change
+alter table statsdb1.testtable1 rename to statsdb2.testtable2;
+describe formatted statsdb2.testtable2;
+describe formatted statsdb2.testtable2 col1;
+describe formatted statsdb2.testtable2 col2;
+describe formatted statsdb2.testtable2 col4;
+
+-- test partitioned table
+analyze table testpart0 compute statistics for columns;
+describe formatted statsdb1.testpart0;
+describe formatted statsdb1.testpart0 partition (part = 'part1');
+describe formatted statsdb1.testpart0 partition (part = 'part1') col1;
+describe formatted statsdb1.testpart0 partition (part = 'part1') col2;
+describe formatted statsdb1.testpart0 partition (part = 'part1') col3;
+describe formatted statsdb1.testpart0 partition (part = 'part2');
+describe formatted statsdb1.testpart0 partition (part = 'part2') col1;
+describe formatted statsdb1.testpart0 partition (part = 'part2') col2;
+describe formatted statsdb1.testpart0 partition (part = 'part2') col3;
+
+-- rename a partitioned table should not change its table, partition, and column stats
+alter table statsdb1.testpart0 rename to statsdb1.testpart1;
+describe formatted statsdb1.testpart1;
+describe formatted statsdb1.testpart1 partition (part = 'part1');
+describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col3;
+describe formatted statsdb1.testpart1 partition (part = 'part2');
+describe formatted statsdb1.testpart1 partition (part = 'part2') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col3;
+
+-- when cascade replacing columns in a partitioned table, the table and partition stats should
not change,
+-- but the stats of the changed columns are removed
+alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade;
+describe formatted statsdb1.testpart1;
+describe formatted statsdb1.testpart1 partition (part = 'part1');
+describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col4;
+describe formatted statsdb1.testpart1 partition (part = 'part2');
+describe formatted statsdb1.testpart1 partition (part = 'part2') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col4;
+
+-- when cascade changing the column type in a partitioned table, the table and partition
stats should not change,
+-- but the stats of the type-changed columns are removed
+alter table statsdb1.testpart1 change column col1 col1 string cascade;
+describe formatted statsdb1.testpart1;
+describe formatted statsdb1.testpart1 partition (part = 'part1');
+describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col4;
+describe formatted statsdb1.testpart1 partition (part = 'part2');
+describe formatted statsdb1.testpart1 partition (part = 'part2') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col4;
+
+-- change database of a partition should not change table, partition and columns stats
+alter table statsdb1.testpart1 rename to statsdb2.testpart2;
+describe formatted statsdb2.testpart2;
+describe formatted statsdb2.testpart2 partition (part = 'part1') col1;
+describe formatted statsdb2.testpart2 partition (part = 'part1') col2;
+describe formatted statsdb2.testpart2 partition (part = 'part1') col4;
+describe formatted statsdb2.testpart2 partition (part = 'part2') col1;
+describe formatted statsdb2.testpart2 partition (part = 'part2') col2;
+describe formatted statsdb2.testpart2 partition (part = 'part2') col4;
+
+use statsdb2;
+drop table statsdb2.testpart2;
+drop table statsdb2.testtable2;
+
+use default;
+drop database statsdb1;
+drop database statsdb2;
+
+set hive.metastore.try.direct.sql=false;
+
+drop database if exists statsdb1;
+create database statsdb1;
+drop database if exists statsdb2;
+create database statsdb2;
+
+create table statsdb1.testtable0 (col1 int, col2 string, col3 string);
+insert into statsdb1.testtable0 select key, value, 'val3' from src limit 10;
+
+create table statsdb1.testpart0 (col1 int, col2 string, col3 string) partitioned by (part
string);
+insert into statsdb1.testpart0 partition (part = 'part1') select key, value, 'val3' from
src limit 10;
+insert into statsdb1.testpart0 partition (part = 'part2') select key, value, 'val3' from
src limit 20;
+
+use statsdb1;
+-- test non-partitioned table
+analyze table testtable0 compute statistics for columns;
+describe formatted statsdb1.testtable0;
+describe formatted statsdb1.testtable0 col1;
+describe formatted statsdb1.testtable0 col2;
+describe formatted statsdb1.testtable0 col3;
+
+-- rename non-partitioned table should not change its table and columns stats
+alter table statsdb1.testtable0 rename to statsdb1.testtable1;
+describe formatted statsdb1.testtable1;
+describe formatted statsdb1.testtable1 col1;
+describe formatted statsdb1.testtable1 col2;
+describe formatted statsdb1.testtable1 col3;
+
+-- when replacing columns in a non-partitioned table, the table stats should not change,
+-- but the stats of the changed columns are removed
+alter table testtable1 replace columns (col1 int, col2 string, col4 string);
+describe formatted statsdb1.testtable1;
+describe formatted statsdb1.testtable1 col1;
+describe formatted statsdb1.testtable1 col2;
+describe formatted statsdb1.testtable1 col4;
+
+-- when changing the column type in a non-partitioned table, the table stats should not change,
+-- but the stats of the type-changed columns are removed
+alter table testtable1 change col1 col1 string;
+describe formatted statsdb1.testtable1;
+describe formatted statsdb1.testtable1 col1;
+describe formatted statsdb1.testtable1 col2;
+describe formatted statsdb1.testtable1 col4;
+
+-- rename the db of a non-partitoned table, the table and columns stats should not change
+alter table statsdb1.testtable1 rename to statsdb2.testtable2;
+describe formatted statsdb2.testtable2;
+describe formatted statsdb2.testtable2 col1;
+describe formatted statsdb2.testtable2 col2;
+describe formatted statsdb2.testtable2 col4;
+
+-- test partitioned table
+analyze table testpart0 compute statistics for columns;
+describe formatted statsdb1.testpart0;
+describe formatted statsdb1.testpart0 partition (part = 'part1');
+describe formatted statsdb1.testpart0 partition (part = 'part1') col1;
+describe formatted statsdb1.testpart0 partition (part = 'part1') col2;
+describe formatted statsdb1.testpart0 partition (part = 'part1') col3;
+describe formatted statsdb1.testpart0 partition (part = 'part2');
+describe formatted statsdb1.testpart0 partition (part = 'part2') col1;
+describe formatted statsdb1.testpart0 partition (part = 'part2') col2;
+describe formatted statsdb1.testpart0 partition (part = 'part2') col3;
+
+-- rename a partitioned table should not change its table, partition, and column stats
+alter table statsdb1.testpart0 rename to statsdb1.testpart1;
+describe formatted statsdb1.testpart1;
+describe formatted statsdb1.testpart1 partition (part = 'part1');
+describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col3;
+describe formatted statsdb1.testpart1 partition (part = 'part2');
+describe formatted statsdb1.testpart1 partition (part = 'part2') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col3;
+
+-- when cascade replacing columns in a partitioned table, the table and partition stats should
not change,
+-- but the stats of the changed columns are removed
+alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade;
+describe formatted statsdb1.testpart1;
+describe formatted statsdb1.testpart1 partition (part = 'part1');
+describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col4;
+describe formatted statsdb1.testpart1 partition (part = 'part2');
+describe formatted statsdb1.testpart1 partition (part = 'part2') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col4;
+
+-- when cascade changing the column type in a partitioned table, the table and partition
stats should not change,
+-- but the stats of the type-changed columns are removed
+alter table statsdb1.testpart1 change column col1 col1 string cascade;
+describe formatted statsdb1.testpart1;
+describe formatted statsdb1.testpart1 partition (part = 'part1');
+describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part1') col4;
+describe formatted statsdb1.testpart1 partition (part = 'part2');
+describe formatted statsdb1.testpart1 partition (part = 'part2') col1;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col2;
+describe formatted statsdb1.testpart1 partition (part = 'part2') col4;
+
+-- change database of a partition should not change table, partition and columns stats
+alter table statsdb1.testpart1 rename to statsdb2.testpart2;
+describe formatted statsdb2.testpart2;
+describe formatted statsdb2.testpart2 partition (part = 'part1') col1;
+describe formatted statsdb2.testpart2 partition (part = 'part1') col2;
+describe formatted statsdb2.testpart2 partition (part = 'part1') col4;
+describe formatted statsdb2.testpart2 partition (part = 'part2') col1;
+describe formatted statsdb2.testpart2 partition (part = 'part2') col2;
+describe formatted statsdb2.testpart2 partition (part = 'part2') col4;
+
+use statsdb2;
+drop table statsdb2.testpart2;
+drop table statsdb2.testtable2;
+
+use default;
+drop database statsdb1;
+drop database statsdb2;

http://git-wip-us.apache.org/repos/asf/hive/blob/57de93a8/ql/src/test/queries/clientpositive/alter_table_invalidate_column_stats.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/alter_table_invalidate_column_stats.q b/ql/src/test/queries/clientpositive/alter_table_invalidate_column_stats.q
deleted file mode 100644
index a478451..0000000
--- a/ql/src/test/queries/clientpositive/alter_table_invalidate_column_stats.q
+++ /dev/null
@@ -1,153 +0,0 @@
-set hive.mapred.mode=nonstrict;
-set hive.metastore.try.direct.sql=true;
-
-drop database if exists statsdb1;
-create database statsdb1;
-drop database if exists statsdb2;
-create database statsdb2;
-
-create table statsdb1.testtable1 (col1 int, col2 string, col3 string);
-insert into statsdb1.testtable1 select key, value, 'val3' from src limit 10;
-
-create table statsdb1.testpart1 (col1 int, col2 string, col3 string) partitioned by (part
string);
-insert into statsdb1.testpart1 partition (part = 'part1') select key, value, 'val3' from
src limit 10;
-insert into statsdb1.testpart1 partition (part = 'part2') select key, value, 'val3' from
src limit 20;
-
-use statsdb1;
-
-analyze table testtable1 compute statistics for columns;
-
-describe formatted statsdb1.testtable1 col1;
-describe formatted statsdb1.testtable1 col2;
-describe formatted statsdb1.testtable1 col3;
-
-alter table testtable1 replace columns (col1 int, col2 string, col4 string);
-describe formatted statsdb1.testtable1 col1;
-describe formatted statsdb1.testtable1 col2;
-describe formatted statsdb1.testtable1 col4;
-
-alter table testtable1 change col1 col1 string;
-describe formatted statsdb1.testtable1 col1;
-describe formatted statsdb1.testtable1 col2;
-describe formatted statsdb1.testtable1 col4;
-
-alter table statsdb1.testtable1 rename to statsdb2.testtable2;
-
-
-analyze table testpart1 compute statistics for columns;
-
-describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col3;
-describe formatted statsdb1.testpart1 partition (part = 'part2') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part2') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part2') col3;
-
-alter table statsdb1.testpart1 partition (part = 'part2') rename to partition (part = 'part3');
-describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col3;
-describe formatted statsdb1.testpart1 partition (part = 'part3') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part3') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part3') col3;
-
-alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col4;
-
-alter table statsdb1.testpart1 change column col1 col1 string;
-set hive.exec.dynamic.partition = true;
-alter table statsdb1.testpart1 partition (part) change column col1 col1 string;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col4;
-
-alter table statsdb1.testpart1 rename to statsdb2.testpart2;
-use statsdb2;
-
-alter table statsdb2.testpart2 drop partition (part = 'part1');
-drop table statsdb2.testpart2;
-
-drop table statsdb2.testtable2;
-
-use default;
-drop database statsdb1;
-drop database statsdb2;
-
-
-set hive.metastore.try.direct.sql=false;
-
-drop database if exists statsdb1;
-create database statsdb1;
-drop database if exists statsdb2;
-create database statsdb2;
-
-create table statsdb1.testtable1 (col1 int, col2 string, col3 string);
-insert into statsdb1.testtable1 select key, value, 'val3' from src limit 10;
-
-create table statsdb1.testpart1 (col1 int, col2 string, col3 string) partitioned by (part
string);
-insert into statsdb1.testpart1 partition (part = 'part1') select key, value, 'val3' from
src limit 10;
-insert into statsdb1.testpart1 partition (part = 'part2') select key, value, 'val3' from
src limit 20;
-
-use statsdb1;
-
-analyze table testtable1 compute statistics for columns;
-
-describe formatted statsdb1.testtable1 col1;
-describe formatted statsdb1.testtable1 col2;
-describe formatted statsdb1.testtable1 col3;
-
-alter table testtable1 replace columns (col1 int, col2 string, col4 string);
-describe formatted statsdb1.testtable1 col1;
-describe formatted statsdb1.testtable1 col2;
-describe formatted statsdb1.testtable1 col4;
-
-alter table testtable1 change col1 col1 string;
-describe formatted statsdb1.testtable1 col1;
-describe formatted statsdb1.testtable1 col2;
-describe formatted statsdb1.testtable1 col4;
-
-alter table statsdb1.testtable1 rename to statsdb2.testtable2;
-
-
-analyze table testpart1 compute statistics for columns;
-
-describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col3;
-describe formatted statsdb1.testpart1 partition (part = 'part2') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part2') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part2') col3;
-
-alter table statsdb1.testpart1 partition (part = 'part2') rename to partition (part = 'part3');
-describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col3;
-describe formatted statsdb1.testpart1 partition (part = 'part3') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part3') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part3') col3;
-
-alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col4;
-
-alter table statsdb1.testpart1 change column col1 col1 string;
-set hive.exec.dynamic.partition = true;
-alter table statsdb1.testpart1 partition (part) change column col1 col1 string;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col1;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col2;
-describe formatted statsdb1.testpart1 partition (part = 'part1') col4;
-
-alter table statsdb1.testpart1 rename to statsdb2.testpart2;
-use statsdb2;
-
-alter table statsdb2.testpart2 drop partition (part = 'part1');
-drop table statsdb2.testpart2;
-
-drop table statsdb2.testtable2;
-
-use default;
-drop database statsdb1;
-drop database statsdb2;


Mime
View raw message