hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject hive git commit: HIVE-11061 Table renames not propagated to partition table in HBase metastore (gates)
Date Wed, 01 Jul 2015 22:47:45 GMT
Repository: hive
Updated Branches:
  refs/heads/hbase-metastore 6ea5c0956 -> 1a1c0d817


HIVE-11061 Table renames not propagated to partition table in HBase metastore (gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1a1c0d81
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1a1c0d81
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1a1c0d81

Branch: refs/heads/hbase-metastore
Commit: 1a1c0d817da036025f1c3b953af7b52403c048ba
Parents: 6ea5c09
Author: Alan Gates <gates@hortonworks.com>
Authored: Wed Jul 1 15:47:03 2015 -0700
Committer: Alan Gates <gates@hortonworks.com>
Committed: Wed Jul 1 15:47:03 2015 -0700

----------------------------------------------------------------------
 .../metastore/hbase/TestHBaseMetastoreSql.java  | 27 +++++++++++++
 .../hadoop/hive/metastore/HiveAlterHandler.java | 16 ++++----
 .../hive/metastore/hbase/HBaseReadWrite.java    | 40 ++++++++++++++++----
 .../hadoop/hive/metastore/hbase/HBaseStore.java | 21 +++++++++-
 4 files changed, 85 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
index 119e5aa..c61ebb7 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
@@ -192,5 +192,32 @@ public class TestHBaseMetastoreSql extends HBaseIntegrationTests {
     Assert.assertEquals(0, rsp.getResponseCode());
   }
 
+  @Test
+  public void alterRenamePartitioned() throws Exception {
+    driver.run("create table alterrename (c int) partitioned by (ds string)");
+    driver.run("alter table alterrename add partition (ds = 'a')");
+    CommandProcessorResponse rsp = driver.run("describe extended alterrename partition (ds='a')");
+    Assert.assertEquals(0, rsp.getResponseCode());
+    rsp = driver.run("alter table alterrename rename to alter_renamed");
+    Assert.assertEquals(0, rsp.getResponseCode());
+    rsp = driver.run("describe extended alter_renamed partition (ds='a')");
+    Assert.assertEquals(0, rsp.getResponseCode());
+    rsp = driver.run("describe extended alterrename partition (ds='a')");
+    Assert.assertEquals(10001, rsp.getResponseCode());
+  }
+
+  @Test
+  public void alterRename() throws Exception {
+    driver.run("create table alterrename1 (c int)");
+    CommandProcessorResponse rsp = driver.run("describe alterrename1");
+    Assert.assertEquals(0, rsp.getResponseCode());
+    rsp = driver.run("alter table alterrename1 rename to alter_renamed1");
+    Assert.assertEquals(0, rsp.getResponseCode());
+    rsp = driver.run("describe alter_renamed1");
+    Assert.assertEquals(0, rsp.getResponseCode());
+    rsp = driver.run("describe alterrename1");
+    Assert.assertEquals(10001, rsp.getResponseCode());
+  }
+
 
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 99d6680..f402f73 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -17,12 +17,7 @@
  */
 package org.apache.hadoop.hive.metastore;
 
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
+import com.google.common.collect.Lists;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -48,7 +43,11 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hive.common.util.HiveStringUtils;
 
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 
 /**
  * Hive specific implementation of alter
@@ -121,8 +120,7 @@ public class HiveAlterHandler implements AlterHandler {
       // get old table
       oldt = msdb.getTable(dbname, name);
       if (oldt == null) {
-        throw new InvalidOperationException("table " + newt.getDbName() + "."
-            + newt.getTableName() + " doesn't exist");
+        throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist");
       }
 
       if (HiveConf.getBoolVar(hiveConf,

http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
index b54afb9..fd6f9f5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
@@ -541,6 +541,9 @@ class HBaseReadWrite {
     byte[][] serialized = HBaseUtils.serializePartition(newPart, hash);
     store(PART_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
     partCache.put(newPart.getDbName(), newPart.getTableName(), newPart);
+    if (!oldPart.getTableName().equals(newPart.getTableName())) {
+      deletePartition(oldPart.getDbName(), oldPart.getTableName(), oldPart.getValues());
+    }
   }
 
   /**
@@ -568,7 +571,7 @@ class HBaseReadWrite {
     if (oldParts.size() != newParts.size()) {
       throw new RuntimeException("Number of old and new partitions must match.");
     }
-    List<Put> puts = new ArrayList<Put>(newParts.size());
+    List<Put> puts = new ArrayList<>(newParts.size());
     for (int i = 0; i < newParts.size(); i++) {
       byte[] hash;
       byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldParts.get(i).getSd(), md);
@@ -584,6 +587,11 @@ class HBaseReadWrite {
       p.add(CATALOG_CF, CATALOG_COL, serialized[1]);
       puts.add(p);
       partCache.put(newParts.get(i).getDbName(), newParts.get(i).getTableName(), newParts.get(i));
+      if (!newParts.get(i).getTableName().equals(oldParts.get(i).getTableName())) {
+        // We need to remove the old record as well.
+        deletePartition(oldParts.get(i).getDbName(), oldParts.get(i).getTableName(),
+            oldParts.get(i).getValues(), false);
+      }
     }
     HTableInterface htab = conn.getHBaseTable(PART_TABLE);
     htab.put(puts);
@@ -734,10 +742,17 @@ class HBaseReadWrite {
    * @throws IOException
    */
   void deletePartition(String dbName, String tableName, List<String> partVals) throws
IOException {
+    deletePartition(dbName, tableName, partVals, true);
+  }
+
+  private void deletePartition(String dbName, String tableName, List<String> partVals,
+                               boolean decrementRefCnt) throws IOException {
     // Find the partition so I can get the storage descriptor and drop it
     partCache.remove(dbName, tableName, partVals);
-    Partition p = getPartition(dbName, tableName, partVals, false);
-    decrementStorageDescriptorRefCount(p.getSd());
+    if (decrementRefCnt) {
+      Partition p = getPartition(dbName, tableName, partVals, false);
+      decrementStorageDescriptorRefCount(p.getSd());
+    }
     byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName, partVals);
     delete(PART_TABLE, key, null, null);
   }
@@ -1287,8 +1302,10 @@ class HBaseReadWrite {
     }
     byte[][] serialized = HBaseUtils.serializeTable(newTable, hash);
     store(TABLE_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-    tableCache.put(new ObjectPair<String, String>(newTable.getDbName(), newTable.getTableName()),
-        newTable);
+    tableCache.put(new ObjectPair<>(newTable.getDbName(), newTable.getTableName()),
newTable);
+    if (!oldTable.getTableName().equals(newTable.getTableName())) {
+      deleteTable(oldTable.getDbName(), oldTable.getTableName());
+    }
   }
 
   /**
@@ -1298,10 +1315,17 @@ class HBaseReadWrite {
    * @throws IOException
    */
   void deleteTable(String dbName, String tableName) throws IOException {
+    deleteTable(dbName, tableName, true);
+  }
+
+  private void deleteTable(String dbName, String tableName, boolean decrementRefCnt)
+      throws IOException {
     tableCache.remove(new ObjectPair<String, String>(dbName, tableName));
-    // Find the table so I can get the storage descriptor and drop it
-    Table t = getTable(dbName, tableName, false);
-    decrementStorageDescriptorRefCount(t.getSd());
+    if (decrementRefCnt) {
+      // Find the table so I can get the storage descriptor and drop it
+      Table t = getTable(dbName, tableName, false);
+      decrementStorageDescriptorRefCount(t.getSd());
+    }
     byte[] key = HBaseUtils.buildKey(dbName, tableName);
     delete(TABLE_TABLE, key, null, null);
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
index 0dbdba2..4fa2ae5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
@@ -404,8 +404,25 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      Table oldTable = getHBase().getTable(dbname, name);
-      getHBase().replaceTable(oldTable, newTable);
+      getHBase().replaceTable(getHBase().getTable(dbname, name), newTable);
+      if (newTable.getPartitionKeys() != null && newTable.getPartitionKeys().size()
> 0
+          && !name.equals(newTable.getTableName())) {
+        // They renamed the table, so we need to change each partition as well, since it
changes
+        // the key.
+        try {
+          List<Partition> oldParts = getPartitions(dbname, name, -1);
+          List<Partition> newParts = new ArrayList<>(oldParts.size());
+          for (Partition oldPart : oldParts) {
+            Partition newPart = oldPart.deepCopy();
+            newPart.setTableName(newTable.getTableName());
+            newParts.add(newPart);
+          }
+          getHBase().replacePartitions(oldParts, newParts);
+        } catch (NoSuchObjectException e) {
+          LOG.debug("No partitions found for old table so not worrying about it");
+        }
+
+      }
       commit = true;
     } catch (IOException e) {
       LOG.error("Unable to alter table " + tableNameForErrorMsg(dbname, name), e);


Mime
View raw message