hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1214380 - in /hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore: AlterHandler.java HiveAlterHandler.java HiveMetaStore.java
Date Wed, 14 Dec 2011 17:52:39 GMT
Author: namit
Date: Wed Dec 14 17:52:39 2011
New Revision: 1214380

URL: http://svn.apache.org/viewvc?rev=1214380&view=rev
Log:
HIVE-2640 Add alterPartition to AlterHandler interface
(Kevin Wilfong via namit)


Modified:
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java?rev=1214380&r1=1214379&r2=1214380&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java Wed Dec
14 17:52:39 2011
@@ -17,9 +17,14 @@
  */
 package org.apache.hadoop.hive.metastore;
 
+import java.util.List;
+
 import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.Table;
 
 /**
@@ -29,7 +34,7 @@ public interface AlterHandler extends Co
 
   /**
    * handles alter table
-   * 
+   *
    * @param msdb
    *          object to get metadata
    * @param wh
@@ -49,4 +54,29 @@ public interface AlterHandler extends Co
   public abstract void alterTable(RawStore msdb, Warehouse wh, String dbname,
       String name, Table newTable) throws InvalidOperationException,
       MetaException;
+
+  /**
+   * handles alter partition
+   *
+   * @param msdb
+   *          object to get metadata
+   * @param wh
+   * @param dbname
+   *          database of the partition being altered
+   * @param name
+   *          table of the partition being altered
+   * @param part_vals
+   *          original values of the partition being altered
+   * @param new_part
+   *          new partition object
+   * @return
+   * @throws InvalidOperationException
+   * @throws InvalidObjectException
+   * @throws AlreadyExistsException
+   * @throws MetaException
+   */
+  public abstract Partition alterPartition(final RawStore msdb, Warehouse wh, final String
dbname,
+      final String name, final List<String> part_vals, final Partition new_part)
+      throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,
+      MetaException;
 }

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java?rev=1214380&r1=1214379&r2=1214380&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java Wed
Dec 14 17:52:39 2011
@@ -28,6 +28,8 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.Constants;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
@@ -41,7 +43,7 @@ import org.apache.hadoop.hive.metastore.
  */
 public class HiveAlterHandler implements AlterHandler {
 
-  private Configuration hiveConf;
+  protected Configuration hiveConf;
   private static final Log LOG = LogFactory.getLog(HiveAlterHandler.class
       .getName());
 
@@ -224,6 +226,147 @@ public class HiveAlterHandler implements
     }
   }
 
+  public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
+      final String name, final List<String> part_vals, final Partition new_part)
+      throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,
+      MetaException {
+    boolean success = false;
+
+    Path srcPath = null;
+    Path destPath = null;
+    FileSystem srcFs = null;
+    FileSystem destFs = null;
+    Table tbl = null;
+    Partition oldPart = null;
+    String oldPartLoc = null;
+    String newPartLoc = null;
+    // Set DDL time to now if not specified
+    if (new_part.getParameters() == null ||
+        new_part.getParameters().get(Constants.DDL_TIME) == null ||
+        Integer.parseInt(new_part.getParameters().get(Constants.DDL_TIME)) == 0) {
+      new_part.putToParameters(Constants.DDL_TIME, Long.toString(System
+          .currentTimeMillis() / 1000));
+    }
+    //alter partition
+    if (part_vals == null || part_vals.size() == 0) {
+      try {
+        oldPart = msdb.getPartition(dbname, name, new_part.getValues());
+        msdb.alterPartition(dbname, name, new_part.getValues(), new_part);
+      } catch (InvalidObjectException e) {
+        throw new InvalidOperationException("alter is not possible");
+      } catch (NoSuchObjectException e){
+        //old partition does not exist
+        throw new InvalidOperationException("alter is not possible");
+      }
+      return oldPart;
+    }
+    //rename partition
+    try {
+      msdb.openTransaction();
+      try {
+        oldPart = msdb.getPartition(dbname, name, part_vals);
+      } catch (NoSuchObjectException e) {
+        // this means there is no existing partition
+        throw new InvalidObjectException(
+            "Unable to rename partition because old partition does not exist");
+      }
+      Partition check_part = null;
+      try {
+        check_part = msdb.getPartition(dbname, name, new_part.getValues());
+      } catch(NoSuchObjectException e) {
+        // this means there is no existing partition
+        check_part = null;
+      }
+      if (check_part != null) {
+        throw new AlreadyExistsException("Partition already exists:" + dbname + "." + name
+ "." +
+            new_part.getValues());
+      }
+      tbl = msdb.getTable(dbname, name);
+      if (tbl == null) {
+        throw new InvalidObjectException(
+            "Unable to rename partition because table or database do not exist");
+      }
+      try {
+        destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name),
+            Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()));
+      } catch (NoSuchObjectException e) {
+        LOG.debug(e);
+        throw new InvalidOperationException(
+            "Unable to change partition or table. Database " + dbname + " does not exist"
+                + " Check metastore logs for detailed stack." + e.getMessage());
+      }
+      if (destPath != null) {
+        newPartLoc = destPath.toString();
+        oldPartLoc = oldPart.getSd().getLocation();
+
+        srcPath = new Path(oldPartLoc);
+
+        LOG.info("srcPath:" + oldPartLoc);
+        LOG.info("descPath:" + newPartLoc);
+        srcFs = wh.getFs(srcPath);
+        destFs = wh.getFs(destPath);
+        // check that src and dest are on the same file system
+        if (srcFs != destFs) {
+          throw new InvalidOperationException("table new location " + destPath
+              + " is on a different file system than the old location "
+              + srcPath + ". This operation is not supported");
+        }
+        try {
+          srcFs.exists(srcPath); // check that src exists and also checks
+          if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
+            throw new InvalidOperationException("New location for this table "
+                + tbl.getDbName() + "." + tbl.getTableName()
+                + " already exists : " + destPath);
+          }
+        } catch (IOException e) {
+          Warehouse.closeFs(srcFs);
+          Warehouse.closeFs(destFs);
+          throw new InvalidOperationException("Unable to access new location "
+              + destPath + " for partition " + tbl.getDbName() + "."
+              + tbl.getTableName() + " " + new_part.getValues());
+        }
+        new_part.getSd().setLocation(newPartLoc);
+        msdb.alterPartition(dbname, name, part_vals, new_part);
+      }
+
+      success = msdb.commitTransaction();
+    } finally {
+      if (!success) {
+        msdb.rollbackTransaction();
+      }
+      if (success && newPartLoc.compareTo(oldPartLoc) != 0) {
+        //rename the data directory
+        try{
+          if (srcFs.exists(srcPath)) {
+            //if destPath's parent path doesn't exist, we should mkdir it
+            Path destParentPath = destPath.getParent();
+            if (!wh.mkdirs(destParentPath)) {
+                throw new IOException("Unable to create path " + destParentPath);
+            }
+            srcFs.rename(srcPath, destPath);
+            LOG.info("rename done!");
+          }
+        } catch (IOException e) {
+          boolean revertMetaDataTransaction = false;
+          try {
+            msdb.openTransaction();
+            msdb.alterPartition(dbname, name, new_part.getValues(), oldPart);
+            revertMetaDataTransaction = msdb.commitTransaction();
+          } catch (Exception e1) {
+            LOG.error("Reverting metadata opeation failed During HDFS operation failed",
e1);
+            if (!revertMetaDataTransaction) {
+              msdb.rollbackTransaction();
+            }
+          }
+          throw new InvalidOperationException("Unable to access old location "
+              + srcPath + " for partition " + tbl.getDbName() + "."
+              + tbl.getTableName() + " " + part_vals);
+        }
+      }
+    }
+    return oldPart;
+  }
+
   private boolean checkPartialPartKeysEqual(List<FieldSchema> oldPartKeys,
       List<FieldSchema> newPartKeys) {
     //return true if both are null, or false if one is null and the other isn't

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1214380&r1=1214379&r2=1214380&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Wed
Dec 14 17:52:39 2011
@@ -41,7 +41,6 @@ import org.apache.commons.cli.OptionBuil
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.common.LogUtils;
@@ -1872,15 +1871,21 @@ public class HiveMetaStore extends Thrif
         LOG.info("Old Partition values:" + part_vals);
       }
 
-      boolean success = false;
+      Partition oldPart = null;
       try {
-        success = executeWithRetry(new Command<Boolean>() {
-          @Override
-          public Boolean run(RawStore ms) throws Exception {
-            alter_partition_core(ms, db_name, tbl_name, part_vals, new_part);
-            return Boolean.TRUE;
-          }
-        });
+        oldPart =
+          executeWithRetry(new Command<Partition>() {
+            @Override
+            public Partition run(RawStore ms) throws Exception {
+              return alterHandler.alterPartition(ms, wh, db_name, tbl_name, part_vals, new_part);
+            }
+          });
+        for (MetaStoreEventListener listener : listeners) {
+          listener.onAlterPartition(new AlterPartitionEvent(oldPart, new_part, true, this));
+        }
+      } catch (NoSuchObjectException e){
+        //old partition does not exist
+        throw new InvalidOperationException("alter is not possible");
       } catch (InvalidObjectException e) {
         throw new InvalidOperationException(e.getMessage());
       } catch (AlreadyExistsException e) {
@@ -1893,153 +1898,12 @@ public class HiveMetaStore extends Thrif
         assert(e instanceof RuntimeException);
         throw (RuntimeException)e;
       } finally {
-        endFunction("alter_partition", success);
+        endFunction("alter_partition", oldPart != null);
       }
       return;
     }
 
-    private void alter_partition_core(final RawStore ms, final String dbname, final String
name, final List<String> part_vals, final Partition new_part)
-        throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,
MetaException {
-      boolean success = false;
-
-      Path srcPath = null;
-      Path destPath = null;
-      FileSystem srcFs = null;
-      FileSystem destFs = null;
-      Table tbl = null;
-      Partition oldPart = null;
-      String oldPartLoc = null;
-      String newPartLoc = null;
-      // Set DDL time to now if not specified
-      if (new_part.getParameters() == null ||
-          new_part.getParameters().get(Constants.DDL_TIME) == null ||
-          Integer.parseInt(new_part.getParameters().get(Constants.DDL_TIME)) == 0) {
-        new_part.putToParameters(Constants.DDL_TIME, Long.toString(System
-            .currentTimeMillis() / 1000));
-      }
-      //alter partition
-      if (part_vals == null || part_vals.size() == 0) {
-        try {
-          oldPart = ms.getPartition(dbname, name, new_part.getValues());
-          ms.alterPartition(dbname, name, new_part.getValues(), new_part);
-          for (MetaStoreEventListener listener : listeners) {
-            listener.onAlterPartition(new AlterPartitionEvent(oldPart, new_part, true, this));
-          }
-        } catch (InvalidObjectException e) {
-          throw new InvalidOperationException("alter is not possible");
-        } catch (NoSuchObjectException e){
-          //old partition does not exist
-          throw new InvalidOperationException("alter is not possible");
-        }
-        return;
-      }
-      //rename partition
-      try {
-        ms.openTransaction();
-        try {
-          oldPart = ms.getPartition(dbname, name, part_vals);
-        } catch (NoSuchObjectException e) {
-          // this means there is no existing partition
-          throw new InvalidObjectException(
-              "Unable to rename partition because old partition does not exist");
-        }
-        Partition check_part = null;
-        try {
-          check_part = ms.getPartition(dbname, name, new_part.getValues());
-        } catch(NoSuchObjectException e) {
-          // this means there is no existing partition
-          check_part = null;
-        }
-        if (check_part != null) {
-          throw new AlreadyExistsException("Partition already exists:" + dbname + "." + name
+ "." + new_part.getValues());
-        }
-        tbl = ms.getTable(dbname, name);
-        if (tbl == null) {
-          throw new InvalidObjectException(
-              "Unable to rename partition because table or database do not exist");
-        }
-        try {
-          destPath = new Path(wh.getTablePath(ms.getDatabase(dbname), name), Warehouse.makePartName(tbl.getPartitionKeys(),
-            new_part.getValues()));
-        } catch (NoSuchObjectException e) {
-          LOG.debug(e);
-          throw new InvalidOperationException(
-              "Unable to change partition or table. Database " + dbname + " does not exist"
-                  + " Check metastore logs for detailed stack." + e.getMessage());
-        }
-        if (destPath != null) {
-          newPartLoc = destPath.toString();
-          oldPartLoc = oldPart.getSd().getLocation();
-
-          srcPath = new Path(oldPartLoc);
-
-          LOG.info("srcPath:" + oldPartLoc);
-          LOG.info("descPath:" + newPartLoc);
-          srcFs = wh.getFs(srcPath);
-          destFs = wh.getFs(destPath);
-          // check that src and dest are on the same file system
-          if (srcFs != destFs) {
-            throw new InvalidOperationException("table new location " + destPath
-                + " is on a different file system than the old location "
-                + srcPath + ". This operation is not supported");
-          }
-          try {
-            srcFs.exists(srcPath); // check that src exists and also checks
-            if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath))
{
-              throw new InvalidOperationException("New location for this table "
-                  + tbl.getDbName() + "." + tbl.getTableName()
-                  + " already exists : " + destPath);
-            }
-          } catch (IOException e) {
-            Warehouse.closeFs(srcFs);
-            Warehouse.closeFs(destFs);
-            throw new InvalidOperationException("Unable to access new location "
-                + destPath + " for partition " + tbl.getDbName() + "."
-                + tbl.getTableName() + " " + new_part.getValues());
-          }
-          new_part.getSd().setLocation(newPartLoc);
-          ms.alterPartition(dbname, name, part_vals, new_part);
-        }
 
-        success = ms.commitTransaction();
-      } finally {
-        if (!success) {
-          ms.rollbackTransaction();
-        }
-        if (success && newPartLoc.compareTo(oldPartLoc) != 0) {
-          //rename the data directory
-          try{
-            if (srcFs.exists(srcPath)) {
-              //if destPath's parent path doesn't exist, we should mkdir it
-              Path destParentPath = destPath.getParent();
-              if (!wh.mkdirs(destParentPath)) {
-                  throw new IOException("Unable to create path " + destParentPath);
-              }
-              srcFs.rename(srcPath, destPath);
-              LOG.info("rename done!");
-            }
-          } catch (IOException e) {
-            boolean revertMetaDataTransaction = false;
-            try {
-              ms.openTransaction();
-              ms.alterPartition(dbname, name, new_part.getValues(), oldPart);
-              revertMetaDataTransaction = ms.commitTransaction();
-            } catch (Exception e1) {
-              LOG.error("Reverting metadata opeation failed During HDFS operation failed",
e1);
-              if (!revertMetaDataTransaction) {
-                ms.rollbackTransaction();
-              }
-            }
-            throw new InvalidOperationException("Unable to access old location "
-                + srcPath + " for partition " + tbl.getDbName() + "."
-                + tbl.getTableName() + " " + part_vals);
-          }
-        }
-        for (MetaStoreEventListener listener : listeners) {
-          listener.onAlterPartition(new AlterPartitionEvent(oldPart, new_part, true, this));
-        }
-      }
-    }
 
     public boolean create_index(Index index_def)
         throws IndexAlreadyExistsException, MetaException {



Mime
View raw message