hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject [1/5] hive git commit: HIVE-16644 : Hook Change Manager to Insert Overwrite (Sankar Hariappan via Thejas Nair)
Date Thu, 08 Jun 2017 23:40:27 GMT
Repository: hive
Updated Branches:
  refs/heads/master 16f68e3e6 -> a1869692a


http://git-wip-us.apache.org/repos/asf/hive/blob/a1869692/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------
diff --git a/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 2cf38b5..02c5717 100644
--- a/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -3080,6 +3080,41 @@ class GetTablesResult
   ::Thrift::Struct.generate_accessors self
 end
 
+class CmRecycleRequest
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  DATAPATH = 1
+  PURGE = 2
+
+  FIELDS = {
+    DATAPATH => {:type => ::Thrift::Types::STRING, :name => 'dataPath'},
+    PURGE => {:type => ::Thrift::Types::BOOL, :name => 'purge'}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required
field dataPath is unset!') unless @dataPath
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required
field purge is unset!') if @purge.nil?
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+
+class CmRecycleResponse
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+
+  FIELDS = {
+
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+
 class TableMeta
   include ::Thrift::Struct, ::Thrift::Struct_Union
   DBNAME = 1

http://git-wip-us.apache.org/repos/asf/hive/blob/a1869692/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
----------------------------------------------------------------------
diff --git a/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb b/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
index 2711381..cfe08f6 100644
--- a/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
+++ b/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
@@ -2553,6 +2553,22 @@ module ThriftHiveMetastore
       return
     end
 
+    def cm_recycle(request)
+      send_cm_recycle(request)
+      return recv_cm_recycle()
+    end
+
+    def send_cm_recycle(request)
+      send_message('cm_recycle', Cm_recycle_args, :request => request)
+    end
+
+    def recv_cm_recycle()
+      result = receive_message(Cm_recycle_result)
+      return result.success unless result.success.nil?
+      raise result.o1 unless result.o1.nil?
+      raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT,
'cm_recycle failed: unknown result')
+    end
+
     def get_file_metadata_by_expr(req)
       send_get_file_metadata_by_expr(req)
       return recv_get_file_metadata_by_expr()
@@ -4556,6 +4572,17 @@ module ThriftHiveMetastore
       write_result(result, oprot, 'flushCache', seqid)
     end
 
+    def process_cm_recycle(seqid, iprot, oprot)
+      args = read_args(iprot, Cm_recycle_args)
+      result = Cm_recycle_result.new()
+      begin
+        result.success = @handler.cm_recycle(args.request)
+      rescue ::MetaException => o1
+        result.o1 = o1
+      end
+      write_result(result, oprot, 'cm_recycle', seqid)
+    end
+
     def process_get_file_metadata_by_expr(seqid, iprot, oprot)
       args = read_args(iprot, Get_file_metadata_by_expr_args)
       result = Get_file_metadata_by_expr_result.new()
@@ -10360,6 +10387,40 @@ module ThriftHiveMetastore
     ::Thrift::Struct.generate_accessors self
   end
 
+  class Cm_recycle_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    REQUEST = 1
+
+    FIELDS = {
+      REQUEST => {:type => ::Thrift::Types::STRUCT, :name => 'request', :class =>
::CmRecycleRequest}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Cm_recycle_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    SUCCESS = 0
+    O1 = 1
+
+    FIELDS = {
+      SUCCESS => {:type => ::Thrift::Types::STRUCT, :name => 'success', :class =>
::CmRecycleResponse},
+      O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
   class Get_file_metadata_by_expr_args
     include ::Thrift::Struct, ::Thrift::Struct_Union
     REQ = 1

http://git-wip-us.apache.org/repos/asf/hive/blob/a1869692/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index e886540..4938fef 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -2066,6 +2066,12 @@ public class HiveMetaStore extends ThriftHiveMetastore {
     }
 
     @Override
+    public CmRecycleResponse cm_recycle(final CmRecycleRequest request) throws MetaException
{
+      wh.recycleDirToCmPath(new Path(request.getDataPath()), request.isPurge());
+      return new CmRecycleResponse();
+    }
+
+    @Override
     public void truncate_table(final String dbName, final String tableName, List<String>
partNames)
       throws NoSuchObjectException, MetaException {
       try {

http://git-wip-us.apache.org/repos/asf/hive/blob/a1869692/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 0ff4c11..2a3dcc4 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -1125,6 +1125,18 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
   }
 
   /**
+   * Recycles the files recursively from the input path to the cmroot directory either by
copying or moving it.
+   *
+   * @param request Inputs for path of the data files to be recycled to cmroot and
+   *                isPurge flag when set to true files which needs to be recycled are not
moved to Trash
+   * @return Response which is currently void
+   */
+  @Override
+  public CmRecycleResponse recycleDirToCmPath(CmRecycleRequest request) throws MetaException,
TException {
+    return client.cm_recycle(request);
+  }
+
+  /**
    * @param type
    * @return true if the type is dropped
    * @throws MetaException

http://git-wip-us.apache.org/repos/asf/hive/blob/a1869692/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 3663305..64a71df 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -32,10 +32,11 @@ import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
 import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.hive.common.classification.RetrySemantics;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.annotation.NoReconnect;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.CmRecycleRequest;
+import org.apache.hadoop.hive.metastore.api.CmRecycleResponse;
 import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
 import org.apache.hadoop.hive.metastore.api.CompactionResponse;
@@ -322,6 +323,15 @@ public interface IMetaStoreClient {
    */
   void truncateTable(String dbName, String tableName, List<String> partNames) throws
MetaException, TException;
 
+  /**
+   * Recycles the files recursively from the input path to the cmroot directory either by
copying or moving it.
+   *
+   * @param request Inputs for path of the data files to be recycled to cmroot and
+   *                isPurge flag when set to true files which needs to be recycled are not
moved to Trash
+   * @return Response which is currently void
+   */
+  CmRecycleResponse recycleDirToCmPath(CmRecycleRequest request) throws MetaException, TException;
+
   boolean tableExists(String databaseName, String tableName) throws MetaException,
       TException, UnknownDBException;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a1869692/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 053a0de..fcbcf62 100755
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@ -220,6 +220,11 @@ public class Warehouse {
     return fsHandler.deleteDir(fs, f, recursive, ifPurge, conf);
   }
 
+  public void recycleDirToCmPath(Path f, boolean ifPurge) throws MetaException {
+    cm.recycle(f, ifPurge);
+    return;
+  }
+
   public boolean isEmpty(Path path) throws IOException, MetaException {
     ContentSummary contents = getFs(path).getContentSummary(path);
     if (contents != null && contents.getFileCount() == 0 && contents.getDirectoryCount()
== 1) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a1869692/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ceb8079..5dbcfde 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -84,6 +84,7 @@ import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.CmRecycleRequest;
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
 import org.apache.hadoop.hive.metastore.api.CompactionResponse;
 import org.apache.hadoop.hive.metastore.api.CompactionType;
@@ -3171,6 +3172,22 @@ private void constructOneLBLocationMap(FileStatus fSta,
     }
   }
 
+  /**
+   * Recycles the files recursively from the input path to the cmroot directory either by
copying or moving it.
+   *
+   * @param dataPath Path of the data files to be recycled to cmroot
+   * @param isPurge
+   *          When set to true files which needs to be recycled are not moved to Trash
+   */
+  public void recycleDirToCmPath(Path dataPath, boolean isPurge) throws HiveException {
+    try {
+      CmRecycleRequest request = new CmRecycleRequest(dataPath.toString(), isPurge);
+      getMSC().recycleDirToCmPath(request);
+    } catch (Exception e) {
+      throw new HiveException(e);
+    }
+  }
+
   //it is assumed that parent directory of the destf should already exist when this
   //method is called. when the replace value is true, this method works a little different
   //from mv command if the destf is a directory, it replaces the destf instead of moving
under
@@ -3447,7 +3464,6 @@ private void constructOneLBLocationMap(FileStatus fSta,
     }
   }
 
-
   /**
    * Replaces files in the partition with new data set specified by srcf. Works
    * by renaming directory of srcf to the destination file.
@@ -3496,7 +3512,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
         FileStatus[] statuses = null;
         try {
           FileSystem oldFs = oldPath.getFileSystem(conf);
-          statuses = oldFs.listStatus(oldPath, FileUtils.HIDDEN_FILES_PATH_FILTER);
+
           // Do not delete oldPath if:
           //  - destf is subdir of oldPath
           isOldPathUnderDestf = isSubDir(oldPath, destf, oldFs, destFs, false);
@@ -3505,6 +3521,10 @@ private void constructOneLBLocationMap(FileStatus fSta,
             // existing content might result in incorrect (extra) data.
             // But not sure why we changed not to delete the oldPath in HIVE-8750 if it is
             // not the destf or its subdir?
+            if (conf.getBoolVar(HiveConf.ConfVars.REPLCMENABLED)) {
+              recycleDirToCmPath(oldPath, purge);
+            }
+            statuses = oldFs.listStatus(oldPath, FileUtils.HIDDEN_FILES_PATH_FILTER);
             oldPathDeleted = trashFiles(oldFs, statuses, conf, purge);
           }
         } catch (IOException e) {
@@ -3518,7 +3538,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
           }
         }
         if (statuses != null && statuses.length > 0) {
-          if (isOldPathUnderDestf && !oldPathDeleted) {
+          if (!oldPathDeleted) {
             throw new HiveException("Destination directory " + destf + " has not be cleaned
up.");
           }
         }


Mime
View raw message