hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject [33/50] [abbrv] hive git commit: HIVE-11824: Insert to local directory causes staging directory to be copied (Prasanth Jayachandran reviewed by Ashutosh Chauhan)
Date Wed, 16 Sep 2015 19:18:51 GMT
HIVE-11824: Insert to local directory causes staging directory to be copied (Prasanth Jayachandran
reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/03f46b2f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/03f46b2f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/03f46b2f

Branch: refs/heads/spark
Commit: 03f46b2f2eb0235e9a2079079e6d643e8cf7c89e
Parents: ba0b33c
Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Authored: Wed Sep 16 01:08:02 2015 -0500
Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Committed: Wed Sep 16 01:08:02 2015 -0500

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/ql/exec/MoveTask.java    | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/03f46b2f/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index a1f8973..6a19cc3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -127,10 +127,22 @@ public class MoveTask extends Task<MoveWork> implements Serializable
{
       LocalFileSystem dstFs = FileSystem.getLocal(conf);
 
       if (dstFs.delete(targetPath, true) || !dstFs.exists(targetPath)) {
-        console.printInfo(mesg, mesg_detail);
         // if source exists, rename. Otherwise, create a empty directory
         if (fs.exists(sourcePath)) {
-          fs.copyToLocalFile(sourcePath, targetPath);
+          try {
+            // create the destination if it does not exist
+            if (!dstFs.exists(targetPath)) {
+              FileUtils.mkdir(dstFs, targetPath, false, conf);
+            }
+          } catch (IOException e) {
+            throw new HiveException("Unable to create target directory for copy" + targetPath,
e);
+          }
+
+          FileSystem srcFs = sourcePath.getFileSystem(conf);
+          FileStatus[] srcs = srcFs.listStatus(sourcePath, FileUtils.HIDDEN_FILES_PATH_FILTER);
+          for (FileStatus status : srcs) {
+            fs.copyToLocalFile(status.getPath(), targetPath);
+          }
         } else {
           if (!dstFs.mkdirs(targetPath)) {
             throw new HiveException("Unable to make local directory: "


Mime
View raw message