Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id EB32118B0D for ; Mon, 28 Sep 2015 19:10:07 +0000 (UTC) Received: (qmail 99013 invoked by uid 500); 28 Sep 2015 19:10:05 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 98870 invoked by uid 500); 28 Sep 2015 19:10:05 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 97359 invoked by uid 99); 28 Sep 2015 19:10:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 28 Sep 2015 19:10:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E853DE0A08; Mon, 28 Sep 2015 19:10:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sershe@apache.org To: commits@hive.apache.org Date: Mon, 28 Sep 2015 19:10:39 -0000 Message-Id: In-Reply-To: <533decb8e0e44ab9b9c45957ea71aceb@git.apache.org> References: <533decb8e0e44ab9b9c45957ea71aceb@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [37/43] hive git commit: HIVE-11940: "INSERT OVERWRITE" query is very slow because it creates one "distcp" per file to copy data from staging directory to target directory (Sergio Pena, reviewd by Ferdinand Xu) HIVE-11940: "INSERT OVERWRITE" query is very slow because it creates one "distcp" per file to copy data from staging directory to target directory (Sergio Pena, reviewd by Ferdinand Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ba21806b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ba21806b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ba21806b Branch: refs/heads/llap Commit: ba21806b77287e237e1aa68fa169d2a81e07346d Parents: abe622b Author: Sergio Pena Authored: Mon Sep 28 09:58:08 2015 -0500 Committer: Sergio Pena Committed: Mon Sep 28 09:58:08 2015 -0500 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/metadata/Hive.java | 25 ++++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ba21806b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 99896c6..10cafb6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2686,16 +2686,31 @@ private void constructOneLBLocationMap(FileStatus fSta, if (srcs.length == 0) { success = true; // Nothing to move. } + + /* Move files one by one because source is a subdirectory of destination */ for (FileStatus status : srcs) { - success = FileUtils.copy(srcf.getFileSystem(conf), status.getPath(), destf.getFileSystem(conf), destf, - true, // delete source - replace, // overwrite destination - conf); + Path destFile; - if (!success) { + /* Append the source filename to the destination directory */ + if (destFs.isDirectory(destf)) { + destFile = new Path(destf, status.getPath().getName()); + } else { + destFile = destf; + } + + // Destination should be replaced, so we delete it first + if (destFs.exists(destFile)) { + if (!destFs.delete(destFile, true)) { + throw new HiveException(String.format("File to replace could not be deleted: %s", destFile)); + } + } + + if (!(destFs.rename(status.getPath(), destFile))) { throw new HiveException("Unable to move source " + status.getPath() + " to destination " + destf); } } + + success = true; } else { success = destFs.rename(srcf, destf); }