hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject [03/11] hive git commit: HIVE-13572 : Redundant setting full file status in Hive::copyFiles (Rui Li via Ashutosh Chauhan)
Date Sun, 01 May 2016 01:30:57 GMT
HIVE-13572 : Redundant setting full file status in Hive::copyFiles (Rui Li via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/076f3655
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/076f3655
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/076f3655

Branch: refs/heads/llap
Commit: 076f3655b846f3214bc304e405d321222a3819ab
Parents: 4377c7f
Author: Rui Li <rui.li@intel.com>
Authored: Tue Apr 26 18:14:00 2016 -0800
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Thu Apr 28 19:22:13 2016 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/metadata/Hive.java    | 39 +++++++++++---------
 .../apache/hadoop/hive/shims/ShimLoader.java    | 10 +++--
 2 files changed, 28 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/076f3655/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ab165f1..4d9c3d2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -2641,26 +2641,29 @@ private void constructOneLBLocationMap(FileStatus fSta,
         files = new FileStatus[] {src};
       }
 
-      for (FileStatus srcFile : files) {
-
-        final Path srcP = srcFile.getPath();
-        final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs);
-        // Strip off the file type, if any so we don't make:
-        // 000000_0.gz -> 000000_0.gz_copy_1
-        final String name;
-        final String filetype;
-        String itemName = srcP.getName();
-        int index = itemName.lastIndexOf('.');
-        if (index >= 0) {
-          filetype = itemName.substring(index);
-          name = itemName.substring(0, index);
-        } else {
-          name = itemName;
-          filetype = "";
-        }
+      final SessionState parentSession = SessionState.get();
+      for (final FileStatus srcFile : files) {
+
         futures.add(pool.submit(new Callable<ObjectPair<Path, Path>>() {
           @Override
           public ObjectPair<Path, Path> call() throws Exception {
+            SessionState.setCurrentSessionState(parentSession);
+            final Path srcP = srcFile.getPath();
+            final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs);
+            // Strip off the file type, if any so we don't make:
+            // 000000_0.gz -> 000000_0.gz_copy_1
+            final String name;
+            final String filetype;
+            String itemName = srcP.getName();
+            int index = itemName.lastIndexOf('.');
+            if (index >= 0) {
+              filetype = itemName.substring(index);
+              name = itemName.substring(0, index);
+            } else {
+              name = itemName;
+              filetype = "";
+            }
+
             Path destPath = new Path(destf, srcP.getName());
             if (!needToCopy && !isSrcLocal) {
               for (int counter = 1; !destFs.rename(srcP,destPath); counter++) {
@@ -2671,7 +2674,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
             }
 
             if (inheritPerms) {
-              ShimLoader.getHadoopShims().setFullFileStatus(conf, fullDestStatus, destFs,
destf);
+              ShimLoader.getHadoopShims().setFullFileStatus(conf, fullDestStatus, destFs,
destPath);
             }
             if (null != newFiles) {
               newFiles.add(destPath);

http://git-wip-us.apache.org/repos/asf/hive/blob/076f3655/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
index 0fe3169..28d3e48 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
@@ -31,7 +31,7 @@ import java.util.Map;
 public abstract class ShimLoader {
   public static String HADOOP23VERSIONNAME = "0.23";
 
-  private static HadoopShims hadoopShims;
+  private static volatile HadoopShims hadoopShims;
   private static JettyShims jettyShims;
   private static AppenderSkeleton eventCounter;
   private static HadoopThriftAuthBridge hadoopThriftAuthBridge;
@@ -88,9 +88,13 @@ public abstract class ShimLoader {
    * Factory method to get an instance of HadoopShims based on the
    * version of Hadoop on the classpath.
    */
-  public static synchronized HadoopShims getHadoopShims() {
+  public static HadoopShims getHadoopShims() {
     if (hadoopShims == null) {
-      hadoopShims = loadShims(HADOOP_SHIM_CLASSES, HadoopShims.class);
+      synchronized (ShimLoader.class) {
+        if (hadoopShims == null) {
+          hadoopShims = loadShims(HADOOP_SHIM_CLASSES, HadoopShims.class);
+        }
+      }
     }
     return hadoopShims;
   }


Mime
View raw message