hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject hive git commit: HIVE-14380: Queries on tables with remote HDFS paths fail in "encryption" checks. (Mithun Radhakrishnan, reviewed by Sergio Pena)
Date Wed, 03 Aug 2016 15:58:51 GMT
Repository: hive
Updated Branches:
  refs/heads/master 7f43dcb39 -> 4817d8bd9


HIVE-14380: Queries on tables with remote HDFS paths fail in "encryption" checks. (Mithun
Radhakrishnan, reviewed by Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4817d8bd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4817d8bd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4817d8bd

Branch: refs/heads/master
Commit: 4817d8bd9bbffcf836c940016c97ce8d8d316542
Parents: 7f43dcb
Author: Mithun Radhakrishnan <mythrocks@gmail.com>
Authored: Wed Aug 3 10:57:42 2016 -0500
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Wed Aug 3 10:57:42 2016 -0500

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/metadata/Hive.java    |  9 +++++---
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  | 11 ++++-----
 .../hadoop/hive/ql/session/SessionState.java    | 24 +++++++++++++++-----
 .../apache/hadoop/hive/shims/Hadoop23Shims.java | 21 +++++++++++++----
 .../apache/hadoop/hive/shims/HadoopShims.java   | 17 ++++++++++++++
 5 files changed, 63 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 9a24ec3..57433bb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -3034,10 +3034,13 @@ private void constructOneLBLocationMap(FileStatus fSta,
     }
 
     //Check if different encryption zones
-    HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim();
+    HadoopShims.HdfsEncryptionShim srcHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(srcFs);
+    HadoopShims.HdfsEncryptionShim destHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(destFs);
     try {
-      return hdfsEncryptionShim != null && (hdfsEncryptionShim.isPathEncrypted(srcf)
|| hdfsEncryptionShim.isPathEncrypted(destf))
-        && !hdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf);
+      return srcHdfsEncryptionShim != null
+          && destHdfsEncryptionShim != null
+          && (srcHdfsEncryptionShim.isPathEncrypted(srcf) || destHdfsEncryptionShim.isPathEncrypted(destf))
+          && !srcHdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf, destHdfsEncryptionShim);
     } catch (IOException e) {
       throw new HiveException(e);
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 2671cb1..a01a7bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -2196,17 +2196,16 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
    * @throws HiveException If an error occurs while checking for encryption
    */
   private boolean isPathEncrypted(Path path) throws HiveException {
-    HadoopShims.HdfsEncryptionShim hdfsEncryptionShim;
 
-    hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim();
-    if (hdfsEncryptionShim != null) {
-      try {
+    try {
+      HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(path.getFileSystem(conf));
+      if (hdfsEncryptionShim != null) {
         if (hdfsEncryptionShim.isPathEncrypted(path)) {
           return true;
         }
-      } catch (Exception e) {
-        throw new HiveException("Unable to determine if " + path + " is encrypted: " + e,
e);
       }
+    } catch (Exception e) {
+        throw new HiveException("Unable to determine if " + path + " is encrypted: " + e,
e);
     }
 
     return false;

http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 83a8610..0581bab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -42,6 +42,7 @@ import java.util.UUID;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.locks.ReentrantLock;
 
+import com.google.common.collect.Maps;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.ArrayUtils;
 import org.slf4j.Logger;
@@ -211,7 +212,7 @@ public class SessionState {
   /**
    * Gets information about HDFS encryption
    */
-  private HadoopShims.HdfsEncryptionShim hdfsEncryptionShim;
+  private Map<URI, HadoopShims.HdfsEncryptionShim> hdfsEncryptionShims = Maps.newHashMap();
 
   /**
    * Lineage state.
@@ -438,20 +439,31 @@ public class SessionState {
   }
 
   public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() throws HiveException {
-    if (hdfsEncryptionShim == null) {
+    try {
+      return getHdfsEncryptionShim(FileSystem.get(sessionConf));
+    }
+    catch(HiveException hiveException) {
+      throw hiveException;
+    }
+    catch(Exception exception) {
+      throw new HiveException(exception);
+    }
+  }
+
+  public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim(FileSystem fs) throws HiveException
{
+    if (!hdfsEncryptionShims.containsKey(fs.getUri())) {
       try {
-        FileSystem fs = FileSystem.get(sessionConf);
         if ("hdfs".equals(fs.getUri().getScheme())) {
-          hdfsEncryptionShim = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, sessionConf);
+          hdfsEncryptionShims.put(fs.getUri(), ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs,
sessionConf));
         } else {
-          LOG.debug("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem.");
+          LOG.info("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem.");
         }
       } catch (Exception e) {
         throw new HiveException(e);
       }
     }
 
-    return hdfsEncryptionShim;
+    return hdfsEncryptionShims.get(fs.getUri());
   }
 
   // SessionState is not available in runtime and Hive.get().getConf() is not safe to call

http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index a4c7808..84b1174 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1124,11 +1124,11 @@ public class Hadoop23Shims extends HadoopShimsSecure {
 
     @Override
     public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException
{
-      EncryptionZone zone1, zone2;
-
-      zone1 = hdfsAdmin.getEncryptionZoneForPath(path1);
-      zone2 = hdfsAdmin.getEncryptionZoneForPath(path2);
+      return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1),
+                                       hdfsAdmin.getEncryptionZoneForPath(path2));
+    }
 
+    private boolean equivalentEncryptionZones(EncryptionZone zone1, EncryptionZone zone2)
{
       if (zone1 == null && zone2 == null) {
         return true;
       } else if (zone1 == null || zone2 == null) {
@@ -1139,6 +1139,19 @@ public class Hadoop23Shims extends HadoopShimsSecure {
     }
 
     @Override
+    public boolean arePathsOnSameEncryptionZone(Path path1, Path path2,
+                                                HadoopShims.HdfsEncryptionShim encryptionShim2)
throws IOException {
+      if (!(encryptionShim2 instanceof Hadoop23Shims.HdfsEncryptionShim)) {
+        LOG.warn("EncryptionShim for path2 (" + path2 + ") is of unexpected type: " + encryptionShim2.getClass()
+            + ". Assuming path2 is on the same EncryptionZone as path1(" + path1 + ").");
+        return true;
+      }
+
+      return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1),
+          ((HdfsEncryptionShim)encryptionShim2).hdfsAdmin.getEncryptionZoneForPath(path2));
+    }
+
+    @Override
     public int comparePathKeyStrength(Path path1, Path path2) throws IOException {
       EncryptionZone zone1, zone2;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 9b0bc35..04f075b 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -512,6 +512,17 @@ public interface HadoopShims {
     public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException;
 
     /**
+     * Checks if two HDFS paths are on the same encrypted or unencrypted zone.
+     *
+     * @param path1 Path to HDFS file system
+     * @param path2 Path to HDFS file system
+     * @param encryptionShim2 The encryption-shim corresponding to path2.
+     * @return True if both paths are in the same zone; False otherwise.
+     * @throws IOException If an error occurred attempting to get encryption information
+     */
+    public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim
encryptionShim2) throws IOException;
+
+    /**
      * Compares two encrypted path strengths.
      *
      * @param path1 HDFS path to compare.
@@ -566,6 +577,12 @@ public interface HadoopShims {
     }
 
     @Override
+    public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim
encryptionShim2) throws IOException {
+      // Not supported.
+      return true;
+    }
+
+    @Override
     public int comparePathKeyStrength(Path path1, Path path2) throws IOException {
     /* not supported */
       return 0;


Mime
View raw message