drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [1/2] drill git commit: DRILL-4065: Maintain a cache of created UserGroupInformation objects so that we don't accidentally generate thousands of FileSystem objects & threads.
Date Wed, 11 Nov 2015 05:20:54 GMT
Repository: drill
Updated Branches:
  refs/heads/1.3.0 [created] e465cd90c


DRILL-4065: Maintain a cache of created UserGroupInformation objects so that we don't accidentally
generate thousands of FileSystem objects & threads.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ba4baea7
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ba4baea7
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ba4baea7

Branch: refs/heads/1.3.0
Commit: ba4baea736b24daac3d9b1a8d3ba198e412f6887
Parents: e78e286
Author: Jacques Nadeau <jacques@apache.org>
Authored: Tue Nov 10 16:04:09 2015 -0800
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Tue Nov 10 16:04:09 2015 -0800

----------------------------------------------------------------------
 .../drill/exec/util/ImpersonationUtil.java      | 80 ++++++++++++++++++--
 1 file changed, 72 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/ba4baea7/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
index 37fdc40..38d2797 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
@@ -17,18 +17,24 @@
  */
 package org.apache.drill.exec.util;
 
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.exec.ops.OperatorStats;
 import org.apache.drill.exec.store.dfs.DrillFileSystem;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.Set;
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Sets;
 
 /**
  * Utilities for impersonation purpose.
@@ -36,8 +42,66 @@ import java.util.Set;
 public class ImpersonationUtil {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ImpersonationUtil.class);
 
+  private static final LoadingCache<Key, UserGroupInformation> CACHE = CacheBuilder.newBuilder()
+      .maximumSize(100)
+      .expireAfterAccess(60, TimeUnit.MINUTES)
+      .build(new CacheLoader<Key, UserGroupInformation>() {
+        @Override
+        public UserGroupInformation load(Key key) throws Exception {
+          return UserGroupInformation.createProxyUser(key.proxyUserName, key.loginUser);
+        }
+      });
+
   private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
 
+  private static class Key {
+    final String proxyUserName;
+    final UserGroupInformation loginUser;
+
+    public Key(String proxyUserName, UserGroupInformation loginUser) {
+      super();
+      this.proxyUserName = proxyUserName;
+      this.loginUser = loginUser;
+    }
+    @Override
+    public int hashCode() {
+      final int prime = 31;
+      int result = 1;
+      result = prime * result + ((loginUser == null) ? 0 : loginUser.hashCode());
+      result = prime * result + ((proxyUserName == null) ? 0 : proxyUserName.hashCode());
+      return result;
+    }
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (obj == null) {
+        return false;
+      }
+      if (getClass() != obj.getClass()) {
+        return false;
+      }
+      Key other = (Key) obj;
+      if (loginUser == null) {
+        if (other.loginUser != null) {
+          return false;
+        }
+      } else if (!loginUser.equals(other.loginUser)) {
+        return false;
+      }
+      if (proxyUserName == null) {
+        if (other.proxyUserName != null) {
+          return false;
+        }
+      } else if (!proxyUserName.equals(other.proxyUserName)) {
+        return false;
+      }
+      return true;
+    }
+
+
+  }
   /**
    * Create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation}
of operator owner if operator
    * owner is valid. Otherwise create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation}
for
@@ -83,8 +147,8 @@ public class ImpersonationUtil {
         return getProcessUserUGI();
       }
 
-      return UserGroupInformation.createProxyUser(proxyUserName, UserGroupInformation.getLoginUser());
-    } catch(IOException e) {
+      return CACHE.get(new Key(proxyUserName, UserGroupInformation.getLoginUser()));
+    } catch (IOException | ExecutionException e) {
       final String errMsg = "Failed to create proxy user UserGroupInformation object: " +
e.getMessage();
       logger.error(errMsg, e);
       throw new DrillRuntimeException(errMsg, e);


Mime
View raw message