hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject hive git commit: HIVE-16225: Memory leak in webhcat service (FileSystem CACHE entries) (Daniel Dai, reviewec by Thejas Nair)
Date Mon, 03 Apr 2017 19:01:43 GMT
Repository: hive
Updated Branches:
  refs/heads/master 5bf52be60 -> a4a0ae1ff


HIVE-16225: Memory leak in webhcat service (FileSystem CACHE entries) (Daniel Dai, reviewec
by Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a4a0ae1f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a4a0ae1f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a4a0ae1f

Branch: refs/heads/master
Commit: a4a0ae1ffd3e7d12e3ab4d94c7cf2af0b22c02bd
Parents: 5bf52be
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Mon Apr 3 12:01:26 2017 -0700
Committer: Daniel Dai <daijy@hortonworks.com>
Committed: Mon Apr 3 12:01:26 2017 -0700

----------------------------------------------------------------------
 .../hcatalog/templeton/DeleteDelegator.java     |  6 ++-
 .../hcatalog/templeton/LauncherDelegator.java   | 47 +++++++++++++-------
 .../hive/hcatalog/templeton/ListDelegator.java  |  6 ++-
 .../hcatalog/templeton/SecureProxySupport.java  |  3 ++
 .../hcatalog/templeton/StatusDelegator.java     |  6 ++-
 .../hcatalog/templeton/tool/TempletonUtils.java |  1 +
 6 files changed, 51 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java
index 4b2dfec..622f92d 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java
@@ -24,6 +24,7 @@ import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.mapred.JobID;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -41,10 +42,11 @@ public class DeleteDelegator extends TempletonDelegator {
   public QueueStatusBean run(String user, String id)
     throws NotAuthorizedException, BadParam, IOException, InterruptedException
   {
-    UserGroupInformation ugi = UgiFactory.getUgi(user);
+    UserGroupInformation ugi = null;
     WebHCatJTShim tracker = null;
     JobState state = null;
     try {
+      ugi = UgiFactory.getUgi(user);
       tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi);
       JobID jobid = StatusDelegator.StringToJobID(id);
       if (jobid == null)
@@ -69,6 +71,8 @@ public class DeleteDelegator extends TempletonDelegator {
         tracker.close();
       if (state != null)
         state.close();
+      if (ugi != null)
+        FileSystem.closeAllForUGI(ugi);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
index 1455316..9bea897 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
@@ -28,6 +28,7 @@ import java.util.concurrent.TimeoutException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.shims.HadoopShimsSecure;
@@ -178,8 +179,9 @@ public class LauncherDelegator extends TempletonDelegator {
                      List<String> args, TempletonControllerJob controllerJob)
     throws NotAuthorizedException, BusyException,
     IOException, QueueException {
+    UserGroupInformation ugi = null;
     try {
-      UserGroupInformation ugi = UgiFactory.getUgi(user);
+      ugi = UgiFactory.getUgi(user);
 
       final long startTime = System.nanoTime();
 
@@ -197,6 +199,10 @@ public class LauncherDelegator extends TempletonDelegator {
       return new EnqueueBean(id);
     } catch (InterruptedException e) {
       throw new QueueException("Unable to launch job " + e);
+    } finally {
+      if (ugi != null) {
+        FileSystem.closeAllForUGI(ugi);
+      }
     }
   }
 
@@ -344,24 +350,35 @@ public class LauncherDelegator extends TempletonDelegator {
    */
   private String getShimLibjars() {
     WebHCatJTShim shim = null;
+    UserGroupInformation ugi = null;
     try {
-      shim = ShimLoader.getHadoopShims().getWebHCatShim(appConf, UserGroupInformation.getCurrentUser());
+      ugi = UserGroupInformation.getCurrentUser();
+      shim = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi);
+
+      // Besides the HiveShims jar which is Hadoop version dependent we also
+      // always need to include hive shims common jars.
+      Path shimCommonJar = new Path(
+          TempletonUtils.findContainingJar(ShimLoader.class, HIVE_SHIMS_FILENAME_PATTERN));
+      Path shimCommonSecureJar = new Path(
+          TempletonUtils.findContainingJar(HadoopShimsSecure.class, HIVE_SHIMS_FILENAME_PATTERN));
+      Path shimJar = new Path(
+          TempletonUtils.findContainingJar(shim.getClass(), HIVE_SHIMS_FILENAME_PATTERN));
+
+      return String.format(
+          "%s,%s,%s",
+          shimCommonJar.toString(), shimCommonSecureJar.toString(), shimJar.toString());
     } catch (IOException e) {
-      throw new RuntimeException("Failed to get WebHCatShim", e);
+      throw new RuntimeException("Failed to get shimLibJars", e);
+    } finally {
+      try {
+        if (ugi != null) {
+          FileSystem.closeAllForUGI(ugi);
+        }
+      } catch (IOException e) {
+        throw new RuntimeException("Failed to closeAllForUGI", e);
+      }
     }
 
-    // Besides the HiveShims jar which is Hadoop version dependent we also
-    // always need to include hive shims common jars.
-    Path shimCommonJar = new Path(
-        TempletonUtils.findContainingJar(ShimLoader.class, HIVE_SHIMS_FILENAME_PATTERN));
-    Path shimCommonSecureJar = new Path(
-        TempletonUtils.findContainingJar(HadoopShimsSecure.class, HIVE_SHIMS_FILENAME_PATTERN));
-    Path shimJar = new Path(
-        TempletonUtils.findContainingJar(shim.getClass(), HIVE_SHIMS_FILENAME_PATTERN));
-
-    return String.format(
-        "%s,%s,%s",
-        shimCommonJar.toString(), shimCommonSecureJar.toString(), shimJar.toString());
   }
 
   // Storage vars

http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java
index ed4cea9..dfa59f8 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeoutException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.mapred.JobStatus;
@@ -104,11 +105,12 @@ public class ListDelegator extends TempletonDelegator {
                                     int numRecords, boolean showDetails)
     throws NotAuthorizedException, BadParam, IOException, InterruptedException {
 
-    UserGroupInformation ugi = UgiFactory.getUgi(user);
+    UserGroupInformation ugi = null;
     WebHCatJTShim tracker = null;
     ArrayList<String> ids = new ArrayList<String>();
 
     try {
+      ugi = UgiFactory.getUgi(user);
       tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi);
 
       JobStatus[] jobs = tracker.getAllJobs();
@@ -125,6 +127,8 @@ public class ListDelegator extends TempletonDelegator {
     } finally {
       if (tracker != null)
         tracker.close();
+      if (ugi != null)
+        FileSystem.closeAllForUGI(ugi);
     }
 
     return getJobStatus(ids, user, showall, jobId, numRecords, showDetails);

http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
index 5aed3b3..793881b 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
@@ -170,6 +170,7 @@ public class SecureProxySupport {
         return null;
       }
     });
+    FileSystem.closeAllForUGI(ugi);
     return twrapper.tokens;
   }
   private static void collectTokens(FileSystem fs, TokenWrapper twrapper, Credentials creds,
String userName) throws IOException {
@@ -204,6 +205,7 @@ public class SecureProxySupport {
         return null;
       }
     });
+    FileSystem.closeAllForUGI(ugi);
 
   }
 
@@ -220,6 +222,7 @@ public class SecureProxySupport {
         return client.getDelegationToken(c.getUser(), u);
       }
     });
+    FileSystem.closeAllForUGI(ugi);
     return s;
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java
index 4112eef..c042ae8 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java
@@ -25,6 +25,7 @@ import java.util.concurrent.TimeoutException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.mapred.JobID;
@@ -112,8 +113,9 @@ public class StatusDelegator extends TempletonDelegator {
   {
     WebHCatJTShim tracker = null;
     JobState state = null;
+    UserGroupInformation ugi = null;
     try {
-      UserGroupInformation ugi = UgiFactory.getUgi(user);
+      ugi = UgiFactory.getUgi(user);
       tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi);
       JobID jobid = StatusDelegator.StringToJobID(id);
       if (jobid == null)
@@ -127,6 +129,8 @@ public class StatusDelegator extends TempletonDelegator {
         tracker.close();
       if (state != null)
         state.close();
+      if (ugi != null)
+        FileSystem.closeAllForUGI(ugi);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
index 07b005b..e0ccc70 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
@@ -362,6 +362,7 @@ public class TempletonUtils {
     if (hadoopFsIsMissing(defaultFs, p))
       throw new FileNotFoundException("File " + fname + " does not exist.");
 
+    FileSystem.closeAllForUGI(ugi);
     return p;
   }
 


Mime
View raw message