hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject hive git commit: HIVE-16323: HS2 JDOPersistenceManagerFactory.pmCache leaks after HIVE-14204 (Daniel Dai, reviewed by Thejas Nair)
Date Fri, 09 Jun 2017 21:46:38 GMT
Repository: hive
Updated Branches:
  refs/heads/master 8877e9821 -> 302360f51


HIVE-16323: HS2 JDOPersistenceManagerFactory.pmCache leaks after HIVE-14204 (Daniel Dai, reviewed
by Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/302360f5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/302360f5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/302360f5

Branch: refs/heads/master
Commit: 302360f51c2f6d93db244b6e67fc05517a654b2b
Parents: 8877e98
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Fri Jun 9 14:46:11 2017 -0700
Committer: Daniel Dai <daijy@hortonworks.com>
Committed: Fri Jun 9 14:46:11 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/metastore/ObjectStore.java   |  1 +
 .../org/apache/hadoop/hive/ql/metadata/Hive.java    | 16 ++++++++++++++++
 2 files changed, 17 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/302360f5/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 4676e15..d02c5da 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -562,6 +562,7 @@ public class ObjectStore implements RawStore, Configurable {
       LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm +
           " will be shutdown");
       pm.close();
+      pm = null;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/302360f5/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 5dbcfde..88c73f0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -74,10 +74,12 @@ import org.apache.hadoop.hive.io.HdfsUtils;
 import org.apache.hadoop.hive.metastore.HiveMetaException;
 import org.apache.hadoop.hive.metastore.HiveMetaHook;
 import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.PartitionDropOptions;
+import org.apache.hadoop.hive.metastore.RawStore;
 import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
 import org.apache.hadoop.hive.metastore.SynchronizedMetaStoreClient;
 import org.apache.hadoop.hive.metastore.TableType;
@@ -413,6 +415,9 @@ public class Hive {
       metaStoreClient.close();
       metaStoreClient = null;
     }
+    if (syncMetaStoreClient != null) {
+      syncMetaStoreClient.close();
+    }
     if (owner != null) {
       owner = null;
     }
@@ -1972,6 +1977,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
     try {
       // for each dynamically created DP directory, construct a full partition spec
       // and load the partition based on that
+      final Map<Long, RawStore> rawStoreMap = new HashMap<Long, RawStore>();
       for(final Path partPath : validPartitions) {
         // generate a full partition specification
         final LinkedHashMap<String, String> fullPartSpec = Maps.newLinkedHashMap(partSpec);
@@ -1998,6 +2004,12 @@ private void constructOneLBLocationMap(FileStatus fSta,
                       + partsToLoad + " partitions.");
                 }
               }
+              // Add embedded rawstore, so we can cleanup later to avoid memory leak
+              if (getMSC().isLocalMetaStore()) {
+                if (!rawStoreMap.containsKey(Thread.currentThread().getId())) {
+                  rawStoreMap.put(Thread.currentThread().getId(), HiveMetaStore.HMSHandler.getRawStore());
+                }
+              }
               return null;
             } catch (Exception t) {
               LOG.error("Exception when loading partition with parameters "
@@ -2019,6 +2031,10 @@ private void constructOneLBLocationMap(FileStatus fSta,
       for (Future future : futures) {
         future.get();
       }
+
+      for (RawStore rs : rawStoreMap.values()) {
+        rs.shutdown();
+      }
     } catch (InterruptedException | ExecutionException e) {
       LOG.debug("Cancelling " + futures.size() + " dynamic loading tasks");
       //cancel other futures


Mime
View raw message