hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-10778 : LLAP: Utilities::gWorkMap needs to be cleaned in HiveServer2 (Sergey Shelukhin, reviewed by Vikram Dixit K)
Date Sat, 30 May 2015 00:05:16 GMT
Repository: hive
Updated Branches:
  refs/heads/llap be0c51685 -> e35f5c92b


HIVE-10778 : LLAP: Utilities::gWorkMap needs to be cleaned in HiveServer2 (Sergey Shelukhin,
reviewed by Vikram Dixit K)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e35f5c92
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e35f5c92
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e35f5c92

Branch: refs/heads/llap
Commit: e35f5c92b9e786cd932429bb7b3b9053787ca8c1
Parents: be0c516
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Fri May 29 17:05:00 2015 -0700
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Fri May 29 17:05:00 2015 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java | 3 ++-
 ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java         | 2 ++
 ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java       | 2 ++
 3 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e35f5c92/ql/src/java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java
index f3d8209..6fc56e8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java
@@ -10,6 +10,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.plan.BaseWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
 
 public class GlobalWorkMapFactory {
@@ -81,7 +82,7 @@ public class GlobalWorkMapFactory {
   DummyMap<Path, BaseWork> dummy = new DummyMap<Path, BaseWork>();
 
   public Map<Path, BaseWork> get(Configuration conf) {
-    if (LlapIoProxy.isDaemon()
+    if (LlapIoProxy.isDaemon() || SessionState.get().isHiveServerQuery()
         || HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
       if (threadLocalWorkMap == null) {
         threadLocalWorkMap = new ThreadLocal<Map<Path, BaseWork>>() {

http://git-wip-us.apache.org/repos/asf/hive/blob/e35f5c92/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 716d773..09864cb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -75,6 +75,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.zip.Deflater;
@@ -3721,6 +3722,7 @@ public final class Utilities {
     if (reducePath != null) {
       gWorkMap.get(conf).remove(reducePath);
     }
+    // TODO: should this also clean merge work?
   }
 
   public static void clearWorkMap(Configuration conf) {

http://git-wip-us.apache.org/repos/asf/hive/blob/e35f5c92/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 955c8a3..b2558d1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -363,6 +363,8 @@ public class TezTask extends Task<TezWork> {
         }
       }
     }
+    // Clear the work map after build. TODO: remove caching instead?
+    Utilities.clearWorkMap(conf);
     perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_BUILD_DAG);
     return dag;
   }


Mime
View raw message