hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1668521 - in /hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql: exec/spark/SparkPlanGenerator.java io/CombineHiveInputFormat.java
Date Mon, 23 Mar 2015 06:46:44 GMT
Author: lirui
Date: Mon Mar 23 06:46:44 2015
New Revision: 1668521

URL: http://svn.apache.org/r1668521
Log:
HIVE-10006: RSC has memory leak while execute multi queries.[Spark Branch] (Chengxiang via
Rui, reviewed by Xuefu)

Modified:
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java?rev=1668521&r1=1668520&r2=1668521&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
Mon Mar 23 06:46:44 2015
@@ -96,14 +96,20 @@ public class SparkPlanGenerator {
     workToTranMap.clear();
     workToParentWorkTranMap.clear();
 
-    for (BaseWork work : sparkWork.getAllWork()) {
-      perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_CREATE_TRAN + work.getName());
-      SparkTran tran = generate(work);
-      SparkTran parentTran = generateParentTran(sparkPlan, sparkWork, work);
-      sparkPlan.addTran(tran);
-      sparkPlan.connect(parentTran, tran);
-      workToTranMap.put(work, tran);
-      perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_CREATE_TRAN + work.getName());
+    try {
+      for (BaseWork work : sparkWork.getAllWork()) {
+        perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_CREATE_TRAN + work.getName());
+        SparkTran tran = generate(work);
+        SparkTran parentTran = generateParentTran(sparkPlan, sparkWork, work);
+        sparkPlan.addTran(tran);
+        sparkPlan.connect(parentTran, tran);
+        workToTranMap.put(work, tran);
+        perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_CREATE_TRAN + work.getName());
+      }
+    } finally {
+      // clear all ThreadLocal cached MapWork/ReduceWork after plan generation
+      // as this may executed in a pool thread.
+      Utilities.clearWorkMap();
     }
 
     perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_BUILD_PLAN);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=1668521&r1=1668520&r2=1668521&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
Mon Mar 23 06:46:44 2015
@@ -580,6 +580,10 @@ public class CombineHiveInputFormat<K ex
     if (oldPaths != null) {
       job.set(HiveConf.ConfVars.HADOOPMAPREDINPUTDIR.varname, oldPaths);
     }
+
+    // clear work from ThreadLocal after splits generated in case of thread is reused in
pool.
+    Utilities.clearWorkMapForConf(job);
+
     LOG.info("Number of all splits " + result.size());
     perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS);
     return result.toArray(new InputSplit[result.size()]);



Mime
View raw message