hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject [53/84] [abbrv] hive git commit: HIVE-10781 : HadoopJobExecHelper Leaks RunningJobs (Chinna R Lalam via Ashutosh Chauhan)
Date Fri, 29 May 2015 04:47:32 GMT
HIVE-10781 : HadoopJobExecHelper Leaks RunningJobs (Chinna R Lalam via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c75167d6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c75167d6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c75167d6

Branch: refs/heads/spark
Commit: c75167d6e44e9c50f97e0ac26646a48f92cf36e8
Parents: 1f75e34
Author: Chinna R Lalam <chinna.r.lalam@intel.com>
Authored: Thu May 21 02:29:00 2015 -0700
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Mon May 25 23:10:30 2015 -0700

----------------------------------------------------------------------
 ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java  | 4 +++-
 .../java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java | 6 ++++--
 .../apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java | 4 +++-
 .../hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java  | 4 +++-
 4 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c75167d6/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index d7a08ec..e4f9543 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -462,10 +462,12 @@ public class ExecDriver extends Task<MapredWork> implements Serializable,
Hadoop
           if (returnVal != 0) {
             rj.killJob();
           }
-          HadoopJobExecHelper.runningJobs.remove(rj);
           jobID = rj.getID().toString();
         }
       } catch (Exception e) {
+	LOG.warn(e);
+      } finally {
+	HadoopJobExecHelper.runningJobs.remove(rj);
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/c75167d6/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
index 08ab1c9..bd50b46 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
@@ -176,7 +176,6 @@ public class MergeFileTask extends Task<MergeFileWork> implements
Serializable,
           if (returnVal != 0) {
             rj.killJob();
           }
-          HadoopJobExecHelper.runningJobs.remove(rj);
           jobID = rj.getID().toString();
         }
         // get the list of Dynamic partition paths
@@ -189,7 +188,8 @@ public class MergeFileTask extends Task<MergeFileWork> implements
Serializable,
           }
         }
       } catch (Exception e) {
-        // jobClose needs to execute successfully otherwise fail task
+	// jobClose needs to execute successfully otherwise fail task
+	LOG.warn(e);
         if (success) {
           success = false;
           returnVal = 3;
@@ -198,6 +198,8 @@ public class MergeFileTask extends Task<MergeFileWork> implements
Serializable,
           console.printError(mesg, "\n" +
               org.apache.hadoop.util.StringUtils.stringifyException(e));
         }
+      } finally {
+	HadoopJobExecHelper.runningJobs.remove(rj);
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/c75167d6/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
index f4ffdd6..3cb9e9c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
@@ -245,10 +245,12 @@ public class PartialScanTask extends Task<PartialScanWork> implements
           if (returnVal != 0) {
             rj.killJob();
           }
-          HadoopJobExecHelper.runningJobs.remove(rj);
           jobID = rj.getID().toString();
         }
       } catch (Exception e) {
+	LOG.warn(e);
+      } finally {
+	HadoopJobExecHelper.runningJobs.remove(rj);
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/c75167d6/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
index 4633820..149ad93 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
@@ -213,12 +213,14 @@ public class ColumnTruncateTask extends Task<ColumnTruncateWork>
implements Seri
           if (returnVal != 0) {
             rj.killJob();
           }
-          HadoopJobExecHelper.runningJobs.remove(rj);
           jobID = rj.getID().toString();
         }
         ColumnTruncateMapper.jobClose(outputPath, success, job, console,
           work.getDynPartCtx(), null);
       } catch (Exception e) {
+	LOG.warn(e);
+      } finally {
+	HadoopJobExecHelper.runningJobs.remove(rj);
       }
     }
 


Mime
View raw message