hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From deva...@apache.org
Subject hadoop git commit: MAPREDUCE-6339. Job history file is not flushed correctly because isTimerActive flag is not set true when flushTimerTask is scheduled. Contributed by zhihai xu.
Date Thu, 30 Apr 2015 07:12:18 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 6ac2b5712 -> 9828638fa


MAPREDUCE-6339. Job history file is not flushed correctly because
isTimerActive flag is not set true when flushTimerTask is scheduled.
Contributed by zhihai xu.

(cherry picked from commit f5b38477f9d0827b238fadd260c1dd2889531fd4)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9828638f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9828638f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9828638f

Branch: refs/heads/branch-2.7
Commit: 9828638fac850ff45bf8a80dd29f0254bc61f35e
Parents: 6ac2b57
Author: Devaraj K <devaraj@apache.org>
Authored: Thu Apr 30 12:33:22 2015 +0530
Committer: Devaraj K <devaraj@apache.org>
Committed: Thu Apr 30 12:37:36 2015 +0530

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt                          | 3 +++
 .../hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java   | 7 +++++++
 .../mapreduce/jobhistory/TestJobHistoryEventHandler.java      | 2 ++
 3 files changed, 12 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9828638f/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 10491f9..11475d6 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -26,6 +26,9 @@ Release 2.7.1 - UNRELEASED
     MAPREDUCE-6334. Fetcher#copyMapOutput is leaking usedMemory upon
     IOException during InMemoryMapOutput shuffle handler (Eric Payne via jlowe)
 
+    MAPREDUCE-6339. Job history file is not flushed correctly because isTimerActive 
+    flag is not set true when flushTimerTask is scheduled. (zhihai xu via devaraj)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9828638f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index a0e7041..6b0ea79 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -72,6 +72,7 @@ import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.node.ArrayNode;
 import org.codehaus.jackson.node.ObjectNode;
 
+import com.google.common.annotations.VisibleForTesting;
 /**
  * The job history events get routed to this class. This class writes the Job
  * history events to the DFS directly into a staging dir and then moved to a
@@ -1259,6 +1260,7 @@ public class JobHistoryEventHandler extends AbstractService
           if (!isTimerShutDown) {
             flushTimerTask = new FlushTimerTask(this);
             flushTimer.schedule(flushTimerTask, flushTimeout);
+            isTimerActive = true;
           }
         }
       }
@@ -1378,4 +1380,9 @@ public class JobHistoryEventHandler extends AbstractService
     }
     return JobState.KILLED.toString();
   }
+
+  @VisibleForTesting
+  boolean getFlushTimerStatus() {
+    return isTimerActive;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9828638f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 43e3dbe..7ed0b63 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -223,10 +223,12 @@ public class TestJobHistoryEventHandler {
       }
 
       handleNextNEvents(jheh, 9);
+      Assert.assertTrue(jheh.getFlushTimerStatus());
       verify(mockWriter, times(0)).flush();
 
       Thread.sleep(2 * 4 * 1000l); // 4 seconds should be enough. Just be safe.
       verify(mockWriter).flush();
+      Assert.assertFalse(jheh.getFlushTimerStatus());
     } finally {
       jheh.stop();
       verify(mockWriter).close();


Mime
View raw message