hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r1095343 - in /hadoop/mapreduce/branches/MR-279: ./ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/ mr-clien...
Date Wed, 20 Apr 2011 09:48:11 GMT
Author: sharad
Date: Wed Apr 20 09:48:10 2011
New Revision: 1095343

URL: http://svn.apache.org/viewvc?rev=1095343&view=rev
Log:
Fix race condition in TestJobHistoryEvents and TestJobHistoryParsing.

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1095343&r1=1095342&r2=1095343&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Wed Apr 20 09:48:10 2011
@@ -3,6 +3,7 @@ Hadoop MapReduce Change Log
 Trunk (unreleased changes)
 
   MAPREDUCE-279
+    Fix race condition in TestJobHistoryEvents and TestJobHistoryParsing. (sharad)
     
     Added functionality to stop/start queues. (acmurthy)
     

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1095343&r1=1095342&r2=1095343&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Wed Apr 20 09:48:10 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -36,9 +37,9 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.YarnMRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.YarnMRJobConfig;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -147,12 +148,33 @@ public class JobHistoryEventHandler exte
   @Override
   public void stop() {
     stopped = true;
-    eventHandlingThread.interrupt();
+    //do not interrupt while event handling is in progress
+    synchronized(this) {
+      eventHandlingThread.interrupt();
+    }
+
     try {
       eventHandlingThread.join();
     } catch (InterruptedException ie) {
       LOG.info("Interruped Exception while stopping", ie);
     }
+    //write all the events remaining in queue
+    Iterator<JobHistoryEvent> it = eventQueue.iterator();
+    while(it.hasNext()) {
+      JobHistoryEvent ev = it.next();
+      LOG.info("In stop, writing event " + ev.getType());
+      handleEvent(ev);
+    }
+    
+    //close all file handles
+    for (MetaInfo mi : fileMap.values()) {
+      try {
+        mi.closeWriter();
+      } catch (IOException e) {
+        LOG.info("Exception while closing file " + e.getMessage());
+      }
+    }
+
     super.stop();
   }
 
@@ -219,7 +241,7 @@ public class JobHistoryEventHandler exte
     }
   }
 
-  protected void handleEvent(JobHistoryEvent event) {
+  protected synchronized void handleEvent(JobHistoryEvent event) {
     // check for first event from a job
     if (event.getHistoryEvent().getEventType() == EventType.JOB_SUBMITTED) {
       try {
@@ -314,6 +336,7 @@ public class JobHistoryEventHandler exte
     synchronized void writeEvent(HistoryEvent event) throws IOException {
       if (writer != null) {
         writer.write(event);
+        writer.flush();
       }
     }
   }

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java?rev=1095343&r1=1095342&r2=1095343&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java
Wed Apr 20 09:48:10 2011
@@ -53,6 +53,7 @@ public class TestJobHistoryParsing {
     JobId jobId = job.getID();
     LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
     app.waitForState(job, JobState.SUCCEEDED);
+    app.stop();
     
     String jobhistoryFileName = TypeConverter.fromYarn(jobId).toString();
     String user =

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java?rev=1095343&r1=1095342&r2=1095343&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
Wed Apr 20 09:48:10 2011
@@ -58,6 +58,7 @@ public class TestJobHistoryEvents {
     JobId jobId = job.getID();
     LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
     app.waitForState(job, JobState.SUCCEEDED);
+    app.stop();
     /*
      * Use HistoryContext to read logged events and verify the number of 
      * completed maps 



Mime
View raw message