hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l..@apache.org
Subject svn commit: r1138689 - in /hadoop/common/branches/MR-279/mapreduce: ./ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/
Date Wed, 22 Jun 2011 23:38:49 GMT
Author: llu
Date: Wed Jun 22 23:38:49 2011
New Revision: 1138689

URL: http://svn.apache.org/viewvc?rev=1138689&view=rev
Log:
MAPREDUCE-2615. Make killJob go through AM and fix JobSummaryLog. (Siddharth Seth via llu)

Modified:
    hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
    hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
    hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
    hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java

Modified: hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/CHANGES.txt?rev=1138689&r1=1138688&r2=1138689&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/MR-279/mapreduce/CHANGES.txt Wed Jun 22 23:38:49 2011
@@ -5,6 +5,9 @@ Trunk (unreleased changes)
 
     MAPREDUCE-279
 
+    MAPREDUCE-2615. Make killJob go through AM and fix JobSummaryLog.
+    (Siddharth Seth via llu)
+
     Fix for NPE in YarnChild that was causing lots of tasks to fail. (vinodkv)
 
     Fix for ConcurrentModification exception while iterating through tokens in

Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1138689&r1=1138688&r2=1138689&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Wed Jun 22 23:38:49 2011
@@ -385,7 +385,7 @@ public class JobHistoryEventHandler exte
       try {
         HistoryEvent historyEvent = event.getHistoryEvent();
         mi.writeEvent(historyEvent);
-        processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary());
+        processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(), event.getJobID());
         LOG.info("In HistoryEventHandler "
             + event.getHistoryEvent().getEventType());
       } catch (IOException e) {
@@ -427,7 +427,7 @@ public class JobHistoryEventHandler exte
     }
   }
 
-  private void processEventForJobSummary(HistoryEvent event, JobSummary summary) {
+  private void processEventForJobSummary(HistoryEvent event, JobSummary summary, JobId jobId)
{
     // context.getJob could be used for some of this info as well.
     switch (event.getEventType()) {
     case JOB_SUBMITTED:
@@ -467,9 +467,10 @@ public class JobHistoryEventHandler exte
     case JOB_KILLED:
       JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event;
       summary.setJobStatus(juce.getStatus());
+      summary.setNumFinishedMaps(context.getJob(jobId).getTotalMaps());
+      summary.setNumFinishedReduces(context.getJob(jobId).getTotalReduces());
+      summary.setJobFinishTime(juce.getFinishTime());
       break;
-    // TODO Verify: MRV2 + MRV1. A JOB_FINISHED event will always come in after
-    // this. Stats on taskCounts can be set via that.
     }
   }
 

Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1138689&r1=1138688&r2=1138689&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
Wed Jun 22 23:38:49 2011
@@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest;
 import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest;
 import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest;
 import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest;
 import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
 import org.apache.hadoop.mapreduce.v2.api.records.JobState;
@@ -468,4 +469,38 @@ public class ClientServiceDelegate {
     }
     return true;
   }
+  
+  public boolean killJob(JobID oldJobID)
+  throws YarnRemoteException {
+    org.apache.hadoop.mapreduce.v2.api.records.JobId jobId 
+    = TypeConverter.toYarn(oldJobID);
+    KillJobRequest killRequest = recordFactory.newRecordInstance(KillJobRequest.class);
+    MRClientProtocol protocol = getProxy(oldJobID);
+    if (protocol == null) {
+      return false;
+    }
+    try {
+      killRequest.setJobId(jobId);
+      protocol.killJob(killRequest);
+      return true;
+    } catch(YarnRemoteException yre) {//thrown by remote server, no need to redirect
+      LOG.warn(RPCUtil.toString(yre));
+      throw yre;
+    } catch(Exception e) {
+      // Not really requied - if this is always the history context.
+      LOG.debug("Failed to contact application master ", e);
+      MRClientProtocol proxy = getRefreshedProxy(oldJobID);
+      if (proxy == null) {
+        return false;
+      }
+      try {
+        killRequest.setJobId(jobId);
+        protocol.killJob(killRequest);
+        return true;
+      } catch(YarnRemoteException yre) {
+        LOG.warn(RPCUtil.toString(yre));
+        throw yre;
+      }
+    }
+  }
 }

Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1138689&r1=1138688&r2=1138689&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Wed Jun 22 23:38:49 2011
@@ -537,8 +537,10 @@ public class YARNRunner implements Clien
 
   @Override
   public void killJob(JobID arg0) throws IOException, InterruptedException {
+    if (!clientServiceDelegate.killJob(arg0)) {
     resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
   }
+  }
 
   @Override
   public boolean killTask(TaskAttemptID arg0, boolean arg1) throws IOException,



Mime
View raw message