hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079178 - in /hadoop/mapreduce/branches/yahoo-merge: CHANGES.txt src/java/org/apache/hadoop/mapreduce/Job.java src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java src/webapps/job/jobdetailshistory.jsp
Date Tue, 08 Mar 2011 05:52:38 GMT
Author: omalley
Date: Tue Mar  8 05:52:38 2011
New Revision: 1079178

URL: http://svn.apache.org/viewvc?rev=1079178&view=rev
Log:
commit e3b491c7ffe6611550473522f80aab5a6aea619d
Author: Mahadev Konar <mahadev@yahoo-inc.com>
Date:   Mon Oct 25 18:02:59 2010 -0700

     Please make it clear why the job failed (Krishna Ramachandran via mahadev) from

Modified:
    hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
    hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp

Modified: hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt?rev=1079178&r1=1079177&r2=1079178&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt Tue Mar  8 05:52:38 2011
@@ -7,6 +7,7 @@ Trunk (unreleased changes)
   NEW FEATURES
 
   IMPROVEMENTS
+  Bug-2367384 Please make it clear why the job failed (Krishna Ramachandran via mahadev)
from http://bug.corp.yahoo.com/attachment.cgi?id=1169543 
 
   OPTIMIZATIONS
 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java?rev=1079178&r1=1079177&r2=1079178&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java Tue
Mar  8 05:52:38 2011
@@ -122,7 +122,7 @@ public class Job extends JobContextImpl 
   private JobStatus status;
   private long statustime;
   private Cluster cluster;
-  
+
   @Deprecated
   public Job() throws IOException {
     this(new Configuration());
@@ -360,8 +360,11 @@ public class Job extends JobContextImpl 
   @Override
   public String toString() {
     ensureState(JobState.RUNNING);
+    String reasonforFailure = " ";
     try {
       updateStatus();
+      if (status.getState().equals(JobStatus.State.FAILED))
+        reasonforFailure = getTaskFailureEventString();
     } catch (IOException e) {
     } catch (InterruptedException ie) {
     }
@@ -378,10 +381,34 @@ public class Job extends JobContextImpl 
     sb.append(status.getState()).append("\n");
     sb.append("history URL: ");
     sb.append(status.getHistoryFile()).append("\n");
-    sb.append("retired: ").append(status.isRetired());
+    sb.append("retired: ").append(status.isRetired()).append("\n");
+    sb.append("reason for failure: ").append(reasonforFailure);
     return sb.toString();
   }
-      
+
+  /**
+   * @return taskid which caused job failure
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  String getTaskFailureEventString() throws IOException,
+      InterruptedException {
+    int failCount = 1;
+    TaskCompletionEvent lastEvent = null;
+    for (TaskCompletionEvent event : cluster.getClient().getTaskCompletionEvents(
+        status.getJobID(), 0, 10)) {
+      if (event.getStatus().equals(TaskCompletionEvent.Status.FAILED)) {
+        failCount++;
+        lastEvent = event;
+      }
+    }
+    String[] taskAttemptID = lastEvent.getTaskAttemptId().toString().split("_", 2);
+    String taskID = taskAttemptID[1].substring(0, taskAttemptID[1].length()-2);
+    return (" task " + taskID + " failed " +
+      failCount + " times " + "For details check tasktracker at: " +
+      lastEvent.getTaskTrackerHttp());
+  }
+
   /**
    * Get the information of the current state of the tasks of a job.
    * 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1079178&r1=1079177&r2=1079178&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
Tue Mar  8 05:52:38 2011
@@ -262,6 +262,8 @@ public class JobHistoryParser {
     taskInfo.finishTime = event.getFinishTime();
     taskInfo.error = event.getError();
     taskInfo.failedDueToAttemptId = event.getFailedAttemptID();
+    info.errorInfo = "Task " + taskInfo.taskId +" failed " +
+    taskInfo.attemptsMap.size() + " times ";
   }
 
   private void handleTaskStartedEvent(TaskStartedEvent event) {
@@ -321,6 +323,7 @@ public class JobHistoryParser {
    * The class where job information is aggregated into after parsing
    */
   public static class JobInfo {
+    String errorInfo = "None";
     long submitTime;
     long finishTime;
     JobID jobid;
@@ -406,6 +409,7 @@ public class JobHistoryParser {
     public long getFinishedReduces() { return finishedReduces; }
     /** Get the job status */
     public String getJobStatus() { return jobStatus; }
+    public String getErrorInfo() { return errorInfo; }
     /** Get the counters for the job */
     public Counters getTotalCounters() { return totalCounters; }
     /** Get the map counters for the job */

Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp?rev=1079178&r1=1079177&r2=1079178&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp Tue Mar  8
05:52:38 2011
@@ -45,6 +45,7 @@
 <%! static SimpleDateFormat dateFormat = new SimpleDateFormat("d-MMM-yyyy HH:mm:ss") ;
%>
 <%
     String logFile = request.getParameter("logFile");
+    String reasonforFailure = " ";
     final Path jobFile = new Path(logFile);
     String jobid = JobHistory.getJobIDFromHistoryFilePath(jobFile).toString();
 
@@ -55,6 +56,8 @@
     if (job == null) {
       return;
     }
+    if (job.getJobStatus().equals("FAILED")) 
+      reasonforFailure = job.getErrorInfo();
 %>
 
 <html>
@@ -78,6 +81,7 @@
 <b>Launched At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLaunchTime(),
job.getSubmitTime()) %><br/>
 <b>Finished At: </b>  <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getFinishTime(), job.getLaunchTime()) %><br/>
 <b>Status: </b> <%= ((job.getJobStatus()) == null ? "Incomplete" :job.getJobStatus())
%><br/> 
+<b>ReasonForFailure: </b> <%=reasonforFailure %><br/>
 <%
     HistoryViewer.SummarizedJob sj = new HistoryViewer.SummarizedJob(job);
 %>



Mime
View raw message