hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r637723 [2/2] - in /hadoop/core/trunk: ./ conf/ docs/ src/docs/src/documentation/content/xdocs/ src/java/org/apache/hadoop/mapred/ src/webapps/history/ src/webapps/job/
Date Mon, 17 Mar 2008 05:19:57 GMT
Added: hadoop/core/trunk/src/webapps/job/analysejobhistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/analysejobhistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/analysejobhistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/analysejobhistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,249 @@
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.util.*"
+  import="java.text.SimpleDateFormat"
+  import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<jsp:include page="loadhistory.jsp">
+  <jsp:param name="jobid" value="<%=request.getParameter("jobid") %>"/>
+  <jsp:param name="logFile" value="<%=request.getParameter("logFile") %>"/>
+</jsp:include>
+<%!	private static SimpleDateFormat dateFormat 
+                              = new SimpleDateFormat("d/MM HH:mm:ss") ; 
+%>
+<html><body>
+<%
+  String jobid = request.getParameter("jobid");
+  String logFile = request.getParameter("logFile");
+  String numTasks = request.getParameter("numTasks");
+  int showTasks = 10 ; 
+  if (numTasks != null) {
+    showTasks = Integer.parseInt(numTasks);  
+  }
+  JobInfo job = (JobInfo)request.getSession().getAttribute("job");
+%>
+<h2>Hadoop Job <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=logFile
%>"><%=jobid %> </a></h2>
+<b>User : </b> <%=job.get(Keys.USER) %><br/> 
+<b>JobName : </b> <%=job.get(Keys.JOBNAME) %><br/> 
+<b>JobConf : </b> <%=job.get(Keys.JOBCONF) %><br/> 
+<b>Submitted At : </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.SUBMIT_TIME), 0 ) %><br/> 
+<b>Launched At : </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.LAUNCH_TIME), job.getLong(Keys.SUBMIT_TIME)) %><br/>
+<b>Finished At : </b>  <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME)) %><br/>
+<b>Status : </b> <%= ((job.get(Keys.JOB_STATUS) == null)?"Incomplete" :job.get(Keys.JOB_STATUS))
%><br/> 
+<hr/>
+<center>
+<%
+  if (!Values.SUCCESS.name().equals(job.get(Keys.JOB_STATUS))) {
+    out.print("<h3>No Analysis available as job did not finish</h3>");
+    return;
+  }
+  Map<String, JobHistory.Task> tasks = job.getAllTasks();
+  int finishedMaps = job.getInt(Keys.FINISHED_MAPS)  ;
+  int finishedReduces = job.getInt(Keys.FINISHED_REDUCES) ;
+  JobHistory.Task [] mapTasks = new JobHistory.Task[finishedMaps]; 
+  JobHistory.Task [] reduceTasks = new JobHistory.Task[finishedReduces]; 
+  int mapIndex = 0 , reduceIndex=0; 
+  long avgMapTime = 0;
+  long avgReduceTime = 0;
+  long avgShuffleTime = 0;
+
+  for (JobHistory.Task task : tasks.values()) {
+    Map<String, TaskAttempt> attempts = task.getTaskAttempts();
+    for (JobHistory.TaskAttempt attempt : attempts.values()) {
+      if (attempt.get(Keys.TASK_STATUS).equals(Values.SUCCESS.name())) {
+        long avgFinishTime = (attempt.getLong(Keys.FINISH_TIME) -
+      		                attempt.getLong(Keys.START_TIME));
+        if (Values.MAP.name().equals(task.get(Keys.TASK_TYPE))) {
+          mapTasks[mapIndex++] = attempt ; 
+          avgMapTime += avgFinishTime;
+        } else { 
+          reduceTasks[reduceIndex++] = attempt;
+          avgShuffleTime += (attempt.getLong(Keys.SHUFFLE_FINISHED) - 
+                             attempt.getLong(Keys.START_TIME));
+          avgReduceTime += (attempt.getLong(Keys.FINISH_TIME) -
+                            attempt.getLong(Keys.SHUFFLE_FINISHED));
+        }
+        break;
+      }
+    }
+  }
+	 
+  if (finishedMaps > 0) {
+    avgMapTime /= finishedMaps;
+  }
+  if (finishedReduces > 0) {
+    avgReduceTime /= finishedReduces;
+    avgShuffleTime /= finishedReduces;
+  }
+  Comparator<JobHistory.Task> cMap = new Comparator<JobHistory.Task>(){
+    public int compare(JobHistory.Task t1, JobHistory.Task t2){
+      long l1 = t1.getLong(Keys.FINISH_TIME) - t1.getLong(Keys.START_TIME); 
+      long l2 = t2.getLong(Keys.FINISH_TIME) - t2.getLong(Keys.START_TIME);
+      return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+    }
+  }; 
+  Comparator<JobHistory.Task> cShuffle = new Comparator<JobHistory.Task>(){
+    public int compare(JobHistory.Task t1, JobHistory.Task t2){
+      long l1 = t1.getLong(Keys.SHUFFLE_FINISHED) - 
+                t1.getLong(Keys.START_TIME); 
+      long l2 = t2.getLong(Keys.SHUFFLE_FINISHED) - 
+                t2.getLong(Keys.START_TIME); 
+      return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+    }
+  }; 
+  Arrays.sort(mapTasks, cMap);
+  JobHistory.Task minMap = mapTasks[mapTasks.length-1] ;
+%>
+
+<h3>Time taken by best performing Map task 
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>&taskid=<%=minMap.get(Keys.TASKID)%>">
+<%=minMap.get(Keys.TASKID) %></a> : <%=StringUtils.formatTimeDiff(minMap.getLong(Keys.FINISH_TIME),
minMap.getLong(Keys.START_TIME) ) %></h3>
+<h3>Average time taken by Map tasks: 
+<%=StringUtils.formatTimeDiff(avgMapTime, 0) %></h3>
+<h3>Worse performing map tasks</h3>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Time taken</td></tr>
+<%
+  for (int i=0;i<showTasks && i<mapTasks.length; i++) {
+%>
+    <tr>
+    <td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>&taskid=<%=mapTasks[i].get(Keys.TASKID)%>">
+        <%=mapTasks[i].get(Keys.TASKID) %></a></td>
+    <td><%=StringUtils.formatTimeDiff(mapTasks[i].getLong(Keys.FINISH_TIME), mapTasks[i].getLong(Keys.START_TIME))
%></td>
+    </tr>
+<%
+  }
+%>
+</table>
+<%  
+  Comparator<JobHistory.Task> cFinishMapRed = 
+    new Comparator<JobHistory.Task>() {
+    public int compare(JobHistory.Task t1, JobHistory.Task t2){
+      long l1 = t1.getLong(Keys.FINISH_TIME); 
+      long l2 = t2.getLong(Keys.FINISH_TIME);
+      return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+    }
+  };
+  Arrays.sort(mapTasks, cFinishMapRed);
+  JobHistory.Task lastMap = mapTasks[0] ;
+%>
+
+<h3>The last Map task 
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>
+&taskid=<%=lastMap.get(Keys.TASKID)%>"><%=lastMap.get(Keys.TASKID) %></a>

+finished at (relative to the Job launch time): 
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat, 
+                              lastMap.getLong(Keys.FINISH_TIME), 
+                              job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+<hr/>
+
+<%
+  if (reduceTasks.length <= 0) return;
+  Arrays.sort(reduceTasks, cShuffle); 
+  JobHistory.Task minShuffle = reduceTasks[reduceTasks.length-1] ;
+%>
+<h3>Time taken by best performing shuffle
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>
+&taskid=<%=minShuffle.get(Keys.TASKID)%>"><%=minShuffle.get(Keys.TASKID)%></a>
: 
+<%=StringUtils.formatTimeDiff(minShuffle.getLong(Keys.SHUFFLE_FINISHED), 
+                              minShuffle.getLong(Keys.START_TIME) ) %></h3>
+<h3>Average time taken by Shuffle: 
+<%=StringUtils.formatTimeDiff(avgShuffleTime, 0) %></h3>
+<h3>Worse performing Shuffle(s)</h3>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Time taken</td></tr>
+<%
+  for (int i=0;i<showTasks && i<reduceTasks.length; i++) {
+%>
+    <tr>
+    <td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=
+        <%=logFile%>&taskid=<%=reduceTasks[i].get(Keys.TASKID)%>">
+        <%=reduceTasks[i].get(Keys.TASKID) %></a></td>
+    <td><%=
+           StringUtils.formatTimeDiff(
+                       reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED),
+                       reduceTasks[i].getLong(Keys.START_TIME)) %>
+    </td>
+    </tr>
+<%
+  }
+%>
+</table>
+<%  
+  Comparator<JobHistory.Task> cFinishShuffle = 
+    new Comparator<JobHistory.Task>() {
+    public int compare(JobHistory.Task t1, JobHistory.Task t2){
+      long l1 = t1.getLong(Keys.SHUFFLE_FINISHED); 
+      long l2 = t2.getLong(Keys.SHUFFLE_FINISHED);
+      return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+    }
+  };
+  Arrays.sort(reduceTasks, cFinishShuffle);
+  JobHistory.Task lastShuffle = reduceTasks[0] ;
+%>
+
+<h3>The last Shuffle  
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>
+&taskid=<%=lastShuffle.get(Keys.TASKID)%>"><%=lastShuffle.get(Keys.TASKID)%>
+</a> finished at (relative to the Job launch time): 
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+                              lastShuffle.getLong(Keys.SHUFFLE_FINISHED), 
+                              job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+
+<%
+  Comparator<JobHistory.Task> cReduce = new Comparator<JobHistory.Task>(){
+    public int compare(JobHistory.Task t1, JobHistory.Task t2){
+      long l1 = t1.getLong(Keys.FINISH_TIME) - 
+                t1.getLong(Keys.SHUFFLE_FINISHED); 
+      long l2 = t2.getLong(Keys.FINISH_TIME) - 
+                t2.getLong(Keys.SHUFFLE_FINISHED);
+      return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+    }
+  }; 
+  Arrays.sort(reduceTasks, cReduce); 
+  JobHistory.Task minReduce = reduceTasks[reduceTasks.length-1] ;
+%>
+<hr/>
+<h3>Time taken by best performing Reduce task : 
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>&taskid=<%=minReduce.get(Keys.TASKID)%>">
+<%=minReduce.get(Keys.TASKID) %></a> : 
+<%=StringUtils.formatTimeDiff(minReduce.getLong(Keys.FINISH_TIME),
+    minReduce.getLong(Keys.SHUFFLE_FINISHED) ) %></h3>
+
+<h3>Average time taken by Reduce tasks: 
+<%=StringUtils.formatTimeDiff(avgReduceTime, 0) %></h3>
+<h3>Worse performing reduce tasks</h3>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Time taken</td></tr>
+<%
+  for (int i=0;i<showTasks && i<reduceTasks.length; i++) {
+%>
+    <tr>
+    <td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>&taskid=<%=reduceTasks[i].get(Keys.TASKID)%>">
+        <%=reduceTasks[i].get(Keys.TASKID) %></a></td>
+    <td><%=StringUtils.formatTimeDiff(
+             reduceTasks[i].getLong(Keys.FINISH_TIME), 
+             reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED)) %></td>
+    </tr>
+<%
+  }
+%>
+</table>
+<%  
+  Arrays.sort(reduceTasks, cFinishMapRed);
+  JobHistory.Task lastReduce = reduceTasks[0] ;
+%>
+
+<h3>The last Reduce task 
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>
+&taskid=<%=lastReduce.get(Keys.TASKID)%>"><%=lastReduce.get(Keys.TASKID)%>
+</a> finished at (relative to the Job launch time): 
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+                              lastReduce.getLong(Keys.FINISH_TIME), 
+                              job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+</center>
+</body></html>
\ No newline at end of file

Added: hadoop/core/trunk/src/webapps/job/jobconf_history.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobconf_history.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobconf_history.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/jobconf_history.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,56 @@
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.*"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.net.URL"
+  import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.fs.*"
+  import="org.apache.hadoop.util.*"
+%>
+
+
+<%
+  JobTracker tracker = (JobTracker) application.getAttribute("job.tracker");
+  String jobId = request.getParameter("jobid");
+  if (jobId == null) {
+    out.println("<h2>Missing 'jobid' for fetching job configuration!</h2>");
+ 	return;
+  }
+%>
+  
+<html>
+
+<title>Job Configuration: JobId - <%= jobId %></title>
+
+<body>
+<h2>Job Configuration: JobId - <%= jobId %></h2><br>
+
+<%
+  Path logDir = new Path(request.getParameter("jobLogDir"));
+  Path jobFilePath = new Path(logDir, 
+                       request.getParameter("jobUniqueString") + "_conf.xml");
+  FileSystem fs = (FileSystem)request.getSession().getAttribute("fs");
+  FSDataInputStream jobFile = null; 
+  try {
+    jobFile = fs.open(jobFilePath);
+    JobConf jobConf = new JobConf(jobFilePath);
+    XMLUtils.transform(
+        jobConf.getConfResourceAsInputStream("webapps/static/jobconf.xsl"),
+        jobFile, out);
+  } catch (Exception e) {
+    out.println("Failed to retreive job configuration for job '" + jobId + "!");
+    out.println(e);
+  } finally {
+    if (jobFile != null) {
+      try { 
+        jobFile.close(); 
+      } catch (IOException e) {}
+    }
+  } 
+%>
+
+<br>
+<%
+out.println(ServletUtil.htmlFooter());
+%>

Added: hadoop/core/trunk/src/webapps/job/jobdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobdetailshistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobdetailshistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/jobdetailshistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,192 @@
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="org.apache.hadoop.fs.*"
+  import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.util.*"
+  import="java.text.SimpleDateFormat"
+  import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<jsp:include page="loadhistory.jsp">
+  <jsp:param name="jobid" value="<%=request.getParameter("jobid") %>"/>
+  <jsp:param name="logFile" value="<%=request.getParameter("logFile") %>"/>
+</jsp:include>
+<%! static SimpleDateFormat dateFormat = new SimpleDateFormat("d-MMM-yyyy HH:mm:ss") ;
%>
+<%
+    String jobid = request.getParameter("jobid");
+    String logFile = request.getParameter("logFile");
+	
+    Path jobFile = new Path(logFile);
+    String[] jobDetails = jobFile.getName().split("_");
+    String jobUniqueString = jobDetails[0] + "_" +jobDetails[1] + "_" + jobid ;
+	
+    JobInfo job = (JobInfo)request.getSession().getAttribute("job");
+    FileSystem fs = (FileSystem)request.getSession().getAttribute("fs");
+%>
+<html><body>
+<h2>Hadoop Job <%=jobid %> on <a href="jobhistory.jsp">History Viewer</a></h2>
+
+<b>User: </b> <%=job.get(Keys.USER) %><br/> 
+<b>JobName: </b> <%=job.get(Keys.JOBNAME) %><br/> 
+<b>JobConf: </b> <a href="jobconf_history.jsp?jobid=<%=jobid%>&jobLogDir=<%=new
Path(logFile).getParent().toString()%>&jobUniqueString=<%=jobUniqueString%>">

+                 <%=job.get(Keys.JOBCONF) %></a><br/> 
+<b>Submitted At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.SUBMIT_TIME), 0 )  %><br/> 
+<b>Launched At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.LAUNCH_TIME),
job.getLong(Keys.SUBMIT_TIME)) %><br/>
+<b>Finished At: </b>  <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME)) %><br/>
+<b>Status: </b> <%= ((job.get(Keys.JOB_STATUS) == "")?"Incomplete" :job.get(Keys.JOB_STATUS))
%><br/> 
+<%
+    Map<String, JobHistory.Task> tasks = job.getAllTasks();
+    int totalMaps = 0 ; 
+    int totalReduces = 0; 
+    int numFailedMaps = 0; 
+    int numKilledMaps = 0;
+    int numFailedReduces = 0 ; 
+    int numKilledReduces = 0;
+	
+    long mapStarted = 0 ; 
+    long mapFinished = 0 ; 
+    long reduceStarted = 0 ; 
+    long reduceFinished = 0; 
+        
+    Map <String,String> allHosts = new TreeMap<String,String>();
+    for (JobHistory.Task task : tasks.values()) {
+      Map<String, TaskAttempt> attempts = task.getTaskAttempts();
+      allHosts.put(task.get(Keys.HOSTNAME), "");
+      for (TaskAttempt attempt : attempts.values()) {
+        long startTime = attempt.getLong(Keys.START_TIME) ; 
+        long finishTime = attempt.getLong(Keys.FINISH_TIME) ; 
+        if (Values.MAP.name().equals(task.get(Keys.TASK_TYPE))){
+          if (mapStarted==0 || mapStarted > startTime ) {
+            mapStarted = startTime; 
+          }
+          if (mapFinished < finishTime ) {
+            mapFinished = finishTime ; 
+          }
+          totalMaps++; 
+          if (Values.FAILED.name().equals(attempt.get(Keys.TASK_STATUS))) {
+            numFailedMaps++; 
+          } else if (Values.KILLED.name().equals(attempt.get(Keys.TASK_STATUS))) {
+            numKilledMaps++;
+          }
+        } else {
+          if (reduceStarted==0||reduceStarted > startTime) {
+            reduceStarted = startTime ; 
+          }
+          if (reduceFinished < finishTime) {
+            reduceFinished = finishTime; 
+          }
+          totalReduces++; 
+          if (Values.FAILED.name().equals(attempt.get(Keys.TASK_STATUS))) {
+            numFailedReduces++;
+          } else if (Values.KILLED.name().equals(attempt.get(Keys.TASK_STATUS))) {
+            numKilledReduces++;
+          }
+        }
+      }
+    }
+%>
+<b><a href="analysejobhistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile
%>">Analyse This Job</a></b> 
+<hr/>
+<center>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr>
+<td>Kind</td><td>Total Tasks(successful+failed+killed)</td><td>Successful
tasks</td><td>Failed tasks</td><td>Killed tasks</td><td>Start
Time</td><td>Finish Time</td>
+</tr>
+<tr>
+<td>Map</td>
+    <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile
%>&taskType=<%=Values.MAP.name() %>&status=all">
+        <%=totalMaps %></a></td>
+    <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile
%>&taskType=<%=Values.MAP.name() %>&status=<%=Values.SUCCESS %>">
+        <%=job.getInt(Keys.FINISHED_MAPS) %></a></td>
+    <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile
%>&taskType=<%=Values.MAP.name() %>&status=<%=Values.FAILED %>">
+        <%=numFailedMaps %></a></td>
+    <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile
%>&taskType=<%=Values.MAP.name() %>&status=<%=Values.KILLED %>">
+        <%=numKilledMaps %></a></td>
+    <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, mapStarted, 0) %></td>
+    <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, mapFinished, mapStarted)
%></td>
+</tr>
+<tr>
+<td>Reduce</td>
+    <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile
%>&taskType=<%=Values.REDUCE.name() %>&status=all">
+        <%=totalReduces%></a></td>
+    <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile
%>&taskType=<%=Values.REDUCE.name() %>&status=<%=Values.SUCCESS %>">
+        <%=job.getInt(Keys.FINISHED_REDUCES)%></a></td>
+    <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile
%>&taskType=<%=Values.REDUCE.name() %>&status=<%=Values.FAILED %>">
+        <%=numFailedReduces%></a></td>
+    <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile
%>&taskType=<%=Values.REDUCE.name() %>&status=<%=Values.KILLED %>">
+        <%=numKilledReduces%></a></td>  
+    <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, reduceStarted, 0) %></td>
+    <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, reduceFinished, reduceStarted)
%></td>
+</tr>
+</table>
+
+<br/>
+ <%
+    DefaultJobHistoryParser.FailedOnNodesFilter filter = 
+                 new DefaultJobHistoryParser.FailedOnNodesFilter();
+    JobHistory.parseHistoryFromFS(logFile, filter, fs); 
+    Map<String, Set<String>> badNodes = filter.getValues(); 
+    if (badNodes.size() > 0) {
+ %>
+<h3>Failed tasks attempts by nodes </h3>
+<table border="1">
+<tr><td>Hostname</td><td>Failed Tasks</td></tr>
+ <%	  
+      for (Map.Entry<String, Set<String>> entry : badNodes.entrySet()) {
+        String node = entry.getKey();
+        Set<String> failedTasks = entry.getValue();
+%>
+        <tr>
+        <td><%=node %></td>
+        <td>
+<%
+        for (String t : failedTasks) {
+%>
+          <a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile
%>&taskid=<%=t %>"><%=t %></a>,&nbsp;
+<%		  
+        }
+%>	
+        </td>
+        </tr>
+<%	  
+      }
+	}
+ %>
+</table>
+<br/>
+ <%
+    DefaultJobHistoryParser.KilledOnNodesFilter killedFilter =
+                 new DefaultJobHistoryParser.KilledOnNodesFilter();
+    JobHistory.parseHistoryFromFS(logFile, filter, fs); 
+    badNodes = killedFilter.getValues(); 
+    if (badNodes.size() > 0) {
+ %>
+<h3>Killed tasks attempts by nodes </h3>
+<table border="1">
+<tr><td>Hostname</td><td>Killed Tasks</td></tr>
+ <%	  
+      for (Map.Entry<String, Set<String>> entry : badNodes.entrySet()) {
+        String node = entry.getKey();
+        Set<String> killedTasks = entry.getValue();
+%>
+        <tr>
+        <td><%=node %></td>
+        <td>
+<%
+        for (String t : killedTasks) {
+%>
+          <a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile
%>&taskid=<%=t %>"><%=t %></a>,&nbsp;
+<%		  
+        }
+%>	
+        </td>
+        </tr>
+<%	  
+      }
+    }
+%>
+</table>
+</center>
+</body></html>

Added: hadoop/core/trunk/src/webapps/job/jobhistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobhistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobhistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/jobhistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,105 @@
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="java.io.*"
+  import="java.util.*"
+  import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.util.*"
+  import="org.apache.hadoop.fs.*"
+  import="javax.servlet.jsp.*"
+  import="java.text.SimpleDateFormat"
+  import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<%!	
+  private static SimpleDateFormat dateFormat = 
+                                    new SimpleDateFormat("d/MM HH:mm:ss");
+%>
+<html>
+<head>
+<title>Hadoop Map/Reduce Administration</title>
+<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
+</head>
+<body>
+<h1>Hadoop Map/Reduce History Viewer</h1>
+<hr>
+<h2>Available History </h2>
+<%
+    PathFilter jobLogFileFilter = new PathFilter() {
+      public boolean accept(Path path) {
+        return !(path.getName().endsWith(".xml"));
+      }
+    };
+    
+    FileSystem fs = (FileSystem) application.getAttribute("fileSys");
+    String historyLogDir = (String) application.getAttribute("historyLogDir");
+    if (fs == null) {
+      out.println("Null file system. May be namenode is in safemode!");
+      return;
+    }
+    Path[] jobFiles = fs.listPaths(new Path(historyLogDir), jobLogFileFilter);
+    if (null == jobFiles )  {
+      out.println("NULL files !!!"); 
+      return ; 
+    }
+
+    // sort the files on creation time.
+    Arrays.sort(jobFiles, new Comparator<Path>() {
+      public int compare(Path p1, Path p2) {
+        String[] split1 = p1.getName().split("_");
+        String[] split2 = p2.getName().split("_");
+        
+        // compare job tracker start time
+        int res = new Date(Long.parseLong(split1[1])).compareTo(
+                             new Date(Long.parseLong(split2[1])));
+        if (res == 0) {
+          res = new Date(Long.parseLong(split1[3])).compareTo(
+                           new Date(Long.parseLong(split2[3])));
+        }
+        if (res == 0) {
+          Long l1 = Long.parseLong(split1[4]);
+          res = l1.compareTo(Long.parseLong(split2[4]));
+        }
+        return res;
+      }
+    });
+
+    out.print("<table align=center border=2 cellpadding=\"5\" cellspacing=\"2\">");
+    out.print("<tr><td align=\"center\" colspan=\"9\"><b>Available Jobs
</b></td></tr>\n");
+    out.print("<tr>");
+    out.print("<td>Job tracker Host Name</td>" +
+              "<td>Job tracker Start time</td>" +
+              "<td>Job Id</td><td>Name</td><td>User</td>")
; 
+    out.print("</tr>"); 
+    for (Path jobFile: jobFiles) {
+      String[] jobDetails = jobFile.getName().split("_");
+      String trackerHostName = jobDetails[0];
+      String trackerStartTime = jobDetails[1];
+      String jobId = jobDetails[2] + "_" +jobDetails[3] + "_" + jobDetails[4] ;
+      String user = jobDetails[5];
+      String jobName = jobDetails[6];
+%>
+<center>
+<%	
+      printJob(trackerHostName, trackerStartTime, jobId,
+               jobName, user, jobFile.toString(), out) ; 
+%>
+</center> 
+<%
+    } // end while trackers 
+%>
+<%!
+    private void printJob(String trackerHostName, String trackerid,
+                          String jobId, String jobName,
+                          String user, String logFile, JspWriter out)
+    throws IOException {
+      out.print("<tr>"); 
+      out.print("<td>" + trackerHostName + "</td>"); 
+      out.print("<td>" + new Date(Long.parseLong(trackerid)) + "</td>"); 
+      out.print("<td>" + "<a href=\"jobdetailshistory.jsp?jobid=" + jobId + 
+                "&logFile=" + logFile + "\">" + jobId + "</a></td>");

+      out.print("<td>" + jobName + "</td>"); 
+      out.print("<td>" + user + "</td>"); 
+      out.print("</tr>");
+    }
+%> 
+</body></html>

Added: hadoop/core/trunk/src/webapps/job/jobtaskshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobtaskshistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobtaskshistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/jobtaskshistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,68 @@
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.util.*"
+  import="java.text.SimpleDateFormat"
+  import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<jsp:include page="loadhistory.jsp">
+	<jsp:param name="jobid" value="<%=request.getParameter("jobid") %>"/>
+	<jsp:param name="logFile" value="<%=request.getParameter("logFile") %>"/>
+</jsp:include>
+<%!	
+  private static SimpleDateFormat dateFormat =
+                                    new SimpleDateFormat("d/MM HH:mm:ss") ; 
+%>
+
+<%	
+  String jobid = request.getParameter("jobid");
+  String logFile = request.getParameter("logFile");
+  String taskStatus = request.getParameter("status"); 
+  String taskType = request.getParameter("taskType"); 
+  
+  JobHistory.JobInfo job = (JobHistory.JobInfo)request.
+                            getSession().getAttribute("job");
+  Map<String, JobHistory.Task> tasks = job.getAllTasks(); 
+%>
+<html>
+<body>
+<h2><%=taskStatus%> <%=taskType %> task list for <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=logFile
%>"><%=jobid %> </a></h2>
+<center>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Start Time</td><td>Finish Time<br/></td><td>Error</td></tr>
+<%
+  for (JobHistory.Task task : tasks.values()) {
+    if (taskType.equals(task.get(Keys.TASK_TYPE))){
+      Map <String, TaskAttempt> taskAttempts = task.getTaskAttempts();
+      for (JobHistory.TaskAttempt taskAttempt : taskAttempts.values()) {
+        if (taskStatus.equals(taskAttempt.get(Keys.TASK_STATUS)) || 
+          taskStatus.equals("all")){
+          printTask(jobid, logFile, taskAttempt, out); 
+        }
+      }
+    }
+  }
+%>
+</table>
+<%!
+  private void printTask(String jobid, String trackerId,
+    JobHistory.TaskAttempt attempt, JspWriter out) throws IOException{
+    out.print("<tr>"); 
+    out.print("<td>" + "<a href=\"taskdetailshistory.jsp?jobid=" + jobid + 
+          "&logFile="+ trackerId +"&taskid="+attempt.get(Keys.TASKID)+"\">" +
+          attempt.get(Keys.TASKID) + "</a></td>");
+    out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, 
+          attempt.getLong(Keys.START_TIME), 0 ) + "</td>");
+    out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, 
+          attempt.getLong(Keys.FINISH_TIME),
+          attempt.getLong(Keys.START_TIME) ) + "</td>");
+    out.print("<td>" + attempt.get(Keys.ERROR) + "</td>");
+    out.print("</tr>"); 
+  }
+%>
+</center>
+</body>
+</html>

Modified: hadoop/core/trunk/src/webapps/job/jobtracker.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobtracker.jsp?rev=637723&r1=637722&r2=637723&view=diff
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobtracker.jsp (original)
+++ hadoop/core/trunk/src/webapps/job/jobtracker.jsp Sun Mar 16 22:19:52 2008
@@ -134,7 +134,7 @@
 <hr>
 
 <h2>Local logs</h2>
-<a href="logs/">Log</a> directory, <a href="http://<%=tracker.getHistoryAddress()%>">
+<a href="logs/">Log</a> directory, <a href="jobhistory.jsp">
 Job Tracker History</a>
 
 <%

Added: hadoop/core/trunk/src/webapps/job/loadhistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/loadhistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/loadhistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/loadhistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,36 @@
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="java.io.*"
+  import="java.util.*"
+  import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.fs.*"
+  import="org.apache.hadoop.util.*"
+  import="javax.servlet.jsp.*"
+  import="java.text.SimpleDateFormat"  
+  import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<%
+    PathFilter jobLogFileFilter = new PathFilter() {
+      public boolean accept(Path path) {
+        return !(path.getName().endsWith(".xml"));
+      }
+    };
+    
+    FileSystem fs = (FileSystem) application.getAttribute("fileSys");
+    String jobId =  (String)request.getParameter("jobid");
+    JobHistory.JobInfo job = (JobHistory.JobInfo)
+                               request.getSession().getAttribute("job");
+    if (null != job && (!jobId.equals(job.get(Keys.JOBID)))) {
+      // remove jobInfo from session, keep only one job in session at a time
+      request.getSession().removeAttribute("job"); 
+      job = null ; 
+    }
+	
+    if (null == job) {
+      String jobLogFile = (String)request.getParameter("logFile");
+      job = new JobHistory.JobInfo(jobId); 
+      DefaultJobHistoryParser.parseJobTasks(jobLogFile, job, fs) ; 
+      request.getSession().setAttribute("job", job);
+      request.getSession().setAttribute("fs", fs);
+    }
+%>

Added: hadoop/core/trunk/src/webapps/job/taskdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/taskdetailshistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/taskdetailshistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/taskdetailshistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,74 @@
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.util.*"
+  import="java.text.SimpleDateFormat"
+  import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<jsp:include page="loadhistory.jsp">
+  <jsp:param name="jobid" value="<%=request.getParameter("jobid") %>"/>
+  <jsp:param name="jobTrackerId" value="<%=request.getParameter("jobTrackerId") %>"/>
+</jsp:include>
+<%!	private static SimpleDateFormat dateFormat = new SimpleDateFormat("d/MM HH:mm:ss")
; %>
+
+<%	
+  String jobid = request.getParameter("jobid");
+  String logFile = request.getParameter("logFile");
+  String taskid = request.getParameter("taskid"); 
+  JobHistory.JobInfo job = (JobHistory.JobInfo)
+                              request.getSession().getAttribute("job");
+  JobHistory.Task task = job.getAllTasks().get(taskid); 
+%>
+<html>
+<body>
+<h2><%=taskid %> attempts for <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=logFile
%>"> <%=jobid %> </a></h2>
+<center>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Start Time</td>
+<%	
+  if (Values.REDUCE.name().equals(task.get(Keys.TASK_TYPE))) {
+%>
+    <td>Shuffle Finished</td><td>Sort Finished</td>
+<%
+  }
+%>
+<td>Finish Time</td><td>Host</td><td>Error</td></tr>
+<%
+  for (JobHistory.TaskAttempt attempt : task.getTaskAttempts().values()) {
+    printTaskAttempt(attempt, task.get(Keys.TASK_TYPE), out); 
+  }
+%>
+</table>
+<%!
+  private void printTaskAttempt(JobHistory.TaskAttempt taskAttempt,
+                                String type, JspWriter out) 
+  throws IOException {
+    out.print("<tr>"); 
+    out.print("<td>" + taskAttempt.get(Keys.TASK_ATTEMPT_ID) + "</td>");
+    out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat,
+              taskAttempt.getLong(Keys.START_TIME), 0 ) + "</td>"); 
+    if (Values.REDUCE.name().equals(type)) {
+      JobHistory.ReduceAttempt reduceAttempt = 
+            (JobHistory.ReduceAttempt)taskAttempt; 
+      out.print("<td>" + 
+                StringUtils.getFormattedTimeWithDiff(dateFormat, 
+                reduceAttempt.getLong(Keys.SHUFFLE_FINISHED), 
+                reduceAttempt.getLong(Keys.START_TIME)) + "</td>"); 
+      out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, 
+                reduceAttempt.getLong(Keys.SORT_FINISHED), 
+                reduceAttempt.getLong(Keys.SHUFFLE_FINISHED)) + "</td>"); 
+    }
+    out.print("<td>"+ StringUtils.getFormattedTimeWithDiff(dateFormat,
+              taskAttempt.getLong(Keys.FINISH_TIME), 
+              taskAttempt.getLong(Keys.START_TIME) ) + "</td>"); 
+    out.print("<td>" + taskAttempt.get(Keys.HOSTNAME) + "</td>");
+    out.print("<td>" + taskAttempt.get(Keys.ERROR) + "</td>");
+    out.print("</tr>"); 
+  }
+%>
+</center>
+</body>
+</html>



Mime
View raw message