hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject svn commit: r885145 [34/34] - in /hadoop/mapreduce/branches/MAPREDUCE-233: ./ .eclipse.templates/ .eclipse.templates/.launches/ conf/ ivy/ lib/ src/benchmarks/gridmix/ src/benchmarks/gridmix/pipesort/ src/benchmarks/gridmix2/ src/benchmarks/gridmix2/sr...
Date Sat, 28 Nov 2009 20:26:22 GMT
Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/jobtaskshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/jobtaskshistory.jsp?rev=885145&r1=885144&r2=885145&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/jobtaskshistory.jsp (original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/jobtaskshistory.jsp Sat Nov 28
20:26:01 2009
@@ -22,11 +22,13 @@
   import="javax.servlet.http.*"
   import="java.io.*"
   import="java.util.*"
+  import="org.apache.hadoop.mapreduce.TaskAttemptID"
+  import="org.apache.hadoop.mapreduce.TaskID"
   import="org.apache.hadoop.mapred.*"
   import="org.apache.hadoop.fs.*"
   import="org.apache.hadoop.util.*"
   import="java.text.SimpleDateFormat"
-  import="org.apache.hadoop.mapred.JobHistory.*"
+  import="org.apache.hadoop.mapreduce.jobhistory.*"
 %>
 
 <%!	
@@ -39,28 +41,27 @@
 <%	
   String jobid = request.getParameter("jobid");
   String logFile = request.getParameter("logFile");
-  String encodedLogFileName = JobHistory.JobInfo.encodeJobHistoryFilePath(logFile);
   String taskStatus = request.getParameter("status"); 
   String taskType = request.getParameter("taskType"); 
   
   FileSystem fs = (FileSystem) application.getAttribute("fileSys");
-  JobInfo job = JSPUtil.getJobInfo(request, fs);
-  Map<String, JobHistory.Task> tasks = job.getAllTasks(); 
+  JobHistoryParser.JobInfo job = JSPUtil.getJobInfo(request, fs);
+  Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); 
 %>
 <html>
 <body>
-<h2><%=taskStatus%> <%=taskType %> task list for <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=encodedLogFileName%>"><%=jobid
%> </a></h2>
+<h2><%=taskStatus%> <%=taskType %> task list for <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=logFile%>"><%=jobid
%> </a></h2>
 <center>
 <table border="2" cellpadding="5" cellspacing="2">
 <tr><td>Task Id</td><td>Start Time</td><td>Finish Time<br/></td><td>Error</td></tr>
 <%
-  for (JobHistory.Task task : tasks.values()) {
-    if (taskType.equals(task.get(Keys.TASK_TYPE))){
-      Map <String, TaskAttempt> taskAttempts = task.getTaskAttempts();
-      for (JobHistory.TaskAttempt taskAttempt : taskAttempts.values()) {
-        if (taskStatus.equals(taskAttempt.get(Keys.TASK_STATUS)) || 
-          taskStatus.equals("all")){
-          printTask(jobid, encodedLogFileName, taskAttempt, out); 
+  for (JobHistoryParser.TaskInfo task : tasks.values()) {
+    if (taskType.equalsIgnoreCase(task.getTaskType().toString())) {
+      Map <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> taskAttempts = task.getAllTaskAttempts();
+      for (JobHistoryParser.TaskAttemptInfo taskAttempt : taskAttempts.values()) {
+        if (taskStatus.equals(taskAttempt.getTaskStatus()) || 
+          taskStatus.equalsIgnoreCase("all")){
+          printTask(jobid, logFile, taskAttempt, out); 
         }
       }
     }
@@ -69,17 +70,17 @@
 </table>
 <%!
   private void printTask(String jobid, String logFile,
-    JobHistory.TaskAttempt attempt, JspWriter out) throws IOException{
+    JobHistoryParser.TaskAttemptInfo attempt, JspWriter out) throws IOException{
     out.print("<tr>"); 
     out.print("<td>" + "<a href=\"taskdetailshistory.jsp?jobid=" + jobid + 
-          "&logFile="+ logFile +"&taskid="+attempt.get(Keys.TASKID)+"\">" +
-          attempt.get(Keys.TASKID) + "</a></td>");
+          "&logFile="+ logFile +"&taskid="+attempt.getAttemptId().getTaskID().toString()
+"\">" +
+          attempt.getAttemptId().getTaskID() + "</a></td>");
     out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, 
-          attempt.getLong(Keys.START_TIME), 0 ) + "</td>");
+          attempt.getStartTime(), 0 ) + "</td>");
     out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, 
-          attempt.getLong(Keys.FINISH_TIME),
-          attempt.getLong(Keys.START_TIME) ) + "</td>");
-    out.print("<td>" + attempt.get(Keys.ERROR) + "</td>");
+          attempt.getFinishTime(),
+          attempt.getStartTime() ) + "</td>");
+    out.print("<td>" + attempt.getError() + "</td>");
     out.print("</tr>"); 
   }
 %>

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/jobtracker.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/jobtracker.jsp?rev=885145&r1=885144&r2=885145&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/jobtracker.jsp (original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/jobtracker.jsp Sat Nov 28 20:26:01
2009
@@ -25,6 +25,7 @@
   import="java.util.*"
   import="java.text.DecimalFormat"
   import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.mapreduce.*"
   import="org.apache.hadoop.util.*"
 %>
 <%!	private static final long serialVersionUID = 1L;
@@ -32,45 +33,56 @@
 <%
   JobTracker tracker = (JobTracker) application.getAttribute("job.tracker");
   ClusterStatus status = tracker.getClusterStatus();
+  ClusterMetrics metrics = tracker.getClusterMetrics();
   String trackerName = 
            StringUtils.simpleHostname(tracker.getJobTrackerMachine());
-  JobQueueInfo[] queues = tracker.getQueues();
-  Vector<JobInProgress> runningJobs = tracker.runningJobs();
-  Vector<JobInProgress> completedJobs = tracker.completedJobs();
-  Vector<JobInProgress> failedJobs = tracker.failedJobs();
+  JobQueueInfo[] queues = tracker.getJobQueues();
+  List<JobInProgress> runningJobs = tracker.getRunningJobs();
+  List<JobInProgress> completedJobs = tracker.getCompletedJobs();
+  List<JobInProgress> failedJobs = tracker.getFailedJobs();
 %>
 <%!
   private static DecimalFormat percentFormat = new DecimalFormat("##0.00");
   
-  public void generateSummaryTable(JspWriter out, ClusterStatus status,
+  public void generateSummaryTable(JspWriter out, ClusterMetrics metrics,
                                    JobTracker tracker) throws IOException {
-    String tasksPerNode = status.getTaskTrackers() > 0 ?
-      percentFormat.format(((double)(status.getMaxMapTasks() +
-                      status.getMaxReduceTasks())) / status.getTaskTrackers()):
+    String tasksPerNode = metrics.getTaskTrackerCount() > 0 ?
+      percentFormat.format(((double)(metrics.getMapSlotCapacity() +
+      metrics.getReduceSlotCapacity())) / metrics.getTaskTrackerCount()):
       "-";
     out.print("<table border=\"1\" cellpadding=\"5\" cellspacing=\"0\">\n"+
-              "<tr><th>Maps</th><th>Reduces</th>" + 
+              "<tr><th>Queues</th>" +
+              "<th>Running Map Tasks</th><th>Running Reduce Tasks</th>"
+ 
               "<th>Total Submissions</th>" +
-              "<th>Nodes</th><th>Map Task Capacity</th>" +
-              "<th>Reduce Task Capacity</th><th>Avg. Tasks/Node</th>"
+ 
+              "<th>Nodes</th>" +
+              "<th>Occupied Map Slots</th><th>Occupied Reduce Slots</th>"
+ 
+              "<th>Reserved Map Slots</th><th>Reserved Reduce Slots</th>"
+ 
+              "<th>Map Slot Capacity</th>" +
+              "<th>Reduce Slot Capacity</th><th>Avg. Slots/Node</th>"
+ 
               "<th>Blacklisted Nodes</th>" +
               "<th>Excluded Nodes</th></tr>\n");
-    out.print("<tr><td>" + status.getMapTasks() + "</td><td>" +
-              status.getReduceTasks() + "</td><td>" + 
-              tracker.getTotalSubmissions() +
+    out.print("<tr><td><a href=\"queueinfo.jsp\">" +
+              tracker.getRootQueues().length + "</a></td><td>" + 
+              metrics.getRunningMaps() + "</td><td>" +
+              metrics.getRunningReduces() + "</td><td>" + 
+              metrics.getTotalJobSubmissions() +
               "</td><td><a href=\"machines.jsp?type=active\">" +
-              status.getTaskTrackers() +
-              "</a></td><td>" + status.getMaxMapTasks() +
-              "</td><td>" + status.getMaxReduceTasks() +
+              metrics.getTaskTrackerCount() + "</a></td><td>" + 
+              metrics.getOccupiedMapSlots() + "</td><td>" +
+              metrics.getOccupiedReduceSlots() + "</td><td>" + 
+              metrics.getReservedMapSlots() + "</td><td>" +
+              metrics.getReservedReduceSlots() + "</td><td>" + 
+              + metrics.getMapSlotCapacity() +
+              "</td><td>" + metrics.getReduceSlotCapacity() +
               "</td><td>" + tasksPerNode +
               "</td><td><a href=\"machines.jsp?type=blacklisted\">" +
-              status.getBlacklistedTrackers() + "</a>" +
+              metrics.getBlackListedTaskTrackerCount() + "</a>" +
               "</td><td><a href=\"machines.jsp?type=excluded\">" +
-              status.getNumExcludedNodes() + "</a>" +
+              metrics.getDecommissionedTaskTrackerCount() + "</a>" +
               "</td></tr></table>\n");
 
     out.print("<br>");
-    if (tracker.hasRestarted()) {
+    if (tracker.recoveryManager.shouldRecover()) {
       out.print("<span class=\"small\"><i>");
       if (tracker.hasRecovered()) {
         out.print("The JobTracker got restarted and recovered back in " );
@@ -117,40 +129,9 @@
 <hr>
 <h2>Cluster Summary (Heap Size is <%= StringUtils.byteDesc(status.getUsedMemory())
%>/<%= StringUtils.byteDesc(status.getMaxMemory()) %>)</h2>
 <% 
- generateSummaryTable(out, status, tracker); 
+ generateSummaryTable(out, metrics, tracker); 
 %>
 <hr>
-<h2 id="scheduling_info">Scheduling Information</h2>
-<table border="2" cellpadding="5" cellspacing="2">
-<thead style="font-weight: bold">
-<tr>
-<td> Queue Name </td>
-<td> State </td>
-<td> Scheduling Information</td>
-</tr>
-</thead>
-<tbody>
-<%
-for(JobQueueInfo queue: queues) {
-  String queueName = queue.getQueueName();
-  String state = queue.getQueueState();
-  String schedulingInformation = queue.getSchedulingInfo();
-  if(schedulingInformation == null || schedulingInformation.trim().equals("")) {
-    schedulingInformation = "NA";
-  }
-%>
-<tr>
-<td><a href="jobqueue_details.jsp?queueName=<%=queueName%>"><%=queueName%></a></td>
-<td><%=state%></td>
-<td><%=schedulingInformation.replaceAll("\n","<br/>") %>
-</td>
-</tr>
-<%
-}
-%>
-</tbody>
-</table>
-<hr/>
 <b>Filter (Jobid, Priority, User, Name)</b> <input type="text" id="filter"
onkeyup="applyfilter()"> <br>
 <span class="small">Example: 'user:smith 3200' will filter by 'smith' only in the user
field and '3200' in all fields</span>
 <hr>

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/taskdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/taskdetailshistory.jsp?rev=885145&r1=885144&r2=885145&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/taskdetailshistory.jsp (original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/taskdetailshistory.jsp Sat Nov
28 20:26:01 2009
@@ -26,7 +26,11 @@
   import="org.apache.hadoop.fs.*"
   import="org.apache.hadoop.util.*"
   import="java.text.SimpleDateFormat"
-  import="org.apache.hadoop.mapred.JobHistory.*"
+  import="org.apache.hadoop.mapreduce.TaskType"
+  import="org.apache.hadoop.mapreduce.Counters"
+  import="org.apache.hadoop.mapreduce.TaskID"
+  import="org.apache.hadoop.mapreduce.TaskAttemptID"
+  import="org.apache.hadoop.mapreduce.jobhistory.*"
 %>
 
 <%!	private static SimpleDateFormat dateFormat = new SimpleDateFormat("d/MM HH:mm:ss")
; %>
@@ -36,41 +40,42 @@
 <%	
   String jobid = request.getParameter("jobid");
   String logFile = request.getParameter("logFile");
-  String encodedLogFileName = JobHistory.JobInfo.encodeJobHistoryFilePath(logFile);
   String taskid = request.getParameter("taskid"); 
   FileSystem fs = (FileSystem) application.getAttribute("fileSys");
-  JobInfo job = JSPUtil.getJobInfo(request, fs);
-  JobHistory.Task task = job.getAllTasks().get(taskid); 
-  String type = task.get(Keys.TASK_TYPE);
+  JobHistoryParser.JobInfo job = JSPUtil.getJobInfo(request, fs);
+  JobHistoryParser.TaskInfo task = job.getAllTasks().get(TaskID.forName(taskid)); 
+  TaskType type = task.getTaskType();
 %>
 <html>
 <body>
-<h2><%=taskid %> attempts for <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=encodedLogFileName%>">
<%=jobid %> </a></h2>
+<h2><%=taskid %> attempts for <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=logFile%>">
<%=jobid %> </a></h2>
 <center>
 <table border="2" cellpadding="5" cellspacing="2">
 <tr><td>Task Id</td><td>Start Time</td>
 <%	
-  if (Values.REDUCE.name().equals(type)) {
+  if (TaskType.REDUCE.equals(type)) {
 %>
     <td>Shuffle Finished</td><td>Sort Finished</td>
 <%
   }
 %>
-<td>Finish Time</td><td>Host</td><td>Error</td><td>Task
Logs</td></tr>
+<td>Finish Time</td><td>Host</td><td>Error</td><td>Task
Logs</td>
+<td>Counters</td></tr>
+
 <%
-  for (JobHistory.TaskAttempt attempt : task.getTaskAttempts().values()) {
-    printTaskAttempt(attempt, type, out);
+  for (JobHistoryParser.TaskAttemptInfo attempt : task.getAllTaskAttempts().values()) {
+    printTaskAttempt(attempt, type, out, logFile);
   }
 %>
 </table>
 </center>
 <%	
-  if (Values.MAP.name().equals(type)) {
+  if (TaskType.MAP.equals(type)) {
 %>
 <h3>Input Split Locations</h3>
 <table border="2" cellpadding="5" cellspacing="2">
 <%
-    for (String split : StringUtils.split(task.get(Keys.SPLITS)))
+    for (String split : StringUtils.split(task.getSplitLocations()))
     {
       out.println("<tr><td>" + split + "</td></tr>");
     }
@@ -80,33 +85,31 @@
   }
 %>
 <%!
-  private void printTaskAttempt(JobHistory.TaskAttempt taskAttempt,
-                                String type, JspWriter out) 
+  private void printTaskAttempt(JobHistoryParser.TaskAttemptInfo taskAttempt,
+                                TaskType type, JspWriter out, String logFile) 
   throws IOException {
     out.print("<tr>"); 
-    out.print("<td>" + taskAttempt.get(Keys.TASK_ATTEMPT_ID) + "</td>");
+    out.print("<td>" + taskAttempt.getAttemptId() + "</td>");
     out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat,
-              taskAttempt.getLong(Keys.START_TIME), 0 ) + "</td>"); 
-    if (Values.REDUCE.name().equals(type)) {
-      JobHistory.ReduceAttempt reduceAttempt = 
-            (JobHistory.ReduceAttempt)taskAttempt; 
+              taskAttempt.getStartTime(), 0 ) + "</td>"); 
+    if (TaskType.REDUCE.equals(type)) {
       out.print("<td>" + 
                 StringUtils.getFormattedTimeWithDiff(dateFormat, 
-                reduceAttempt.getLong(Keys.SHUFFLE_FINISHED), 
-                reduceAttempt.getLong(Keys.START_TIME)) + "</td>"); 
+                taskAttempt.getShuffleFinishTime(),
+                taskAttempt.getStartTime()) + "</td>"); 
       out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, 
-                reduceAttempt.getLong(Keys.SORT_FINISHED), 
-                reduceAttempt.getLong(Keys.SHUFFLE_FINISHED)) + "</td>"); 
+                taskAttempt.getSortFinishTime(),
+                taskAttempt.getShuffleFinishTime()) + "</td>"); 
     }
     out.print("<td>"+ StringUtils.getFormattedTimeWithDiff(dateFormat,
-              taskAttempt.getLong(Keys.FINISH_TIME), 
-              taskAttempt.getLong(Keys.START_TIME) ) + "</td>"); 
-    out.print("<td>" + taskAttempt.get(Keys.HOSTNAME) + "</td>");
-    out.print("<td>" + taskAttempt.get(Keys.ERROR) + "</td>");
+              taskAttempt.getFinishTime(),
+              taskAttempt.getStartTime()) + "</td>"); 
+    out.print("<td>" + taskAttempt.getHostname() + "</td>");
+    out.print("<td>" + taskAttempt.getError() + "</td>");
 
     // Print task log urls
     out.print("<td>");	
-    String taskLogsUrl = JobHistory.getTaskLogsUrl(taskAttempt);
+    String taskLogsUrl = HistoryViewer.getTaskLogsUrl(taskAttempt);
     if (taskLogsUrl != null) {
 	    String tailFourKBUrl = taskLogsUrl + "&start=-4097";
 	    String tailEightKBUrl = taskLogsUrl + "&start=-8193";
@@ -118,6 +121,20 @@
         out.print("n/a");
     }
     out.print("</td>");
+    Counters counters = taskAttempt.getCounters();
+    if (counters != null) {
+      TaskAttemptID attemptId = taskAttempt.getAttemptId();
+      TaskID taskId = attemptId.getTaskID();
+      org.apache.hadoop.mapreduce.JobID jobId = taskId.getJobID();
+      out.print("<td>" 
+       + "<a href=\"/taskstatshistory.jsp?jobid=" + jobId
+           + "&taskid=" + taskId
+           + "&attemptid=" + attemptId
+           + "&logFile=" + logFile + "\">"
+           + counters.countCounters() + "</a></td>");
+    } else {
+      out.print("<td></td>");
+    }
     out.print("</tr>"); 
   }
 %>



Mime
View raw message