hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079193 [3/3] - in /hadoop/mapreduce/branches/yahoo-merge/src: contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ contrib/fairscheduler/src/test/org/apache/hadoop/mapred/ contrib/mumak/src/java/org/apache/hadoop/mapred/ java/ ja...
Date Tue, 08 Mar 2011 05:54:14 GMT
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
Tue Mar  8 05:54:13 2011
@@ -302,6 +302,9 @@ public class JobHistoryParser {
     info.launchTime = event.getLaunchTime();
     info.totalMaps = event.getTotalMaps();
     info.totalReduces = event.getTotalReduces();
+    info.isUber = event.getIsUber();
+    info.numUberSubMaps = event.getNumUberSubMaps();
+    info.numUberSubReduces = event.getNumUberSubReduces();
   }
 
   private void handleJobInfoChangeEvent(JobInfoChangeEvent event) {
@@ -334,6 +337,9 @@ public class JobHistoryParser {
     long launchTime;
     int totalMaps;
     int totalReduces;
+    boolean isUber;
+    int numUberSubMaps;
+    int numUberSubReduces;
     int failedMaps;
     int failedReduces;
     int finishedMaps;
@@ -352,8 +358,9 @@ public class JobHistoryParser {
      */
     public JobInfo() {
       submitTime = launchTime = finishTime = -1;
-      totalMaps = totalReduces = failedMaps = failedReduces = 0;
-      finishedMaps = finishedReduces = 0;
+      isUber = false;
+      totalMaps = totalReduces = numUberSubMaps = numUberSubReduces = 0;
+      failedMaps = failedReduces = finishedMaps = finishedReduces = 0;
       username = jobname = jobConfPath = jobQueueName = "";
       tasksMap = new HashMap<TaskID, TaskInfo>();
       jobACLs = new HashMap<JobACL, AccessControlList>();
@@ -370,6 +377,7 @@ public class JobHistoryParser {
       System.out.println("PRIORITY: " + priority);
       System.out.println("TOTAL_MAPS: " + totalMaps);
       System.out.println("TOTAL_REDUCES: " + totalReduces);
+      //GRR FIXME:  add UBER_SUBMAPS and UBER_SUBREDUCES? (or only if isUber == true? coordinate
with TaskInfo printAll() changes)
       System.out.println("MAP_COUNTERS:" + mapCounters.toString());
       System.out.println("REDUCE_COUNTERS:" + reduceCounters.toString());
       System.out.println("TOTAL_COUNTERS: " + totalCounters.toString());
@@ -395,18 +403,24 @@ public class JobHistoryParser {
     public String getJobConfPath() { return jobConfPath; }
     /** Get the job launch time */
     public long getLaunchTime() { return launchTime; }
-    /** Get the total number of maps */
-    public long getTotalMaps() { return totalMaps; }
-    /** Get the total number of reduces */
-    public long getTotalReduces() { return totalReduces; }
+    /** Get the total number of "real" maps */
+    public int getTotalMaps() { return totalMaps; }
+    /** Get the total number of "real" reduces */
+    public int getTotalReduces() { return totalReduces; }
+    /** Was the job small enough to be converted to an UberTask? */
+    public boolean getIsUber() { return isUber; }
+    /** Get the number of sub-MapTasks within the UberTask */
+    public int getNumUberSubMaps() { return numUberSubMaps; }
+    /** Get the number of sub-ReduceTasks within the UberTask */
+    public int getNumUberSubReduces() { return numUberSubReduces; }
     /** Get the total number of failed maps */
-    public long getFailedMaps() { return failedMaps; }
+    public int getFailedMaps() { return failedMaps; }
     /** Get the number of failed reduces */
-    public long getFailedReduces() { return failedReduces; }
+    public int getFailedReduces() { return failedReduces; }
     /** Get the number of finished maps */
-    public long getFinishedMaps() { return finishedMaps; }
+    public int getFinishedMaps() { return finishedMaps; }
     /** Get the number of finished reduces */
-    public long getFinishedReduces() { return finishedReduces; }
+    public int getFinishedReduces() { return finishedReduces; }
     /** Get the job status */
     public String getJobStatus() { return jobStatus; }
     public String getErrorInfo() { return errorInfo; }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
Tue Mar  8 05:54:13 2011
@@ -44,11 +44,15 @@ public class JobInitedEvent implements H
    * @param jobStatus
    */
   public JobInitedEvent(JobID id, long launchTime, int totalMaps,
-                        int totalReduces, String jobStatus) {
+                        int totalReduces, boolean isUber, int numUberSubMaps,
+                        int numUberSubReduces, String jobStatus) {
     datum.jobid = new Utf8(id.toString());
     datum.launchTime = launchTime;
     datum.totalMaps = totalMaps;
     datum.totalReduces = totalReduces;
+    datum.isUber = isUber;
+    datum.numUberSubMaps = numUberSubMaps;
+    datum.numUberSubReduces = numUberSubReduces;
     datum.jobStatus = new Utf8(jobStatus);
   }
 
@@ -61,10 +65,16 @@ public class JobInitedEvent implements H
   public JobID getJobId() { return JobID.forName(datum.jobid.toString()); }
   /** Get the launch time */
   public long getLaunchTime() { return datum.launchTime; }
-  /** Get the total number of maps */
+  /** Get the total number of "real" maps */
   public int getTotalMaps() { return datum.totalMaps; }
-  /** Get the total number of reduces */
+  /** Get the total number of "real" reduces */
   public int getTotalReduces() { return datum.totalReduces; }
+  /** Was the job small enough to be converted to an UberTask? */
+  public boolean getIsUber() { return datum.isUber; }
+  /** Get the number of sub-MapTasks within the UberTask */
+  public int getNumUberSubMaps() { return datum.numUberSubMaps; }
+  /** Get the number of sub-ReduceTasks within the UberTask */
+  public int getNumUberSubReduces() { return datum.numUberSubReduces; }
   /** Get the status */
   public String getStatus() { return datum.jobStatus.toString(); }
  /** Get the event type */

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
Tue Mar  8 05:54:13 2011
@@ -241,8 +241,12 @@ public class TestJobQueueTaskScheduler e
         }
 
         @Override
+        public boolean getIsUber() {
+          return t.isUberTask();
+        }
+
+        @Override
         public void addFetchFailedMap(TaskAttemptID mapTaskId) {
-          
         }
       };
       status.setRunState(TaskStatus.State.RUNNING);

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java
Tue Mar  8 05:54:13 2011
@@ -103,7 +103,7 @@ public class TestTaskStatus {
     // check the default case
     String test = "hi";
     final int maxSize = 16;
-    TaskStatus status = new TaskStatus(null, 0, 0, null, test, test, null, null, 
+    TaskStatus status = new TaskStatus(null, 0, 0, null, test, test, null, null,
                                        null) {
       @Override
       protected int getMaxStringSize() {
@@ -118,6 +118,11 @@ public class TestTaskStatus {
       public boolean getIsMap() {
         return false;
       }
+
+      @Override
+      public boolean getIsUber() {
+        return false;
+      }
     };
     assertEquals("Small diagnostic info test failed", 
                  status.getDiagnosticInfo(), test);
@@ -198,6 +203,11 @@ public class TestTaskStatus {
       public boolean getIsMap() {
         return false;
       }
+
+      @Override
+      public boolean getIsUber() {
+        return false;
+      }
     };
     assertEquals("Large diagnostic info test failed", 
                 maxSize, status.getDiagnosticInfo().length());

Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
Tue Mar  8 05:54:13 2011
@@ -125,10 +125,15 @@ public class Job20LineHistoryEventEmitte
       String status = line.get("JOB_STATUS");
       String totalMaps = line.get("TOTAL_MAPS");
       String totalReduces = line.get("TOTAL_REDUCES");
+      // note:  UberTask playback not supported since uber data not yet logged
+      boolean isUber = false;
+      int numUberSubMaps = 0;
+      int numUberSubReduces = 0;
 
       if (launchTime != null && totalMaps != null && totalReduces != null)
{
-        return new JobInitedEvent(jobID, Long.parseLong(launchTime), Integer
-            .parseInt(totalMaps), Integer.parseInt(totalReduces), status);
+        return new JobInitedEvent(jobID, Long.parseLong(launchTime),
+            Integer.parseInt(totalMaps), Integer.parseInt(totalReduces),
+            isUber, numUberSubMaps, numUberSubReduces, status);
       }
 
       return null;

Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp Tue Mar  8 05:54:13
2011
@@ -365,10 +365,16 @@
               "<th>Killed</th>" +
               "<th><a href=\"jobfailures.jsp?jobid=" + jobId + 
               "\">Failed/Killed<br>Task Attempts</a></th></tr>\n");
-    printTaskSummary(out, jobId, "map", status.mapProgress(), 
-                     job.getTasks(TaskType.MAP));
-    printTaskSummary(out, jobId, "reduce", status.reduceProgress(),
-                     job.getTasks(TaskType.REDUCE));
+    if (job.getUberMode()) {
+      /* placeholder until true task- and attempt-level uber info available */
+      printTaskSummary(out, jobId, "uber", status.reduceProgress(),
+                       job.getTasks(TaskType.REDUCE));
+    } else {
+      printTaskSummary(out, jobId, "map", status.mapProgress(), 
+                       job.getTasks(TaskType.MAP));
+      printTaskSummary(out, jobId, "reduce", status.reduceProgress(),
+                       job.getTasks(TaskType.REDUCE));
+    }
     out.print("</table>\n");
     
     %>
@@ -421,6 +427,7 @@
     %>
     </table>
 
+<%if (job.getTasks(TaskType.MAP).length > 0) { %>
 <hr>Map Completion Graph - 
 <%
 if("off".equals(request.getParameter("map.graph"))) {
@@ -442,10 +449,14 @@ if("off".equals(session.getAttribute("ma
        width="<%=TaskGraphServlet.width + 2 * TaskGraphServlet.xmargin%>" 
        height="<%=TaskGraphServlet.height + 3 * TaskGraphServlet.ymargin%>"
        style="width:100%" type="image/svg+xml" pluginspage="http://www.adobe.com/svg/viewer/install/"
/>
-<%}%>
+<%} }%>
 
-<%if(job.getTasks(TaskType.REDUCE).length > 0) { %>
+<%if (job.getTasks(TaskType.REDUCE).length > 0) { %>
+<%if (job.getUberMode()) { %>
+<hr>UberTask Completion Graph -
+<%} else { %>
 <hr>Reduce Completion Graph -
+<%}%>
 <%if("off".equals(session.getAttribute("reduce.graph"))) { %>
 <a href="/jobdetails.jsp?jobid=<%=jobId%>&refresh=<%=refresh%>&reduce.graph=on"
> open </a>
 <%} else { %> 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp Tue Mar  8
05:54:13 2011
@@ -56,6 +56,12 @@
     if (job == null) {
       return;
     }
+    // MR-1220 FIXME (LATER):  to fully integrate uberization, need task-
+    // and attempt-level JobHistory/Avro/etc. changes to multiple Event
+    // types; instead, going with top-level hack for now
+    boolean isUber = job.getIsUber();
+    int numUberSubMaps = job.getNumUberSubMaps();
+    int numUberSubReduces = job.getNumUberSubReduces();
     if (job.getJobStatus().equals("FAILED")) 
       reasonforFailure = job.getErrorInfo();
 %>
@@ -90,7 +96,7 @@
 <center>
 <table border="2" cellpadding="5" cellspacing="2">
 <tr>
-<td>Kind</td><td>Total Tasks(successful+failed+killed)</td><td>Successful
tasks</td><td>Failed tasks</td><td>Killed tasks</td><td>Start
Time</td><td>Finish Time</td>
+<td>Kind</td><td>Total Tasks (successful+<wbr>failed+<wbr>killed)</td><td>Successful
tasks</td><td>Failed tasks</td><td>Killed tasks</td><td>Start
Time</td><td>Finish Time</td>
 </tr>
 <tr>
 <td>Setup</td>
@@ -106,6 +112,10 @@
     <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getSetupFinished(),
sj.getSetupStarted()) %></td>
 </tr>
 <tr>
+
+<%
+    if (!isUber) {
+%>
 <td>Map</td>
     <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=MAP&status=all">
         <%=sj.getTotalMaps()%></a></td>
@@ -131,6 +141,36 @@
     <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceStarted(),
0) %></td>
     <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceFinished(),
sj.getReduceStarted()) %></td>
 </tr>
+
+<%
+    } else /* isUber */ {
+%>
+
+<tr>
+<td>Uber</td>
+    <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=all">
+        <%=sj.getTotalReduces()%></a></td>
+    <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=SUCCEEDED">
+        <%=job.getFinishedReduces()%></a></td>
+    <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=FAILED">
+        <%=sj.getNumFailedReduces()%></a></td>
+    <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=KILLED">
+        <%=sj.getNumKilledReduces()%></a></td>  
+    <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceStarted(),
0) %></td>
+    <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceFinished(),
sj.getReduceStarted()) %></td>
+</tr>
+<td>Map subtasks</td>
+    <td colspan="6"><%=numUberSubMaps%></td>
+</tr>
+<tr>
+<td>Reduce subtasks</td>
+    <td colspan="6"><%=numUberSubReduces%></td>
+</tr>
+
+<%
+    }
+%>
+
 <tr>
 <td>Cleanup</td>
     <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=JOB_CLEANUP&status=all">



Mime
View raw message