hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r378355 - in /lucene/hadoop/trunk/src: java/org/apache/hadoop/mapred/ webapps/mapred/
Date Thu, 16 Feb 2006 20:27:52 GMT
Author: cutting
Date: Thu Feb 16 12:27:48 2006
New Revision: 378355

URL: http://svn.apache.org/viewcvs?rev=378355&view=rev
Log:
Fix so that task state is displayed even when there are no errors.  Also changed report to
be a datastructure rather than a vector of strings.

Added:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskReport.java
Modified:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java
    lucene/hadoop/trunk/src/webapps/mapred/jobdetails.jsp

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java?rev=378355&r1=378354&r2=378355&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java Thu Feb
16 12:27:48 2006
@@ -55,8 +55,8 @@
     /**
      * Grab a bunch of info on the tasks that make up the job
      */
-    public Vector[] getMapTaskReport(String jobid);
-    public Vector[] getReduceTaskReport(String jobid);
+    public TaskReport[] getMapTaskReports(String jobid);
+    public TaskReport[] getReduceTaskReports(String jobid);
 
     /**
      * A MapReduce system always operates on a single filesystem.  This 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=378355&r1=378354&r2=378355&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Thu Feb 16 12:27:48
2006
@@ -679,10 +679,10 @@
             return null;
         }
     }
-    public synchronized Vector[] getMapTaskReport(String jobid) {
+    public synchronized TaskReport[] getMapTaskReports(String jobid) {
         JobInProgress job = (JobInProgress) jobs.get(jobid);
         if (job == null) {
-            return new Vector[0];
+            return new TaskReport[0];
         } else {
             Vector reports = new Vector();
             Vector completeMapTasks = job.reportTasksInProgress(true, true);
@@ -695,14 +695,14 @@
                 TaskInProgress tip = (TaskInProgress) it.next();
                 reports.add(tip.generateSingleReport());
             }
-            return (Vector[]) reports.toArray(new Vector[reports.size()]);
+            return (TaskReport[]) reports.toArray(new TaskReport[reports.size()]);
         }
     }
 
-    public synchronized Vector[] getReduceTaskReport(String jobid) {
+    public synchronized TaskReport[] getReduceTaskReports(String jobid) {
         JobInProgress job = (JobInProgress) jobs.get(jobid);
         if (job == null) {
-            return new Vector[0];
+            return new TaskReport[0];
         } else {
             Vector reports = new Vector();
             Vector completeReduceTasks = job.reportTasksInProgress(false, true);
@@ -715,7 +715,7 @@
                 TaskInProgress tip = (TaskInProgress) it.next();
                 reports.add(tip.generateSingleReport());
             }
-            return (Vector[]) reports.toArray(new Vector[reports.size()]);
+            return (TaskReport[]) reports.toArray(new TaskReport[reports.size()]);
         }
     }
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=378355&r1=378354&r2=378355&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Thu Feb 16 12:27:48
2006
@@ -175,11 +175,11 @@
     return new JobProfile(id, job.file, "http://localhost:8080/");
   }
 
-  public Vector[] getMapTaskReport(String id) {
-    return new Vector[0];
+  public TaskReport[] getMapTaskReports(String id) {
+    return new TaskReport[0];
   }
-  public Vector[] getReduceTaskReport(String id) {
-    return new Vector[0];
+  public TaskReport[] getReduceTaskReports(String id) {
+    return new TaskReport[0];
   }
 
   public JobStatus getJobStatus(String id) {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java?rev=378355&r1=378354&r2=378355&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java Thu Feb 16 12:27:48
2006
@@ -60,6 +60,7 @@
     // Status of the TIP
     int numTaskFailures = 0;
     double progress = 0;
+    String state = "";
     long startTime = 0;
     int completes = 0;
     boolean failed = false;
@@ -215,23 +216,14 @@
      * task ID and overall status, plus reports for all the
      * component task-threads that have ever been started.
      */
-    Vector generateSingleReport() {
-        Vector report = new Vector();
-        report.add(getTIPId());
-        report.add("" + progress);
-
-        report.add(new Integer(taskDiagnosticData.size()));
-        for (Iterator it = taskDiagnosticData.keySet().iterator(); it.hasNext(); ) {
-            String taskid = (String) it.next();
-            Vector taskData = (Vector) taskDiagnosticData.get(taskid);
-
-            TaskStatus taskStatus = (TaskStatus) taskStatuses.get(taskid);
-            String taskStateString = taskStatus.getStateString();
-
-            report.add(taskData);
-            report.add(taskStateString);
-        }
-        return report;
+    TaskReport generateSingleReport() {
+      ArrayList diagnostics = new ArrayList();
+      for (Iterator i = taskDiagnosticData.values().iterator(); i.hasNext();) {
+        diagnostics.addAll((Vector)i.next());
+      }
+      return new TaskReport
+        (getTIPId(), (float)progress, state,
+         (String[])diagnostics.toArray(new String[diagnostics.size()]));
     }
 
     ////////////////////////////////////////////////
@@ -333,17 +325,23 @@
             this.progress = 0;
         } else {
             double bestProgress = 0;
+            String bestState = "";
             for (Iterator it = taskStatuses.keySet().iterator(); it.hasNext(); ) {
                 String taskid = (String) it.next();
                 TaskStatus status = (TaskStatus) taskStatuses.get(taskid);
                 if (status.getRunState() == TaskStatus.SUCCEEDED) {
                     bestProgress = 1;
+                    bestState = status.getStateString();
                     break;
                 } else if (status.getRunState() == TaskStatus.RUNNING) {
-                    bestProgress = Math.max(bestProgress, status.getProgress());
+                  if (status.getProgress() >= bestProgress) {
+                    bestProgress = status.getProgress();
+                    bestState = status.getStateString();
+                  }
                 }
             }
             this.progress = bestProgress;
+            this.state = bestState;
         }
     }
 

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskReport.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskReport.java?rev=378355&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskReport.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskReport.java Thu Feb 16 12:27:48
2006
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.io.*;
+
+import java.io.*;
+
+/** A report on the state of a task. */
+public class TaskReport implements Writable {
+  private String taskid;
+  private float progress;
+  private String state;
+  private String[] diagnostics;
+
+  public TaskReport() {}
+
+  TaskReport(String taskid, float progress, String state,
+             String[] diagnostics) {
+    this.taskid = taskid;
+    this.progress = progress;
+    this.state = state;
+    this.diagnostics = diagnostics;
+  }
+    
+  /** The id of the task. */
+  public String getTaskId() { return taskid; }
+  /** The amount completed, between zero and one. */
+  public float getProgress() { return progress; }
+  /** The most recent state, reported by a {@link Reporter}. */
+  public String getState() { return state; }
+  /** A list of error messages. */
+  public String[] getDiagnostics() { return diagnostics; }
+
+  //////////////////////////////////////////////
+  // Writable
+  //////////////////////////////////////////////
+  public void write(DataOutput out) throws IOException {
+    UTF8.writeString(out, taskid);
+    out.writeFloat(progress);
+    UTF8.writeString(out, state);
+    new ObjectWritable(diagnostics).write(out);
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    this.taskid = UTF8.readString(in);
+    this.progress = in.readFloat();
+    this.state = UTF8.readString(in);
+
+    ObjectWritable wrapper = new ObjectWritable();
+    wrapper.readFields(in);
+    diagnostics = (String[])wrapper.get();
+  }
+}

Modified: lucene/hadoop/trunk/src/webapps/mapred/jobdetails.jsp
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/webapps/mapred/jobdetails.jsp?rev=378355&r1=378354&r2=378355&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/mapred/jobdetails.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/mapred/jobdetails.jsp Thu Feb 16 12:27:48 2006
@@ -13,8 +13,8 @@
   JobProfile profile = (job != null) ? (job.getProfile()) : null;
   JobStatus status = (job != null) ? (job.getStatus()) : null;
 
-  Vector mapTaskReports[] = (job != null) ? tracker.getMapTaskReport(jobid) : null;
-  Vector reduceTaskReports[] = (job != null) ? tracker.getReduceTaskReport(jobid) : null;
+  TaskReport[] mapTaskReports = (job != null) ? tracker.getMapTaskReports(jobid) : null;
+  TaskReport[] reduceTaskReports = (job != null) ? tracker.getReduceTaskReports(jobid) :
null;
 %>
 
 <html>
@@ -45,28 +45,21 @@
 <h2>Map Tasks</h2>
   <center>
   <table border=2 cellpadding="5" cellspacing="2">
-  <tr><td align="center">Map Task Id</td><td>Pct Complete</td><td>Diagnostic
Data</td></tr>
+  <tr><td align="center">Task Id</td><td>Complete</td><td>State</td><td>Errors</td></tr>
 
   <%
+
     for (int i = 0; i < mapTaskReports.length; i++) {
-      Vector v = mapTaskReports[i];
-      String tipid = (String) v.elementAt(0);
-      String progress = (String) v.elementAt(1);
-      int diagnosticSize = ((Integer) v.elementAt(2)).intValue();
-
-      out.print("<tr><td>" + tipid + "</td><td>" + progress + "</td><td>");
-      for (int j = 0; j < diagnosticSize; j++) {
-        Vector taskData = (Vector) v.elementAt(3 + ((2 * j)));
-        String taskStateString = (String) v.elementAt(3 + ((2 * j) + 1));
-        out.print(taskStateString);
-        out.print("<b>");
-
-        for (Iterator it2 = taskData.iterator(); it2.hasNext(); ) {
-          out.print("" + it2.next());
-          out.println("<b>");
-        }
+      TaskReport report = mapTaskReports[i];
+
+      out.print("<tr><td>" + report.getTaskId() + "</td>");
+      out.print("<td>" + report.getProgress() + "</td>");
+      out.print("<td>" + report.getState() + "</td>");
+
+      String[] diagnostics = report.getDiagnostics();
+      for (int j = 0; j < diagnostics.length ; j++) {
+        out.print("<td>" + diagnostics[j] + "</td>");
       }
-      out.print("</td>");
       out.print("</tr>\n");
     }
   %>
@@ -78,28 +71,20 @@
 <h2>Reduce Tasks</h2>
   <center>
   <table border=2 cellpadding="5" cellspacing="2">
-  <tr><td align="center">Reduce Task Id</td><td>Pct Complete</td><td>Diagnostic
Data</td></tr>
+  <tr><td align="center">Task Id</td><td>Complete</td><td>State</td><td>Errors</td></tr>
 
   <%
     for (int i = 0; i < reduceTaskReports.length; i++) {
-      Vector v = reduceTaskReports[i];
-      String tipid = (String) v.elementAt(0);
-      String progress = (String) v.elementAt(1);
-      int diagnosticSize = ((Integer) v.elementAt(2)).intValue();
-
-      out.print("<tr><td>" + tipid + "</td><td>" + progress + "</td><td>");
-      for (int j = 0; j < diagnosticSize; j++) {
-        Vector taskData = (Vector) v.elementAt(3 + ((2 * j)));
-        String taskStateString = (String) v.elementAt(3 + ((2 * j) + 1));
-        out.print(taskStateString);
-        out.print("<b>");
-
-        for (Iterator it2 = taskData.iterator(); it2.hasNext(); ) {
-          out.print("" + it2.next());
-          out.println("<b>");
-        }
+      TaskReport report = reduceTaskReports[i];
+
+      out.print("<tr><td>" + report.getTaskId() + "</td>");
+      out.print("<td>" + report.getProgress() + "</td>");
+      out.print("<td>" + report.getState() + "</td>");
+
+      String[] diagnostics = report.getDiagnostics();
+      for (int j = 0; j < diagnostics.length ; j++) {
+        out.print("<td>" + diagnostics[j] + "</td>");
       }
-      out.print("</td>");
       out.print("</tr>\n");
     }
   %>



Mime
View raw message