hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r713222 - in /hadoop/core/trunk: ./ src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/ src/mapred/org/apache/hadoop/mapred/ src/webapps/job/
Date Tue, 11 Nov 2008 23:39:53 GMT
Author: szetszwo
Date: Tue Nov 11 15:39:52 2008
New Revision: 713222

URL: http://svn.apache.org/viewvc?rev=713222&view=rev
Log:
HADOOP-3923. Remove org.apache.hadoop.mapred.StatusHttpServer.  (szetszwo)

Added:
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskGraphServlet.java
Removed:
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/StatusHttpServer.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/core/trunk/src/webapps/job/jobdetails.jsp

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=713222&r1=713221&r2=713222&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Nov 11 15:39:52 2008
@@ -88,6 +88,8 @@
     HADOOP-4583. Several code optimizations in HDFS.  (Suresh Srinivas via
     szetszwo)
 
+    HADOOP-3923. Remove org.apache.hadoop.mapred.StatusHttpServer.  (szetszwo)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java?rev=713222&r1=713221&r2=713222&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
(original)
+++ hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
Tue Nov 11 15:39:52 2008
@@ -34,6 +34,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.mapred.JobStatus;
 import org.apache.hadoop.util.ReflectionUtils;
 
@@ -145,7 +146,7 @@
       // Register servlet with JobTracker's Jetty server
       if (taskTrackerManager instanceof JobTracker) {
         JobTracker jobTracker = (JobTracker) taskTrackerManager;
-        StatusHttpServer infoServer = jobTracker.infoServer;
+        HttpServer infoServer = jobTracker.infoServer;
         infoServer.setAttribute("scheduler", this);
         infoServer.addServlet("scheduler", "/scheduler",
             FairSchedulerServlet.class);

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=713222&r1=713221&r2=713222&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java Tue Nov 11 15:39:52
2008
@@ -50,6 +50,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.ipc.Server;
@@ -1036,7 +1037,7 @@
                                    );
 
   // Used to provide an HTML view on Job, Task, and TaskTracker structures
-  StatusHttpServer infoServer;
+  final HttpServer infoServer;
   int infoPort;
 
   Server interTrackerServer;
@@ -1103,7 +1104,7 @@
     String infoBindAddress = infoSocAddr.getHostName();
     int tmpInfoPort = infoSocAddr.getPort();
     this.startTime = System.currentTimeMillis();
-    infoServer = new StatusHttpServer("job", infoBindAddress, tmpInfoPort, 
+    infoServer = new HttpServer("job", infoBindAddress, tmpInfoPort, 
         tmpInfoPort == 0, conf);
     infoServer.setAttribute("job.tracker", this);
     // initialize history parameters.
@@ -1117,6 +1118,7 @@
       historyFS = new Path(historyLogDir).getFileSystem(conf);
       infoServer.setAttribute("fileSys", historyFS);
     }
+    infoServer.addServlet("reducegraph", "/taskgraph", TaskGraphServlet.class);
     infoServer.start();
     
     trackerIdentifier = getDateFormat().format(new Date());

Added: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskGraphServlet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskGraphServlet.java?rev=713222&view=auto
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskGraphServlet.java (added)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskGraphServlet.java Tue Nov 11
15:39:52 2008
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+/** The servlet that outputs svg graphics for map / reduce task
+ *  statuses
+ */
+public class TaskGraphServlet extends HttpServlet {
+
+  private static final long serialVersionUID = -1365683739392460020L;
+
+  /**height of the graph w/o margins*/ 
+  public static final int width = 600;
+  
+  /**height of the graph w/o margins*/ 
+  public static final int height = 200;
+  
+  /**margin space on y axis */
+  public static final int ymargin = 20;
+
+  /**margin space on x axis */
+  public static final int xmargin = 80;
+  
+  private static final float oneThird = 1f / 3f;
+  
+  @Override
+  public void doGet(HttpServletRequest request, HttpServletResponse response)
+    throws ServletException, IOException {
+
+    response.setContentType("image/svg+xml");
+
+    JobTracker tracker = 
+      (JobTracker) getServletContext().getAttribute("job.tracker");
+    
+    String jobIdStr = request.getParameter("jobid");
+    if(jobIdStr == null)
+      return;
+    JobID jobId = JobID.forName(jobIdStr);
+
+    final boolean isMap = "map".equalsIgnoreCase(request.getParameter("type"));
+    final TaskReport[] reports = isMap? tracker.getMapTaskReports(jobId) 
+                                      : tracker.getReduceTaskReports(jobId);
+    if(reports == null || reports.length == 0) {
+      return;
+    }
+
+    final int numTasks = reports.length;     
+    int tasksPerBar = (int)Math.ceil(numTasks / 600d);
+    int numBars = (int) Math.ceil((double)numTasks / tasksPerBar);
+    int w = Math.max(600, numBars);
+    int barWidth = Math.min(10,  w / numBars); //min 1px, max 10px
+    int barsPerNotch = (int)Math.ceil(10d / barWidth);
+    w = w + numBars / barsPerNotch;
+    int totalWidth = w + 2 * xmargin;
+    
+    //draw a white rectangle
+    final PrintWriter out = response.getWriter();
+    out.print("<?xml version=\"1.0\" standalone=\"no\"?>\n" + 
+      "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\" \n" + 
+      "\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n" +
+      "<?xml-stylesheet type=\"text/css\" href=\"/static/hadoop.css\"?>\n\n"+
+      "<svg width=\"");out.print(totalWidth);
+    out.print("\" height=\"");out.print(height + 2 * ymargin);
+    out.print("\" version=\"1.1\"\n" + 
+      "xmlns=\"http://www.w3.org/2000/svg\">\n\n"); 
+    
+    //axes
+    printLine(out, xmargin - 1, xmargin - 1, height + ymargin + 1
+        , ymargin - 1, "black" );
+    printLine(out, xmargin - 1, w + xmargin + 1 ,height + ymargin + 1 
+        , height + ymargin + 1, "black" );
+    
+    //borderlines
+    printLine(out, w + xmargin + 1 , w + xmargin +1
+        , height + ymargin + 1,ymargin - 1, "#CCCCCC" );
+    printLine(out, xmargin - 1, w + xmargin + 1
+        , ymargin - 1 , ymargin - 1, "#CCCCCC" );
+    
+    String[]  colors = new String[] {"#00DD00", "#E50000", "#AAAAFF"};
+    
+    //determine the notch interval using the number of digits for numTasks
+    int xNotchInterval = (int)(Math.ceil( numTasks / 10d));
+    
+    int xOffset = -1; 
+    int xNotchCount = 0;
+    //task bar graph
+    for(int i=0, barCnt=0; ;i+=tasksPerBar, barCnt++) {
+      if(barCnt % barsPerNotch == 0) {
+        xOffset++;
+      }
+      int x = barCnt * barWidth + xmargin + xOffset;
+      //x axis notches
+      if(i >= xNotchInterval * xNotchCount) {
+        printLine(out, x, x, height + ymargin + 3 
+            , height + ymargin - 2, "black");
+        printText(out, x, height + ymargin + 15 
+            , String.valueOf(xNotchInterval * xNotchCount++ ), "middle");
+      }
+      if(i >= reports.length) break;
+      
+      if(isMap) {
+        float progress = getMapAvarageProgress(tasksPerBar, i, reports);
+        int barHeight = (int)Math.ceil(height * progress);
+        int y = height - barHeight + ymargin;
+        printRect(out, barWidth, barHeight,x , y , colors[2]);
+      }
+      else {
+        float[] progresses 
+          = getReduceAvarageProgresses(tasksPerBar, i, reports);
+        //draw three bars stacked, for copy, sort, reduce
+        
+        int prevHeight =0;
+        for(int j=0; j < 3 ; j++) {
+          int barHeight = (int)((height / 3) * progresses[j]);
+          if(barHeight > height/ 3 - 3)//fix rounding error
+            barHeight = height / 3 + 1;
+          
+          int y = height - barHeight + ymargin - prevHeight;
+          prevHeight += barHeight;
+          printRect(out, barWidth, barHeight, x, y, colors[j] );
+        }
+      }
+    }
+    
+    //y axis notches
+    for(int i=0;i<=10;i++) {
+      printLine(out, xmargin-3 , xmargin+2 , ymargin + (i * height) / 10
+          , ymargin + (i * height) / 10 , "black");
+      printText(out, xmargin - 10 , ymargin + 4 + (i * height) / 10 
+          , String.valueOf(100 - i * 10), "end");
+    }
+    
+    if(!isMap) {
+      //print color codes for copy, sort, reduce
+      printRect(out, 14, 14, xmargin + w + 4, ymargin + 20, colors[0]);
+      printText(out, xmargin + w + 24, ymargin + 30, "copy", "start");
+      printRect(out, 14, 14, xmargin + w + 4, ymargin + 50, colors[1]);
+      printText(out, xmargin + w + 24, ymargin + 60, "sort", "start");
+      printRect(out, 14, 14, xmargin + w + 4, ymargin + 80, colors[2]);
+      printText(out, xmargin + w + 24, ymargin + 90, "reduce", "start");
+    }
+    
+    
+    //firefox curently does not support vertical text
+    //out.print("<text x=\"");out.print(6);
+    //out.print("\" y=\""); out.print(ymargin + height / 2); 
+    //out.print("\" style=\"text-anchor:middle;writing-mode:tb\">"
+    //+"Percent</text>\n");
+    
+    out.print("</svg>");
+  }
+
+  /**Computes average progress per bar*/
+  private float getMapAvarageProgress(int tasksPerBar, int index
+      , TaskReport[] reports ) {
+    float progress = 0f;
+    int k=0;
+    for(;k < tasksPerBar && index + k < reports.length; k++) { 
+      progress += reports[index + k].getProgress();
+    }
+    progress /= k;
+    return progress;
+  }
+
+  /**Computes average progresses per bar*/
+  private float[] getReduceAvarageProgresses(int tasksPerBar, int index
+      , TaskReport[] reports ) {
+    float[] progresses = new float[] {0,0,0};
+    int k=0;
+    for(;k < tasksPerBar && index + k < reports.length; k++) {
+      float progress = reports[index+k].getProgress();
+      for(int j=0; progress > 0 ; j++, progress -= oneThird) {
+        if(progress > oneThird)
+          progresses[j] += 1f;
+        else 
+          progresses[j] += progress * 3 ;
+      }
+    }
+    for(int j=0; j<3; j++) { progresses[j] /= k;}
+    
+    return progresses;
+  }
+  
+  private void printRect(PrintWriter out, int width, int height
+      , int x, int y, String color) throws IOException {
+    if(height > 0) {
+      out.print("<rect width=\"");out.print(width);
+      out.print("\" height=\"");  out.print(height);
+      out.print("\" x=\""); out.print(x);
+      out.print("\" y=\""); out.print(y);
+      out.print("\" style=\"fill:"); out.print(color);out.print("\"/>\n");
+    }
+  }
+  private void printLine(PrintWriter out, int x1, int x2
+      , int y1, int y2, String color) throws IOException {
+    out.print("<line x1=\"");out.print(x1);
+    out.print("\" x2=\"");out.print(x2);
+    out.print("\" y1=\"");out.print(y1);
+    out.print("\" y2=\""); out.print(y2);
+    out.print("\" class=\"taskgraphline\" style=\"stroke:"); 
+    out.print(color); out.print("\"/>\n"); 
+  }
+  private void printText(PrintWriter out, int x, int y, String text
+      , String anchor) throws IOException {
+    out.print("<text x=\"");out.print(String.valueOf(x));
+    out.print("\" y=\""); out.print(String.valueOf(y));
+    out.print("\" style=\"fill:black;font-family:sans-serif;" 
+        + "text-anchor:");out.print(anchor); out.print("\">");
+    out.print(text); out.print("</text>\n");
+  }
+}
+

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=713222&r1=713221&r2=713222&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Tue Nov 11 15:39:52
2008
@@ -17,15 +17,10 @@
  */
  package org.apache.hadoop.mapred;
 
-import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.io.PrintStream;
 import java.io.RandomAccessFile;
 import java.net.InetSocketAddress;
 import java.net.URI;
@@ -35,13 +30,13 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.LinkedHashMap;
 import java.util.Vector;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -59,20 +54,18 @@
 import org.apache.hadoop.filecache.DistributedCache;
 import org.apache.hadoop.fs.DF;
 import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSError;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.mapred.JobClient.TaskStatusFilter;
 import org.apache.hadoop.mapred.TaskStatus.Phase;
 import org.apache.hadoop.mapred.pipes.Submitter;
 import org.apache.hadoop.metrics.MetricsContext;
@@ -80,7 +73,6 @@
 import org.apache.hadoop.metrics.MetricsRecord;
 import org.apache.hadoop.metrics.MetricsUtil;
 import org.apache.hadoop.metrics.Updater;
-import org.apache.hadoop.metrics.jvm.JvmMetrics;
 import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.util.DiskChecker;
@@ -91,7 +83,6 @@
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.log4j.LogManager;
 
 /*******************************************************
  * TaskTracker is a process that starts and tracks MR Tasks
@@ -149,7 +140,7 @@
   // The filesystem where job files are stored
   FileSystem systemFS = null;
   
-  StatusHttpServer server = null;
+  private final HttpServer server;
     
   volatile boolean shuttingDown = false;
     
@@ -583,7 +574,6 @@
           }
           synchronized (waitingOn) {
             try {
-              int waitTime;
               if (!fetchAgain) {
                 waitingOn.wait(heartbeatInterval);
               }
@@ -623,7 +613,6 @@
       // Note that the sync is first on fromEventId and then on allMapEvents
       synchronized (fromEventId) {
         synchronized (allMapEvents) {
-          int index = 0;
           if (allMapEvents.size() > lastKnownIndex) {
             fromEventId.set(lastKnownIndex);
             allMapEvents = allMapEvents.subList(0, lastKnownIndex);
@@ -874,7 +863,7 @@
     InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
     String httpBindAddress = infoSocAddr.getHostName();
     int httpPort = infoSocAddr.getPort();
-    this.server = new StatusHttpServer("task", httpBindAddress, httpPort,
+    this.server = new HttpServer("task", httpBindAddress, httpPort,
         httpPort == 0, conf);
     workerThreads = conf.getInt("tasktracker.http.threads", 40);
     this.shuffleServerMetrics = new ShuffleServerMetrics(conf);

Modified: hadoop/core/trunk/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobdetails.jsp?rev=713222&r1=713221&r2=713222&view=diff
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobdetails.jsp (original)
+++ hadoop/core/trunk/src/webapps/job/jobdetails.jsp Tue Nov 11 15:39:52 2008
@@ -179,7 +179,7 @@
     }
 %>
 
-<%@page import="org.apache.hadoop.mapred.StatusHttpServer.TaskGraphServlet"%>
+<%@page import="org.apache.hadoop.mapred.TaskGraphServlet"%>
 <html>
 <head>
   <% 
@@ -328,8 +328,8 @@
 <%} else { %> 
 <a href="/jobdetails.jsp?jobid=<%=jobId%>&refresh=<%=refresh%>&map.graph=off"
> close </a>
 <br><embed src="/taskgraph?type=map&jobid=<%=jobId%>" 
-       width="<%=StatusHttpServer.TaskGraphServlet.width + 2 * StatusHttpServer.TaskGraphServlet.xmargin%>"

-       height="<%=StatusHttpServer.TaskGraphServlet.height + 3 * StatusHttpServer.TaskGraphServlet.ymargin%>"
+       width="<%=TaskGraphServlet.width + 2 * TaskGraphServlet.xmargin%>" 
+       height="<%=TaskGraphServlet.height + 3 * TaskGraphServlet.ymargin%>"
        style="width:100%" type="image/svg+xml" pluginspage="http://www.adobe.com/svg/viewer/install/"
/>
 <%}%>
 
@@ -341,8 +341,8 @@
 <a href="/jobdetails.jsp?jobid=<%=jobId%>&refresh=<%=refresh%>&reduce.graph=off"
> close </a>
  
  <br><embed src="/taskgraph?type=reduce&jobid=<%=jobId%>" 
-       width="<%=StatusHttpServer.TaskGraphServlet.width + 2 * StatusHttpServer.TaskGraphServlet.xmargin%>"

-       height="<%=StatusHttpServer.TaskGraphServlet.height + 3 * StatusHttpServer.TaskGraphServlet.ymargin%>"

+       width="<%=TaskGraphServlet.width + 2 * TaskGraphServlet.xmargin%>" 
+       height="<%=TaskGraphServlet.height + 3 * TaskGraphServlet.ymargin%>" 
        style="width:100%" type="image/svg+xml" pluginspage="http://www.adobe.com/svg/viewer/install/"
/>
 <%} }%>
 



Mime
View raw message