hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r816800 - in /hadoop/mapreduce/trunk: ./ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapreduce/server/jobtracker/ src/test/mapred/org/apache/hadoop/mapred/ src/webapps/job/
Date Fri, 18 Sep 2009 22:09:24 GMT
Author: tomwhite
Date: Fri Sep 18 22:09:24 2009
New Revision: 816800

URL: http://svn.apache.org/viewvc?rev=816800&view=rev
Log:
MAPREDUCE-679. XML-based metrics as JSP servlet for JobTracker. Contributed by Aaron Kimball.

Added:
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JobTrackerJspHelper.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerXmlJsp.java
    hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jspx
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/build.xml
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JSPUtil.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=816800&r1=816799&r2=816800&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Sep 18 22:09:24 2009
@@ -118,6 +118,9 @@
     or output format taking advantage of parallel read and write properties of 
     the DBMS. (Omer Trajman via ddas)
 
+    MAPREDUCE-679. XML-based metrics as JSP servlet for JobTracker.
+    (Aaron Kimball via tomwhite)
+
   IMPROVEMENTS
 
     MAPREDUCE-816. Rename "local" mysql import to "direct" in Sqoop.

Modified: hadoop/mapreduce/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/build.xml?rev=816800&r1=816799&r2=816800&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/build.xml (original)
+++ hadoop/mapreduce/trunk/build.xml Fri Sep 18 22:09:24 2009
@@ -289,16 +289,10 @@
     <copy todir="${build.webapps}">
       <fileset dir="${src.webapps}">
         <exclude name="**/*.jsp" />
+        <exclude name="**/*.jspx" />
       </fileset>
     </copy>
 
-    <unzip src="${lib.dir}/hadoop-hdfs-${hadoop-hdfs.version}.jar"
-        dest="${build.dir}">
-      <patternset>
-        <include name="webapps/**"/>
-      </patternset>
-    </unzip>
-
     <copy todir="${conf.dir}" verbose="true">
       <fileset dir="${conf.dir}" includes="**/*.template"/>
       <mapper type="glob" from="*.template" to="*"/>

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JSPUtil.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JSPUtil.java?rev=816800&r1=816799&r2=816800&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JSPUtil.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JSPUtil.java Fri Sep 18 22:09:24
2009
@@ -27,6 +27,7 @@
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import javax.servlet.jsp.JspWriter;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -259,6 +260,43 @@
     return sb.toString();
   }
 
+  @SuppressWarnings("unchecked")
+  public static void generateRetiredJobXml(JspWriter out, JobTracker tracker, int rowId)
+      throws IOException {
+
+    Iterator<JobStatus> iterator =
+      tracker.retireJobs.getAll().descendingIterator();
+
+    for (int i = 0; i < 100 && iterator.hasNext(); i++) {
+      JobStatus status = iterator.next();
+      StringBuilder sb = new StringBuilder();
+      sb.append("<retired_job rowid=\"" + rowId + "\" jobid=\"" + status.getJobId() +
"\">");
+      sb.append("<jobid>" + status.getJobId() + "</jobid>");
+      sb.append("<history_url>jobdetailshistory.jsp?jobid=" + status.getJobId()
+          + "&amp;logFile="
+          + URLEncoder.encode(status.getHistoryFile().toString(), "UTF-8")
+          + "</history_url>");
+      sb.append("<priority>" + status.getJobPriority().toString()
+          + "</priority>");
+      sb.append("<user>" + status.getUsername() + "</user>");
+      sb.append("<name>" + status.getJobName() + "</name>");
+      sb.append("<run_state>" + JobStatus.getJobRunState(status.getRunState())
+          + "</run_state>");
+      sb.append("<start_time>" + new Date(status.getStartTime())
+          + "</start_time>");
+      sb.append("<finish_time>" + new Date(status.getFinishTime())
+          + "</finish_time>");
+      sb.append("<map_complete>" + StringUtils.formatPercent(
+          status.mapProgress(), 2) + "</map_complete>");
+      sb.append("<reduce_complete>" + StringUtils.formatPercent(
+          status.reduceProgress(), 2) + "</reduce_complete>");
+      sb.append("<scheduling_info>" + status.getSchedulingInfo() + "</scheduling_info>");
+      sb.append("</retired_job>\n");
+      out.write(sb.toString());
+      rowId++;
+    }
+  }
+
   static final boolean privateActionsAllowed() {
     return conf.getBoolean(PRIVATE_ACTIONS_KEY, false);
   }

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=816800&r1=816799&r2=816800&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Sep 18 22:09:24
2009
@@ -1926,6 +1926,13 @@
     }
     return v;
   }
+
+  public synchronized List<JobInProgress> getFailedJobs() {
+    synchronized (jobs) {
+      return failedJobs();
+    }
+  }
+
   public Vector<JobInProgress> completedJobs() {
     Vector<JobInProgress> v = new Vector<JobInProgress>();
     for (Iterator it = jobs.values().iterator(); it.hasNext();) {
@@ -1938,6 +1945,12 @@
     return v;
   }
 
+  public synchronized List<JobInProgress> getCompletedJobs() {
+    synchronized (jobs) {
+      return completedJobs();
+    }
+  }
+
   /**
    * Get all the task trackers in the cluster
    * 

Added: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JobTrackerJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JobTrackerJspHelper.java?rev=816800&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JobTrackerJspHelper.java
(added)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JobTrackerJspHelper.java
Fri Sep 18 22:09:24 2009
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.server.jobtracker;
+
+import java.io.IOException;
+import java.util.List;
+import java.text.DecimalFormat;
+
+import javax.servlet.jsp.JspWriter;
+import javax.servlet.http.*;
+
+import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.JobInProgress;
+import org.apache.hadoop.mapred.JobProfile;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.TaskTrackerStatus;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Methods to help format output for JobTracker XML JSPX
+ */
+public class JobTrackerJspHelper {
+
+  public JobTrackerJspHelper() {
+    percentFormat = new DecimalFormat("##0.00");
+  }
+
+  private DecimalFormat percentFormat;
+
+  /**
+   * Returns an XML-formatted table of the jobs in the list.
+   * This is called repeatedly for different lists of jobs (e.g., running, completed, failed).
+   */
+  public void generateJobTable(JspWriter out, String label, List<JobInProgress> jobs)
+      throws IOException {
+    if (jobs.size() > 0) {
+      for (JobInProgress job : jobs) {
+        JobProfile profile = job.getProfile();
+        JobStatus status = job.getStatus();
+        JobID jobid = profile.getJobID();
+
+        int desiredMaps = job.desiredMaps();
+        int desiredReduces = job.desiredReduces();
+        int completedMaps = job.finishedMaps();
+        int completedReduces = job.finishedReduces();
+        String name = profile.getJobName();
+
+        out.print("<" + label + "_job jobid=\"" + jobid + "\">\n");
+        out.print("  <jobid>" + jobid + "</jobid>\n");
+        out.print("  <user>" + profile.getUser() + "</user>\n");
+        out.print("  <name>" + ("".equals(name) ? "&nbsp;" : name) + "</name>\n");
+        out.print("  <map_complete>" + StringUtils.formatPercent(status.mapProgress(),
2) + "</map_complete>\n");
+        out.print("  <map_total>" + desiredMaps + "</map_total>\n");
+        out.print("  <maps_completed>" + completedMaps + "</maps_completed>\n");
+        out.print("  <reduce_complete>" + StringUtils.formatPercent(status.reduceProgress(),
2) + "</reduce_complete>\n");
+        out.print("  <reduce_total>" + desiredReduces + "</reduce_total>\n");
+        out.print("  <reduces_completed>" + completedReduces + "</reduces_completed>\n");
+        out.print("</" + label + "_job>\n");
+      }
+    }
+  }
+
+  /**
+   * Generates an XML-formatted block that summarizes the state of the JobTracker.
+   */
+  public void generateSummaryTable(JspWriter out,
+                                   JobTracker tracker) throws IOException {
+    ClusterStatus status = tracker.getClusterStatus();
+    int maxMapTasks = status.getMaxMapTasks();
+    int maxReduceTasks = status.getMaxReduceTasks();
+    int numTaskTrackers = status.getTaskTrackers();
+    String tasksPerNodeStr;
+    if (numTaskTrackers > 0) {
+      double tasksPerNodePct = (double) (maxMapTasks + maxReduceTasks) / (double) numTaskTrackers;
+      tasksPerNodeStr = percentFormat.format(tasksPerNodePct);
+    } else {
+      tasksPerNodeStr = "-";
+    }
+    out.print("<maps>" + status.getMapTasks() + "</maps>\n" +
+            "<reduces>" + status.getReduceTasks() + "</reduces>\n" +
+            "<total_submissions>" + tracker.getTotalSubmissions() + "</total_submissions>\n"
+
+            "<nodes>" + status.getTaskTrackers() + "</nodes>\n" +
+            "<map_task_capacity>" + status.getMaxMapTasks() + "</map_task_capacity>\n"
+
+            "<reduce_task_capacity>" + status.getMaxReduceTasks() + "</reduce_task_capacity>\n"
+
+            "<avg_tasks_per_node>" + tasksPerNodeStr + "</avg_tasks_per_node>\n");
+  }
+}

Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerXmlJsp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerXmlJsp.java?rev=816800&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerXmlJsp.java
(added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerXmlJsp.java
Fri Sep 18 22:09:24 2009
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class TestJobTrackerXmlJsp extends ClusterMapReduceTestCase {
+
+  private static final Log LOG = LogFactory.getLog(TestJobTrackerXmlJsp.class);
+
+  /**
+   * Read the jobtracker.jspx status page and validate that the XML is well formed.
+   */
+  public void testXmlWellFormed() throws IOException, ParserConfigurationException, SAXException
{
+    MiniMRCluster cluster = getMRCluster();
+    int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
+
+    String xmlJspUrl = "http://localhost:" + infoPort + "/jobtracker.jspx";
+    LOG.info("Retrieving XML from URL: " + xmlJspUrl);
+
+    DocumentBuilder parser = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+    Document doc = parser.parse(xmlJspUrl);
+
+    // If we get here, then the document was successfully parsed by SAX and is well-formed.
+    LOG.info("Document received and parsed.");
+
+    // Make sure it has a <cluster> element as top-level.
+    NodeList clusterNodes = doc.getElementsByTagName("cluster");
+    assertEquals("There should be exactly 1 <cluster> element", 1, clusterNodes.getLength());
+  }
+}

Modified: hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp?rev=816800&r1=816799&r2=816800&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp (original)
+++ hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp Fri Sep 18 22:09:24 2009
@@ -35,9 +35,9 @@
   String trackerName = 
            StringUtils.simpleHostname(tracker.getJobTrackerMachine());
   JobQueueInfo[] queues = tracker.getJobQueues();
-  Vector<JobInProgress> runningJobs = tracker.runningJobs();
-  Vector<JobInProgress> completedJobs = tracker.completedJobs();
-  Vector<JobInProgress> failedJobs = tracker.failedJobs();
+  List<JobInProgress> runningJobs = tracker.getRunningJobs();
+  List<JobInProgress> completedJobs = tracker.getCompletedJobs();
+  List<JobInProgress> failedJobs = tracker.getFailedJobs();
 %>
 <%!
   private static DecimalFormat percentFormat = new DecimalFormat("##0.00");

Added: hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jspx
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jspx?rev=816800&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jspx (added)
+++ hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jspx Fri Sep 18 22:09:24 2009
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file 
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<cluster xmlns:jsp="http://java.sun.com/JSP/Page">
+  <jsp:directive.page contentType="text/xml; charset=UTF-8" session="false" />
+  <jsp:directive.page import="java.util.Date" />
+  <jsp:directive.page import="java.util.List" />
+  <jsp:directive.page import="org.apache.hadoop.mapreduce.server.jobtracker.JobTrackerJspHelper"
/>
+  <jsp:directive.page import="org.apache.hadoop.util.*" />
+  <jsp:directive.page import="org.apache.hadoop.mapreduce.*" />
+  <jsp:directive.page import="org.apache.hadoop.mapred.JSPUtil" />
+
+  <jsp:scriptlet>
+    response.setHeader("Pragma", "no-cache");
+    response.setHeader("Cache-Control", "no-store");
+    response.setDateHeader("Expires", -1);
+  </jsp:scriptlet>
+  <jsp:scriptlet>
+    JobTracker tracker = (JobTracker) application.getAttribute("job.tracker");
+    String trackerName = StringUtils.simpleHostname(tracker.getJobTrackerMachine());
+    JobTrackerJspHelper jspHelper = new JobTrackerJspHelper();
+
+    List&lt;JobInProgress&gt; runningJobs = tracker.getRunningJobs();
+    List&lt;JobInProgress&gt; completedJobs = tracker.getCompletedJobs();
+    List&lt;JobInProgress&gt; failedJobs = tracker.getFailedJobs();
+  </jsp:scriptlet>
+
+    <tracker_name><jsp:expression>trackerName</jsp:expression></tracker_name>
+
+    <tracker>
+      <state><jsp:expression>tracker.getClusterStatus().getJobTrackerState()</jsp:expression></state>
+      <started><jsp:expression>new Date(tracker.getStartTime())</jsp:expression></started>
+      <version><jsp:expression>VersionInfo.getVersion()</jsp:expression></version>
+      <revision><jsp:expression>VersionInfo.getRevision()</jsp:expression></revision>
+      <compiled_at><jsp:expression>VersionInfo.getDate()</jsp:expression></compiled_at>
+      <compiled_by><jsp:expression>VersionInfo.getUser()</jsp:expression></compiled_by>
+      <identifier><jsp:expression>tracker.getTrackerIdentifier()</jsp:expression></identifier>
+    </tracker>
+
+    <cluster_summary>
+      <jsp:scriptlet>
+        jspHelper.generateSummaryTable(out, tracker);
+      </jsp:scriptlet>
+    </cluster_summary>
+
+    <running_jobs>
+      <jsp:scriptlet>
+        jspHelper.generateJobTable(out, "running", runningJobs);
+      </jsp:scriptlet>
+    </running_jobs>
+
+    <completed_jobs>
+      <jsp:scriptlet>
+        jspHelper.generateJobTable(out, "completed", completedJobs);
+      </jsp:scriptlet>
+    </completed_jobs>
+
+    <failed_jobs>
+      <jsp:scriptlet>
+        jspHelper.generateJobTable(out, "failed", failedJobs);
+      </jsp:scriptlet>
+    </failed_jobs>
+
+    <retired_jobs>
+      <jsp:scriptlet>
+        JSPUtil.generateRetiredJobXml(out, tracker,
+            runningJobs.size() + completedJobs.size() + failedJobs.size());
+      </jsp:scriptlet>
+    </retired_jobs>
+</cluster>



Mime
View raw message