hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r579212 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/mapred/ src/webapps/job/
Date Tue, 25 Sep 2007 12:05:04 GMT
Author: ddas
Date: Tue Sep 25 05:05:02 2007
New Revision: 579212

URL: http://svn.apache.org/viewvc?rev=579212&view=rev
Log:
HADOOP-1921. Save the configuration of completed/failed jobs and make them available via the
web-ui.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    lucene/hadoop/trunk/src/webapps/job/jobconf.jsp
    lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp
    lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue Sep 25 05:05:02 2007
@@ -190,6 +190,9 @@
 
   IMPROVEMENTS
 
+    HADOOP-1921. Save the configuration of completed/failed jobs and make them
+    available via the web-ui. (Amar Kamat via devaraj)
+
     HADOOP-1266. Remove dependency of package org.apache.hadoop.net on 
     org.apache.hadoop.dfs.  (Hairong Kuang via dhruba)
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java Tue Sep 25 05:05:02
2007
@@ -316,23 +316,34 @@
     public Map<String, Task> getAllTasks() { return allTasks; }
     
     /**
+     * Get the path of the locally stored job file
+     * @param jobId id of the job
+     * @return the path of the job file on the local file system 
+     */
+    public static String getLocalJobFilePath(String jobId){
+      return LOG_DIR + File.separator + jobId + "_conf.xml";
+    }
+    
+    /**
      * Log job submitted event to history. Creates a new file in history 
      * for the job. if history file creation fails, it disables history 
      * for all other events. 
-     * @param jobId job id assigned by job tracker. 
-     * @param jobName job name as given by user in job conf
-     * @param user user name
+     * @param jobId job id assigned by job tracker.
+     * @param jobConf job conf of the job
+     * @param jobConfPath path to job conf xml file in HDFS.
      * @param submitTime time when job tracker received the job
-     * @param jobConf path to job conf xml file in HDFS. 
      */
-    public static void logSubmitted(String jobId, String jobName, String user, 
-                                    long submitTime, String jobConf){
-      
+    public static void logSubmitted(String jobId, JobConf jobConf, 
+                                    String jobConfPath, long submitTime) {
+      String jobName = jobConf.getJobName();
+      String user = jobConf.getUser(); 
       if (!disableHistory){
         synchronized(MASTER_INDEX_LOG_FILE){
           JobHistory.log(masterIndex, RecordTypes.Job, 
                          new Enum[]{Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME,
Keys.JOBCONF }, 
-                         new String[]{jobId, jobName, user, String.valueOf(submitTime), jobConf
});
+                         new String[]{jobId, jobName, user, 
+                                      String.valueOf(submitTime), jobConfPath}
+                        );
         }
         // setup the history log file for this job
         String logFileName =  JOBTRACKER_START_TIME + "_" + jobId; 
@@ -344,12 +355,27 @@
           // add to writer as well 
           JobHistory.log(writer, RecordTypes.Job, 
                          new Enum[]{Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME,
Keys.JOBCONF }, 
-                         new String[]{jobId, jobName, user, String.valueOf(submitTime) ,
jobConf}); 
+                         new String[]{jobId, jobName, user, 
+                                      String.valueOf(submitTime) , jobConfPath}
+                        ); 
              
         }catch(IOException e){
           LOG.error("Failed creating job history log file, disabling history", e);
           disableHistory = true; 
         }
+      }
+      /* Storing the job conf on the local file system */
+      String localJobFilePath =  JobInfo.getLocalJobFilePath(jobId); 
+      File localJobFile = new File(localJobFilePath);
+      try {
+        FileOutputStream jobOut = new FileOutputStream(localJobFile);
+        jobConf.write(jobOut);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Job conf for " + jobId + " stored at " 
+                    + localJobFile.getAbsolutePath());
+        }
+      } catch (IOException ioe) {
+        LOG.error("Failed to store job conf on the local filesystem ", ioe);
       }
     }
     /**

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Tue Sep 25 05:05:02
2007
@@ -159,9 +159,8 @@
     this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
     this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
         
-    JobHistory.JobInfo.logSubmitted(jobid, conf.getJobName(), conf.getUser(), 
-                                    System.currentTimeMillis(), 
-                                    jobFile.toString()); 
+    JobHistory.JobInfo.logSubmitted(jobid, conf, jobFile.toString(), 
+                                    System.currentTimeMillis()); 
         
     MetricsContext metricsContext = MetricsUtil.getContext("mapred");
     this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Tue Sep 25 05:05:02
2007
@@ -1832,6 +1832,14 @@
     }
   }
 
+  /**
+   * Get the localized job file path on the job trackers local file system
+   * @param jobId id of the job
+   * @return the path of the job conf file on the local file system
+   */
+  public static String getLocalJobFilePath(String jobId){
+    return JobHistory.JobInfo.getLocalJobFilePath(jobId);
+  }
   ////////////////////////////////////////////////////////////
   // main()
   ////////////////////////////////////////////////////////////

Modified: lucene/hadoop/trunk/src/webapps/job/jobconf.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobconf.jsp?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobconf.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobconf.jsp Tue Sep 25 05:05:02 2007
@@ -26,28 +26,12 @@
 <h2>Job Configuration: JobId - <%= jobId %></h2><br>
 
 <%
-  JobInProgress job = (JobInProgress)tracker.getJob(jobId);
-  if (job == null) {
-    out.print("<h4>Job '" + jobId + "' not found!</h4><br>\n");
-    return;
-  }
-  
-  JobStatus status = job.getStatus();
-  int runState = status.getRunState();
-  if (runState != JobStatus.RUNNING) {
-    out.print("<h4>Job '" + jobId + "' not running!</h4><br>\n");
-    return;
-  }
-  
+  String jobFilePath = tracker.getLocalJobFilePath(jobId);
   try {
-    JobConf jobConf = job.getJobConf();
-    ByteArrayOutputStream jobConfXML = new ByteArrayOutputStream();
-    jobConf.write(jobConfXML);
+    JobConf jobConf = new JobConf(jobFilePath);
     XMLUtils.transform(
         jobConf.getConfResourceAsInputStream("webapps/static/jobconf.xsl"),
-	    new ByteArrayInputStream(jobConfXML.toByteArray()), 
-	    out
-	  );
+        new FileInputStream(jobFilePath), out);
   } catch (Exception e) {
     out.println("Failed to retreive job configuration for job '" + jobId + "!");
     out.println(e);

Modified: lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp Tue Sep 25 05:05:02 2007
@@ -146,12 +146,8 @@
     int flakyTaskTrackers = job.getNoOfBlackListedTrackers();
     out.print("<b>User:</b> " + profile.getUser() + "<br>\n");
     out.print("<b>Job Name:</b> " + profile.getJobName() + "<br>\n");
-    if (runState == JobStatus.RUNNING) {
-      out.print("<b>Job File:</b> <a href=\"jobconf.jsp?jobid=" + jobId +
"\">" + 
-          profile.getJobFile() + "</a><br>\n");
-    } else {
-      out.print("<b>Job File:</b> " + profile.getJobFile() + "<br>\n");
-    }
+    out.print("<b>Job File:</b> <a href=\"jobconf.jsp?jobid=" + jobId + "\">"

+              + profile.getJobFile() + "</a><br>\n");
     if (runState == JobStatus.RUNNING) {
       out.print("<b>Status:</b> Running<br>\n");
       out.print("<b>Started at:</b> " + new Date(job.getStartTime()) + "<br>\n");

Modified: lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp Tue Sep 25 05:05:02 2007
@@ -24,7 +24,8 @@
 
 <b>User: </b> <%=job.get(Keys.USER) %><br/> 
 <b>JobName: </b> <%=job.get(Keys.JOBNAME) %><br/> 
-<b>JobConf: </b> <%=job.get(Keys.JOBCONF) %><br/> 
+<b>JobConf: </b> <a href="jobconf.jsp?jobid=<%=jobid %>"> 
+                 <%=job.get(Keys.JOBCONF) %></a><br/> 
 <b>Submitted At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.SUBMIT_TIME), 0 )  %><br/> 
 <b>Launched At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.LAUNCH_TIME),
job.getLong(Keys.SUBMIT_TIME)) %><br/>
 <b>Finished At: </b>  <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME)) %><br/>



Mime
View raw message