hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r388288 - in /lucene/hadoop/trunk/src: examples/org/apache/hadoop/examples/ java/org/apache/hadoop/mapred/ webapps/mapred/
Date Thu, 23 Mar 2006 22:52:47 GMT
Author: cutting
Date: Thu Mar 23 14:52:44 2006
New Revision: 388288

URL: http://svn.apache.org/viewcvs?rev=388288&view=rev
Log:
Fix for HADOOP-46.  Jobs can now be named.  Contributed by Owen O'Malley.

Modified:
    lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java
    lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
    lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp

Modified: lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java (original)
+++ lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java Thu Mar 23 14:52:44
2006
@@ -17,18 +17,15 @@
 
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 
 import org.apache.hadoop.mapred.lib.RegexMapper;
 import org.apache.hadoop.mapred.lib.InverseMapper;
 import org.apache.hadoop.mapred.lib.LongSumReducer;
-import org.apache.hadoop.mapred.lib.IdentityReducer;
 
 import org.apache.hadoop.io.UTF8;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.WritableComparator;
 
 import org.apache.hadoop.conf.Configuration;
 
@@ -52,6 +49,7 @@
                Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
 
     JobConf grepJob = new JobConf(defaults, Grep.class);
+    grepJob.setJobName("grep-search");
 
     grepJob.setInputDir(new File(args[0]));
 
@@ -71,6 +69,7 @@
     JobClient.runJob(grepJob);
 
     JobConf sortJob = new JobConf(defaults, Grep.class);
+    sortJob.setJobName("grep-sort");
 
     sortJob.setInputDir(tempDir);
     sortJob.setInputFormat(SequenceFileInputFormat.class);

Modified: lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java (original)
+++ lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java Thu Mar 23
14:52:44 2006
@@ -17,7 +17,6 @@
 package org.apache.hadoop.examples;
 
 import java.io.*;
-import java.net.URL;
 import java.util.*;
 
 import org.apache.hadoop.conf.Configuration;
@@ -110,24 +109,25 @@
   public static void main(String[] args) throws IOException {
     Configuration defaults = new Configuration();
     
-    JobConf countJob = new JobConf(defaults, WordCount.class);
+    JobConf conf = new JobConf(defaults, WordCount.class);
+    conf.setJobName("wordcount");
  
     // the keys are words (strings)
-    countJob.setOutputKeyClass(UTF8.class);
+    conf.setOutputKeyClass(UTF8.class);
     // the values are counts (ints)
-    countJob.setOutputValueClass(IntWritable.class);
+    conf.setOutputValueClass(IntWritable.class);
     
-    countJob.setMapperClass(MapClass.class);        
-    countJob.setCombinerClass(Reduce.class);
-    countJob.setReducerClass(Reduce.class);
+    conf.setMapperClass(MapClass.class);        
+    conf.setCombinerClass(Reduce.class);
+    conf.setReducerClass(Reduce.class);
     
     List other_args = new ArrayList();
     for(int i=0; i < args.length; ++i) {
       try {
         if ("-m".equals(args[i])) {
-          countJob.setNumMapTasks(Integer.parseInt(args[++i]));
+          conf.setNumMapTasks(Integer.parseInt(args[++i]));
         } else if ("-r".equals(args[i])) {
-          countJob.setNumReduceTasks(Integer.parseInt(args[++i]));
+          conf.setNumReduceTasks(Integer.parseInt(args[++i]));
         } else {
           other_args.add(args[i]);
         }
@@ -146,13 +146,13 @@
           other_args.size() + " instead of 2.");
       printUsage();
     }
-    countJob.setInputDir(new File((String) other_args.get(0)));
-    countJob.setOutputDir(new File((String) other_args.get(1)));
+    conf.setInputDir(new File((String) other_args.get(0)));
+    conf.setOutputDir(new File((String) other_args.get(1)));
     
     // Uncomment to run locally in a single process
     // countJob.set("mapred.job.tracker", "local");
     
-    JobClient.runJob(countJob);
+    JobClient.runJob(conf);
   }
   
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Thu Mar 23 14:52:44
2006
@@ -302,6 +302,23 @@
   public int getNumReduceTasks() { return getInt("mapred.reduce.tasks", 1); }
   public void setNumReduceTasks(int n) { setInt("mapred.reduce.tasks", n); }
 
+  /**
+   * Get the user-specified job name. This is only used to identify the 
+   * job to the user.
+   * @return the job's name, defaulting to ""
+   */
+  public String getJobName() {
+    return get("mapred.job.name", "");
+  }
+  
+  /**
+   * Set the user-specified job name.
+   * @param name the job's new name
+   */
+  public void setJobName(String name) {
+    set("mapred.job.name", name);
+  }
+  
   public Object newInstance(Class theClass) {
     Object result;
     try {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Thu Mar 23 14:52:44
2006
@@ -74,8 +74,8 @@
         fs.copyToLocalFile(new File(jobFile), localJobFile);
 
         conf = new JobConf(localJobFile);
-        this.profile = new JobProfile(conf.getUser(), jobid, jobFile, url);
-
+        this.profile = new JobProfile(conf.getUser(), jobid, jobFile, url,
+                                      conf.getJobName());
         String jarFile = conf.getJar();
         if (jarFile != null) {
           fs.copyToLocalFile(new File(jarFile), localJarFile);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java Thu Mar 23 14:52:44
2006
@@ -40,6 +40,7 @@
     String jobid;
     String jobFile;
     String url;
+    String name;
 
     /**
      */
@@ -48,11 +49,13 @@
 
     /**
      */
-    public JobProfile(String user, String jobid, String jobFile, String url) {
+    public JobProfile(String user, String jobid, String jobFile, String url,
+                      String name) {
         this.user = user;
         this.jobid = jobid;
         this.jobFile = jobFile;
         this.url = url;
+        this.name = name;
     }
 
     /**
@@ -85,6 +88,13 @@
         }
     }
 
+    /**
+     * Get the user-specified job name.
+     */
+    public String getJobName() {
+      return name;
+    }
+    
     ///////////////////////////////////////
     // Writable
     ///////////////////////////////////////
@@ -93,12 +103,14 @@
         UTF8.writeString(out, jobFile);
         UTF8.writeString(out, url);
         UTF8.writeString(out, user);
+        UTF8.writeString(out, name);
     }
     public void readFields(DataInput in) throws IOException {
         this.jobid = UTF8.readString(in);
         this.jobFile = UTF8.readString(in);
         this.url = UTF8.readString(in);
         this.user = UTF8.readString(in);
+        this.name = UTF8.readString(in);
     }
 }
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Thu Mar 23 14:52:44
2006
@@ -56,7 +56,7 @@
       fs.copyToLocalFile(new File(file), localFile);
       this.job = new JobConf(localFile);
       profile = new JobProfile(job.getUser(), id, file, 
-                               "http://localhost:8080/");
+                               "http://localhost:8080/", job.getJobName());
       this.status.jobid = id;
       this.status.runState = JobStatus.RUNNING;
 

Modified: lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp Thu Mar 23 14:52:44 2006
@@ -48,6 +48,7 @@
       out.print("<tr><td align=\"center\" colspan=\"8\"><b>" + label +
" Jobs </b></td></tr>\n");
       if (jobs.size() > 0) {
         out.print("<tr><td><b>Jobid</b></td><td><b>User</b></td>");
+        out.print("<td><b>Name</b></td>");
         out.print("<td><b>% complete</b></td><td><b>Required
maps</b></td>");
         out.print("<td><b>maps completed</b></td>");
         out.print("<td><b>Required reduces</b></td>");
@@ -64,9 +65,11 @@
           int desiredReduces = job.desiredReduces();
           int completedMaps = job.finishedMaps();
           int completedReduces = job.finishedReduces();
+          String name = profile.getJobName();
 
           out.print("<tr><td><a href=\"jobdetails.jsp?jobid=" + jobid + "\">"
+ 
                     jobid + "</a></td><td>"+ profile.getUser() + "</td><td>"
+
+                    ("".equals(name) ? "&nbsp;" : name) + "</td><td>" +
                     percentFormat.format(completedRatio) + "%</td><td>" + 
                     desiredMaps + "</td><td>" + completedMaps + "</td><td>"
+ 
                     desiredReduces + "</td><td> " + completedReduces + 



Mime
View raw message