hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1177531 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avr...
Date Fri, 30 Sep 2011 06:47:54 GMT
Author: acmurthy
Date: Fri Sep 30 06:47:53 2011
New Revision: 1177531

URL: http://svn.apache.org/viewvc?rev=1177531&view=rev
Log:
MAPREDUCE-2996. Add uber-ness information to JobHistory. Contributed by Jonathan Eagles.

Modified:
    hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
    hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1177531&r1=1177530&r2=1177531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Sep 30 06:47:53 2011
@@ -1481,6 +1481,9 @@ Release 0.23.0 - Unreleased
     MAPREDUCE-2791. Added missing info on 'job -status' output. (Devaraj K via
     acmurthy)
 
+    MAPREDUCE-2996. Add uber-ness information to JobHistory. (Jonathan Eagles
+    via acmurthy)
+
 Release 0.22.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1177531&r1=1177530&r2=1177531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
Fri Sep 30 06:47:53 2011
@@ -1135,7 +1135,8 @@ public class JobImpl implements org.apac
         new JobInitedEvent(job.oldJobId,
              job.startTime,
              job.numMapTasks, job.numReduceTasks,
-             job.getState().toString()); //Will transition to state running. Currently in
INITED
+             job.getState().toString(),
+             job.isUber()); //Will transition to state running. Currently in INITED
       job.eventHandler.handle(new JobHistoryEvent(job.jobId, jie));
       JobInfoChangeEvent jice = new JobInfoChangeEvent(job.oldJobId,
           job.submitTime, job.startTime);

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr?rev=1177531&r1=1177530&r2=1177531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
Fri Sep 30 06:47:53 2011
@@ -64,7 +64,8 @@
           {"name": "launchTime", "type": "long"},
           {"name": "totalMaps", "type": "int"},
           {"name": "totalReduces", "type": "int"},
-          {"name": "jobStatus", "type": "string"}
+          {"name": "jobStatus", "type": "string"},
+          {"name": "uberized", "type": "boolean"}
       ]
      },
 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1177531&r1=1177530&r2=1177531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
Fri Sep 30 06:47:53 2011
@@ -302,6 +302,7 @@ public class JobHistoryParser {
     info.launchTime = event.getLaunchTime();
     info.totalMaps = event.getTotalMaps();
     info.totalReduces = event.getTotalReduces();
+    info.uberized = event.getUberized();
   }
 
   private void handleJobInfoChangeEvent(JobInfoChangeEvent event) {
@@ -346,6 +347,7 @@ public class JobHistoryParser {
     Map<JobACL, AccessControlList> jobACLs;
     
     Map<TaskID, TaskInfo> tasksMap;
+    boolean uberized;
     
     /** Create a job info object where job information will be stored
      * after a parse
@@ -373,7 +375,8 @@ public class JobHistoryParser {
       System.out.println("MAP_COUNTERS:" + mapCounters.toString());
       System.out.println("REDUCE_COUNTERS:" + reduceCounters.toString());
       System.out.println("TOTAL_COUNTERS: " + totalCounters.toString());
-      
+      System.out.println("UBERIZED: " + uberized);
+
       for (TaskInfo ti: tasksMap.values()) {
         ti.printAll();
       }
@@ -421,6 +424,8 @@ public class JobHistoryParser {
     /** @return the priority of this job */
     public String getPriority() { return priority.toString(); }
     public Map<JobACL, AccessControlList> getJobACLs() { return jobACLs; }
+    /** @return the uberized status of this job */
+    public boolean getUberized() { return uberized; }
   }
   
   /**

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java?rev=1177531&r1=1177530&r2=1177531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
Fri Sep 30 06:47:53 2011
@@ -42,14 +42,16 @@ public class JobInitedEvent implements H
    * @param totalMaps
    * @param totalReduces
    * @param jobStatus
+   * @param uberized True if the job's map and reduce stages were combined
    */
   public JobInitedEvent(JobID id, long launchTime, int totalMaps,
-                        int totalReduces, String jobStatus) {
+                        int totalReduces, String jobStatus, boolean uberized) {
     datum.jobid = new Utf8(id.toString());
     datum.launchTime = launchTime;
     datum.totalMaps = totalMaps;
     datum.totalReduces = totalReduces;
     datum.jobStatus = new Utf8(jobStatus);
+    datum.uberized = uberized;
   }
 
   JobInitedEvent() { }
@@ -67,9 +69,10 @@ public class JobInitedEvent implements H
   public int getTotalReduces() { return datum.totalReduces; }
   /** Get the status */
   public String getStatus() { return datum.jobStatus.toString(); }
- /** Get the event type */
+  /** Get the event type */
   public EventType getEventType() {
     return EventType.JOB_INITED;
   }
-
+  /** Get whether the job's map and reduce stages were combined */
+  public boolean getUberized() { return datum.uberized; }
 }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1177531&r1=1177530&r2=1177531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
Fri Sep 30 06:47:53 2011
@@ -290,8 +290,7 @@ public class CompletedJob implements org
 
   @Override
   public boolean isUber() {
-    LOG.warn("isUber is not yet implemented");
-    return false;
+    return jobInfo.getUberized();
   }
 
   @Override

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java?rev=1177531&r1=1177530&r2=1177531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
Fri Sep 30 06:47:53 2011
@@ -95,6 +95,8 @@ public class TestJobHistoryParsing {
         2, jobInfo.getFinishedMaps());
     Assert.assertEquals("incorrect finishedReduces ",
         1, jobInfo.getFinishedReduces());
+    Assert.assertEquals("incorrect uberized ",
+        job.isUber(), jobInfo.getUberized());
     int totalTasks = jobInfo.getAllTasks().size();
     Assert.assertEquals("total number of tasks is incorrect  ", 3, totalTasks);
 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1177531&r1=1177530&r2=1177531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java
Fri Sep 30 06:47:53 2011
@@ -697,7 +697,8 @@ public class JobInProgress {
     JobInitedEvent jie = new JobInitedEvent(
         profile.getJobID(),  this.launchTime,
         numMapTasks, numReduceTasks,
-        JobStatus.getJobRunState(JobStatus.PREP));
+        JobStatus.getJobRunState(JobStatus.PREP),
+        false);
     
     jobHistory.logEvent(jie, jobId);
    

Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java?rev=1177531&r1=1177530&r2=1177531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
Fri Sep 30 06:47:53 2011
@@ -125,10 +125,12 @@ public class Job20LineHistoryEventEmitte
       String status = line.get("JOB_STATUS");
       String totalMaps = line.get("TOTAL_MAPS");
       String totalReduces = line.get("TOTAL_REDUCES");
+      String uberized = line.get("UBERIZED");
 
       if (launchTime != null && totalMaps != null && totalReduces != null)
{
         return new JobInitedEvent(jobID, Long.parseLong(launchTime), Integer
-            .parseInt(totalMaps), Integer.parseInt(totalReduces), status);
+            .parseInt(totalMaps), Integer.parseInt(totalReduces), status,
+            Boolean.parseBoolean(uberized));
       }
 
       return null;



Mime
View raw message