hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r1092803 - in /hadoop/mapreduce/branches/MR-279: ./ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ mr-client/had...
Date Fri, 15 Apr 2011 20:39:36 GMT
Author: mahadev
Date: Fri Apr 15 20:39:35 2011
New Revision: 1092803

URL: http://svn.apache.org/viewvc?rev=1092803&view=rev
Log:
MAPREDUCE-2424. Polish uber-AppMaster: add uber-AM counters and GUI indicators.  (Greg Roelofs
via mahadev)

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri Apr 15 20:39:35 2011
@@ -16,6 +16,13 @@ Trunk (unreleased changes)
   
    Fix YarnRemoteException to give more details. (siddharth seth via mahadev)
 
+    MAPREDUCE-2405. Implement uber-AppMaster (in-cluster LocalJobRunner for
+    MRv2). (Greg Roelofs and Sharad Agarwal via mahadev)
+
+    MAPREDUCE-2424. Polish uber-AppMaster: add uber-AM counters and GUI
+    indicators.  (Greg Roelofs via mahadev)
+
+  
   INCOMPATIBLE CHANGES
 
     MAPREDUCE-1866. Removes deprecated class
@@ -431,7 +438,7 @@ Release 0.21.0 - 2010-08-13
   NEW FEATURES
 
     MAPREDUCE-1774. Large-scale Automated Framework (Sharad Agarwal, Sreekanth
-    Ramakrishnan, Konstantin Boudnik, at all via cos)
+    Ramakrishnan, Konstantin Boudnik, et al. via cos)
 
     MAPREDUCE-706. Support for FIFO pools in the fair scheduler.
     (Matei Zaharia)

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
Fri Apr 15 20:39:35 2011
@@ -138,7 +138,7 @@ public class LocalContainerLauncher exte
    *     - creates "remoteTask" for us == real Task
    *     - sends CONTAINER_REMOTE_LAUNCH
    *     - TA: UNASSIGNED -> ASSIGNED
-   * - CONTAINER_REMOTE_LAUNCH handled by LocalContainerLauncher (us!)
+   * - CONTAINER_REMOTE_LAUNCH handled by LocalContainerLauncher (us)
    *   - sucks "remoteTask" out of TaskAttemptImpl via getRemoteTask()
    *   - sends TA_CONTAINER_LAUNCHED
    *     [[ elsewhere...
@@ -251,6 +251,13 @@ public class LocalContainerLauncher exte
       try {
         JobConf conf = new JobConf(getConfig());
 
+        // mark this as an uberized subtask so it can set task counter
+        // (longer-term/FIXME:  could redefine as job counter and send
+        // "JobCounterEvent" to JobImpl on [successful] completion of subtask;
+        // will need new Job state-machine transition and JobImpl jobCounters
+        // map to handle)
+        conf.setBoolean("mapreduce.task.uberized", true);
+
         // META-FIXME: do we want the extra sanity-checking (doneWithMaps,
         // etc.), or just assume/hope the state machine(s) and uber-AM work
         // as expected?

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
Fri Apr 15 20:39:35 2011
@@ -122,7 +122,8 @@ public class JobImpl implements org.apac
   // Maximum no. of fetch-failure notifications after which map task is failed
   private static final int MAX_FETCH_FAILURES_NOTIFICATIONS = 3;
 
-  private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+  private final RecordFactory recordFactory =
+      RecordFactoryProvider.getRecordFactory(null);
   
   //final fields
   private final Clock clock;
@@ -131,14 +132,25 @@ public class JobImpl implements org.apac
   private final JobId jobId;
   private final org.apache.hadoop.mapreduce.JobID oldJobId;
   private final TaskAttemptListener taskAttemptListener;
-  private boolean lazyTasksCopyNeeded = false;
   private final Object tasksSyncHandle = new Object();
-  private volatile Map<TaskId, Task> tasks = new LinkedHashMap<TaskId, Task>();
   private final Set<TaskId> mapTasks = new LinkedHashSet<TaskId>();
   private final Set<TaskId> reduceTasks = new LinkedHashSet<TaskId>();
   private final EventHandler eventHandler;
+
+  private boolean lazyTasksCopyNeeded = false;
+  private volatile Map<TaskId, Task> tasks = new LinkedHashMap<TaskId, Task>();
+  private Counters jobCounters = newCounters();
+    // FIXME:  support job-level counters
+    //
+    // Presumably want to define new event type that job-related entities
+    // (e.g., MRAppMaster or LocalContainerLauncher) can emit with some sort
+    // of payload (maybe just Counters?); then define new Job state-machine
+    // transition to handle the event and update jobCounters with payload data.
+    // Can then replace task-level uber counters (MR-2424) with job-level ones
+    // sent from LocalContainerLauncher, and eventually including a count of
+    // of uber-AM attempts (probably sent from MRAppMaster).
   public Configuration conf;
-  
+
   //fields initialized in init
   private FileSystem fs;
   private Path remoteJobSubmitDir;
@@ -404,9 +416,10 @@ public class JobImpl implements org.apac
   @Override
   public Counters getCounters() {
     Counters counters = newCounters();
-    // TODO: compute job level counters
+    // TODO: compute job-level counters
     readLock.lock();
     try {
+      incrAllCounters(counters, jobCounters);
       return incrTaskCounters(counters, tasks.values());
     } finally {
       readLock.unlock();
@@ -414,7 +427,8 @@ public class JobImpl implements org.apac
   }
 
   public static Counters newCounters() {
-    Counters counters = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Counters.class);
+    Counters counters = RecordFactoryProvider.getRecordFactory(null)
+        .newRecordInstance(Counters.class);
 //    counters.groups = new HashMap<String, CounterGroup>();
     return counters;
   }
@@ -432,7 +446,8 @@ public class JobImpl implements org.apac
       for (CounterGroup otherGroup: other.getAllCounterGroups().values()) {
         CounterGroup group = counters.getCounterGroup(otherGroup.getName());
         if (group == null) {
-          group = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(CounterGroup.class);
+          group = RecordFactoryProvider.getRecordFactory(null)
+              .newRecordInstance(CounterGroup.class);
 //          group.counters = new HashMap<CharSequence, Counter>();
           group.setName(otherGroup.getName());
           counters.setCounterGroup(group.getName(), group);
@@ -441,7 +456,8 @@ public class JobImpl implements org.apac
         for (Counter otherCounter : otherGroup.getAllCounters().values()) {
           Counter counter = group.getCounter(otherCounter.getName());
           if (counter == null) {
-            counter = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Counter.class);
+            counter = RecordFactoryProvider.getRecordFactory(null)
+                .newRecordInstance(Counter.class);
             counter.setName(otherCounter.getName());
             group.setCounter(counter.getName(), counter);
           }

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
Fri Apr 15 20:39:35 2011
@@ -72,6 +72,7 @@ public class JobBlock extends HtmlBlock 
     info("Job Overview").
         _("Job Name:", job.getName()).
         _("State:", job.getState()).
+        _("Uberized:", job.isUber()).
         _("Started:", new Date(jobReport.getStartTime())).
         _("Elapsed:", StringUtils.formatTime(System.currentTimeMillis()
                                              - jobReport.getStartTime()));

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
Fri Apr 15 20:39:35 2011
@@ -325,6 +325,11 @@ class MapTask extends Task {
     boolean useNewApi = job.getUseNewMapper();
     initialize(job, getJobID(), reporter, useNewApi);
 
+    if (job.getBoolean("mapreduce.task.uberized", false)) {
+      reporter.getCounter(TaskCounter.NUM_UBER_AM_MAPS).increment(1);
+      // or getCounters().findCounter( ... ).increment(1);
+    }
+
     // check if it is a cleanupJobTask
     if (jobCleanup) {
       runJobCleanupTask(umbilical, reporter);

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
Fri Apr 15 20:39:35 2011
@@ -367,6 +367,11 @@ public class ReduceTask extends Task {
     boolean useNewApi = job.getUseNewReducer();
     initialize(job, getJobID(), reporter, useNewApi);
 
+    if (job.getBoolean("mapreduce.task.uberized", false)) {
+      reporter.getCounter(TaskCounter.NUM_UBER_AM_REDUCES).increment(1);
+      // or getCounters().findCounter( ... ).increment(1);
+    }
+
     // check if it is a cleanupJobTask
     if (jobCleanup) {
       runJobCleanupTask(umbilical, reporter);

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java
Fri Apr 15 20:39:35 2011
@@ -42,6 +42,8 @@ public enum TaskCounter {
   SHUFFLED_MAPS, 
   FAILED_SHUFFLE,
   MERGED_MAP_OUTPUTS,
+  NUM_UBER_AM_MAPS,
+  NUM_UBER_AM_REDUCES,
   GC_TIME_MILLIS,
   CPU_MILLISECONDS,
   PHYSICAL_MEMORY_BYTES,

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties
Fri Apr 15 20:39:35 2011
@@ -31,6 +31,8 @@ SPILLED_RECORDS.name=          Spilled R
 SHUFFLED_MAPS.name=            Shuffled Maps 
 FAILED_SHUFFLE.name=           Failed Shuffles
 MERGED_MAP_OUTPUTS.name=       Merged Map outputs
+NUM_UBER_AM_MAPS.name=         Uberized sub-MapTasks
+NUM_UBER_AM_REDUCES.name=      Uberized sub-ReduceTasks
 GC_TIME_MILLIS.name=           GC time elapsed (ms)
 CPU_MILLISECONDS.name=         CPU time spent (ms)
 PHYSICAL_MEMORY_BYTES.name=    Physical memory (bytes) snapshot

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Fri Apr 15 20:39:35 2011
@@ -265,7 +265,7 @@ public class YARNRunner implements Clien
     appContext.setApplicationId(applicationId);
     Resource capability = recordFactory.newRecordInstance(Resource.class);
     capability.setMemory(conf.getInt(YARN_AM_VMEM_MB, DEFAULT_YARN_AM_VMEM_MB));
-    LOG.info("Master capability = " + capability);
+    LOG.info("AppMaster capability = " + capability);
     appContext.setMasterCapability(capability);
 
     FileContext defaultFS = FileContext.getFileContext(conf);



Mime
View raw message