hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject svn commit: r1388593 - in /hadoop/common/branches/MR-3902/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/a...
Date Fri, 21 Sep 2012 18:09:02 GMT
Author: sseth
Date: Fri Sep 21 18:09:01 2012
New Revision: 1388593

URL: http://svn.apache.org/viewvc?rev=1388593&view=rev
Log:
MAPREDUCE-4665. Use the configured shuffle port and application ACLs (sseth)

Modified:
    hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
    hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl2.java
    hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HeartbeatHandlerBase.java
    hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/AppContext.java
    hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java
    hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/MRApp.java
    hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/TestRuntimeEstimators.java

Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902?rev=1388593&r1=1388592&r2=1388593&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902 (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt.MR-3902 Fri Sep 21
18:09:01 2012
@@ -20,3 +20,5 @@ Branch MR-3902
   MAPREDUCE-4617. Re-wire AM Recovery (sseth)
 
   MAPREDUCE-4618. Re-wire LocalContainerAllocator/UberAM (sseth)
+
+  MAPREDUCE-4665. Use the configured shuffle port and application ACLs (sseth)

Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl2.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl2.java?rev=1388593&r1=1388592&r2=1388593&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl2.java
(original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl2.java
Fri Sep 21 18:09:01 2012
@@ -74,12 +74,11 @@ public class TaskAttemptListenerImpl2 ex
   // .... TaskAttemptId registration and unregistration by the TaskAttempt. Can this be split
into a 
   // ContainerListener + TaskAttemptListener ?
   
-  // TODO XXX. Re-look at big chungs. Possibly redo bits.
+  // TODO XXX. Re-look at big chunks. Possibly redo bits.
   // ..launchedJvm map etc.
   // ..Sending back errors for unknown tasks.
   
   private static final JvmTask TASK_FOR_INVALID_JVM = new JvmTask(null, true);
-  private static final JvmTask UNASSIGNED_TASK = new JvmTask(null, false);
 
   private static final Log LOG = LogFactory.getLog(TaskAttemptListenerImpl2.class);
 
@@ -444,9 +443,10 @@ public class TaskAttemptListenerImpl2 ex
         jvmTask = null;
       } else {
         TaskAttemptId yTaskAttemptId = TypeConverter.toYarn(task.getTaskID());
-        // TODO XXX: Generate this event properly - proper params etc etc etc.s
-        // TODO XXX: Fix the hardcoded port.
-        context.getEventHandler().handle(new TaskAttemptRemoteStartEvent(yTaskAttemptId,
containerId, null, 8080));
+        context.getEventHandler().handle(
+            new TaskAttemptRemoteStartEvent(yTaskAttemptId, containerId,
+                context.getApplicationACLs(), context.getAllContainers()
+                    .get(containerId).getShufflePort()));
         LOG.info("JVM with ID: " + jvmId + " given task: " + task.getTaskID());
         registerTaskAttempt(yTaskAttemptId, wJvmID);
         jvmTask = new JvmTask(task, false);

Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HeartbeatHandlerBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HeartbeatHandlerBase.java?rev=1388593&r1=1388592&r2=1388593&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HeartbeatHandlerBase.java
(original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HeartbeatHandlerBase.java
Fri Sep 21 18:09:01 2012
@@ -33,6 +33,7 @@ public abstract class HeartbeatHandlerBa
     this.eventHandler = appContext.getEventHandler();
     this.clock = appContext.getClock();
     this.appContext = appContext;
+    numThreads = numThreads == 0 ? 1 : numThreads;
     this.runningMap = new ConcurrentHashMap<T, HeartbeatHandlerBase.ReportTime>(
         16, 0.75f, numThreads);
   }
@@ -40,7 +41,6 @@ public abstract class HeartbeatHandlerBa
   @Override
   public void init(Configuration conf) {
     super.init(conf);
-    // TODO XXX: TaskTimeout / ContainerTimeOut
     timeOut = getConfiguredTimeout(conf);
     timeOutCheckInterval = getConfiguredTimeoutCheckInterval(conf);
   }

Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/AppContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/AppContext.java?rev=1388593&r1=1388592&r2=1388593&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/AppContext.java
(original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/AppContext.java
Fri Sep 21 18:09:01 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.mapreduce.v2.app2.rm.node.AMNodeMap;
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.ClusterInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -47,6 +48,8 @@ public interface AppContext {
   ApplicationAttemptId getApplicationAttemptId();
 
   String getApplicationName();
+  
+  Map<ApplicationAccessType, String> getApplicationACLs();
 
   long getStartTime();
 

Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java?rev=1388593&r1=1388592&r2=1388593&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java
(original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/main/java/org/apache/hadoop/mapreduce/v2/app2/MRAppMaster.java
Fri Sep 21 18:09:01 2012
@@ -107,6 +107,7 @@ import org.apache.hadoop.yarn.SystemCloc
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -897,6 +898,15 @@ public class MRAppMaster extends Composi
     public AMNodeMap getAllNodes() {
       return nodes;
     }
+
+    @Override
+    public Map<ApplicationAccessType, String> getApplicationACLs() {
+      if (getServiceState() != STATE.STARTED) {
+        throw new YarnException(
+            "Cannot get ApplicationACLs before all services have started");
+      }
+      return ((RMCommunicator) containerRequestor).getApplicationAcls();
+    }
   }
 
   @SuppressWarnings("unchecked")

Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/MRApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/MRApp.java?rev=1388593&r1=1388592&r2=1388593&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/MRApp.java
(original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/MRApp.java
Fri Sep 21 18:09:01 2012
@@ -108,6 +108,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.service.Service;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
@@ -598,9 +599,16 @@ public class MRApp extends MRAppMaster {
     return new MRAppAMScheduler();
   }
 
-  protected class MRAppAMScheduler implements ContainerAllocator {
+  protected class MRAppAMScheduler extends AbstractService implements ContainerAllocator{
     private int containerCount;
     
+    MRAppAMScheduler() {
+      super(MRAppAMScheduler.class.getSimpleName());
+    }
+    
+    public void start() {}
+    public void init(Configuration conf) {}
+    public void stop() {}
     
     @Override
     public void handle(AMSchedulerEvent rawEvent) {

Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/TestRuntimeEstimators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/TestRuntimeEstimators.java?rev=1388593&r1=1388592&r2=1388593&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/TestRuntimeEstimators.java
(original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/TestRuntimeEstimators.java
Fri Sep 21 18:09:01 2012
@@ -65,6 +65,7 @@ import org.apache.hadoop.security.author
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.ClusterInfo;
 import org.apache.hadoop.yarn.SystemClock;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -862,5 +863,10 @@ public class TestRuntimeEstimators {
       // TODO Auto-generated method stub
       return null;
     }
+
+    @Override
+    public Map<ApplicationAccessType, String> getApplicationACLs() {
+      throw new UnsupportedOperationException("Not supported yet.");
+    }
   }
 }



Mime
View raw message