hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1166968 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server...
Date Fri, 09 Sep 2011 02:19:24 GMT
Author: acmurthy
Date: Fri Sep  9 02:19:24 2011
New Revision: 1166968

URL: http://svn.apache.org/viewvc?rev=1166968&view=rev
Log:
MAPREDUCE-2953. Fix a race condition on submission which caused client to incorrectly assume
application was gone by making submission synchronous for RMAppManager. Contributed by Thomas
Graves.  

Modified:
    hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1166968&r1=1166967&r2=1166968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Sep  9 02:19:24 2011
@@ -1247,6 +1247,10 @@ Release 0.23.0 - Unreleased
    MAPREDUCE-2937. Ensure reason for application failure is displayed to the
    user. (mahadev via acmurthy) 
 
+   MAPREDUCE-2953. Fix a race condition on submission which caused client to 
+   incorrectly assume application was gone by making submission synchronous
+   for RMAppManager. (Thomas Graves via acmurthy) 
+
 Release 0.22.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1166968&r1=1166967&r2=1166968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
Fri Sep  9 02:19:24 2011
@@ -91,6 +91,7 @@ public class ClientRMService extends Abs
   final private YarnScheduler scheduler;
   final private RMContext rmContext;
   private final AMLivelinessMonitor amLivelinessMonitor;
+  private final RMAppManager rmAppManager;
 
   private String clientServiceBindAddress;
   private Server server;
@@ -100,11 +101,13 @@ public class ClientRMService extends Abs
   private  ApplicationACLsManager aclsManager;
   private Map<ApplicationACL, AccessControlList> applicationACLs;
   
-  public ClientRMService(RMContext rmContext, YarnScheduler scheduler) {
+  public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
+      RMAppManager rmAppManager) {
     super(ClientRMService.class.getName());
     this.scheduler = scheduler;
     this.rmContext = rmContext;
     this.amLivelinessMonitor = rmContext.getAMLivelinessMonitor();
+    this.rmAppManager = rmAppManager;
   }
   
   @Override
@@ -201,8 +204,10 @@ public class ClientRMService extends Abs
         throw new IOException("Application with id " + applicationId
             + " is already present! Cannot add a duplicate!");
       }
-      this.rmContext.getDispatcher().getEventHandler().handle(
-          new RMAppManagerSubmitEvent(submissionContext));
+      // This needs to be synchronous as the client can query 
+      // immediately following the submission to get the application status.
+      // So call handle directly and do not send an event.
+      rmAppManager.handle(new RMAppManagerSubmitEvent(submissionContext));
 
       LOG.info("Application with id " + applicationId.getId() + 
           " submitted by user " + user + " with " + submissionContext);

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1166968&r1=1166967&r2=1166968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
Fri Sep  9 02:19:24 2011
@@ -210,7 +210,7 @@ public class RMAppManager implements Eve
     }
   }
 
-  protected void submitApplication(ApplicationSubmissionContext submissionContext) {
+  protected synchronized void submitApplication(ApplicationSubmissionContext submissionContext)
{
     ApplicationId applicationId = submissionContext.getApplicationId();
     RMApp application = null;
     try {

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1166968&r1=1166967&r2=1166968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Fri Sep  9 02:19:24 2011
@@ -99,7 +99,7 @@ public class ResourceManager extends Com
   protected NMLivelinessMonitor nmLivelinessMonitor;
   protected NodesListManager nodesListManager;
   private SchedulerEventDispatcher schedulerDispatcher;
-  private RMAppManager rmAppManager;
+  protected RMAppManager rmAppManager;
 
   private final AtomicBoolean shutdown = new AtomicBoolean(false);
   private WebApp webApp;
@@ -176,13 +176,13 @@ public class ResourceManager extends Com
     masterService = createApplicationMasterService();
     addService(masterService) ;
 
-    clientRM = createClientRMService();
-    addService(clientRM);
-
     this.rmAppManager = createRMAppManager();
     // Register event handler for RMAppManagerEvents
     this.rmDispatcher.register(RMAppManagerEventType.class,
         this.rmAppManager);
+
+    clientRM = createClientRMService();
+    addService(clientRM);
     
     adminService = createAdminService();
     addService(adminService);
@@ -441,7 +441,7 @@ public class ResourceManager extends Com
   }
 
   protected ClientRMService createClientRMService() {
-    return new ClientRMService(this.rmContext, scheduler);
+    return new ClientRMService(this.rmContext, scheduler, this.rmAppManager);
   }
 
   protected ApplicationMasterService createApplicationMasterService() {

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1166968&r1=1166967&r2=1166968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
Fri Sep  9 02:19:24 2011
@@ -60,13 +60,9 @@ public class MockRM extends ResourceMana
 
   public void waitForState(ApplicationId appId, RMAppState finalState) 
       throws Exception {
+    RMApp app = getRMContext().getRMApps().get(appId);
+    Assert.assertNotNull("app shouldn't be null", app);
     int timeoutSecs = 0;
-    RMApp app = null;
-    while ((app == null) && timeoutSecs++ < 20) {
-      app = getRMContext().getRMApps().get(appId);
-      Thread.sleep(500);
-    }
-    timeoutSecs = 0;
     while (!finalState.equals(app.getState()) &&
         timeoutSecs++ < 20) {
       System.out.println("App State is : " + app.getState() +
@@ -95,6 +91,7 @@ public class MockRM extends ResourceMana
     req.setApplicationSubmissionContext(sub);
     
     client.submitApplication(req);
+    // make sure app is immediately available after submit
     waitForState(appId, RMAppState.ACCEPTED);
     return getRMContext().getRMApps().get(appId);
   }
@@ -131,7 +128,7 @@ public class MockRM extends ResourceMana
 
   @Override
   protected ClientRMService createClientRMService() {
-    return new ClientRMService(getRMContext(), getResourceScheduler()) {
+    return new ClientRMService(getRMContext(), getResourceScheduler(), rmAppManager) {
       @Override
       public void start() {
         //override to not start rpc handler



Mime
View raw message