hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r1098425 - in /hadoop/mapreduce/branches/MR-279: ./ yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/...
Date Sun, 01 May 2011 22:35:52 GMT
Author: mahadev
Date: Sun May  1 22:35:51 2011
New Revision: 1098425

URL: http://svn.apache.org/viewvc?rev=1098425&view=rev
Log:
Fix to send finish application event only when the application is finished (mahadev)

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Sun May  1 22:35:51 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
 
   MAPREDUCE-279
 
+    Fix to send finish application event only when the application is finished (mahadev)
+
     RM Restart Phase 2 - Completed the recovery of components in the RM (mahadev)
 
     MAPREDUCE-2434. Metrics for ResourceManager. (Luke Lu via acmurthy)

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
Sun May  1 22:35:51 2011
@@ -197,6 +197,8 @@ AMRMProtocol, EventHandler<ASMEvent<Appl
       case REMOVE:
         responseMap.remove(id);
         break;
+      case EXPIRE:
+        responseMap.remove(id);
       default: 
         break;
       }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
Sun May  1 22:35:51 2011
@@ -288,7 +288,7 @@ public class ApplicationMasterInfo imple
       masterInfo.handler.handle(new ASMEvent<AMLauncherEventType>(
         AMLauncherEventType.CLEANUP, masterInfo));
       masterInfo.handler.handle(new ASMEvent<ApplicationTrackerEventType>(
-      ApplicationTrackerEventType.REMOVE, masterInfo));
+      ApplicationTrackerEventType.EXPIRE, masterInfo));
       masterInfo.numFailed++;
     }
   }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java
Sun May  1 22:35:51 2011
@@ -41,7 +41,8 @@ public class ApplicationMasterEvents {
   /* event generated for components tracking application adding/tracking/removal */
   public enum ApplicationTrackerEventType {
     ADD,
-    REMOVE
+    REMOVE,
+    EXPIRE
   }
   
   public enum ApplicationEventType {

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
Sun May  1 22:35:51 2011
@@ -63,8 +63,10 @@ public interface YarnScheduler {
   /**
    * A submitted application has completed.
    * @param applicationId completed application
+   * @param finishApplication true if the application is completed and the
+   * scheduler needs to notify other components of application completion.
    */
-  public void removeApplication(ApplicationId applicationId)
+  public void removeApplication(ApplicationId applicationId, boolean finishApplication)
   throws IOException;
 
 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Sun May  1 22:35:51 2011
@@ -252,7 +252,7 @@ implements ResourceScheduler, CapacitySc
   }
 
   @Override
-  public void removeApplication(ApplicationId applicationId)
+  public void removeApplication(ApplicationId applicationId, boolean finishApplication)
   throws IOException {
     Application application = getApplication(applicationId);
 
@@ -270,11 +270,11 @@ implements ResourceScheduler, CapacitySc
 
     // Release containers and update queue capacities
     processReleasedContainers(application, application.getCurrentContainers());
-
-    // Inform all NodeManagers about completion of application
-    finishedApplication(applicationId, 
-        application.getAllNodesForApplication());
-
+    if (finishApplication) {
+      // Inform all NodeManagers about completion of application
+      finishedApplication(applicationId, 
+          application.getAllNodesForApplication());
+    }
     // Remove from our data-structure
     applications.remove(applicationId);
   }
@@ -462,7 +462,16 @@ implements ResourceScheduler, CapacitySc
       break;
     case REMOVE:
       try {
-        removeApplication(event.getAppContext().getApplicationID());
+        removeApplication(event.getAppContext().getApplicationID(), true);
+      } catch(IOException ie) {
+        LOG.error("Error in removing application", ie);
+        //TODO have to be shutdown the RM in case of this.
+        // do a graceful shutdown.
+      }
+      break;
+    case EXPIRE:
+      try {
+        removeApplication(event.getAppContext().getApplicationID(), false);
       } catch(IOException ie) {
         LOG.error("Error in removing application", ie);
         //TODO have to be shutdown the RM in case of this.

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Sun May  1 22:35:51 2011
@@ -238,7 +238,7 @@ public class FifoScheduler implements Re
   }
 
   @Override
-  public synchronized void removeApplication(ApplicationId applicationId)
+  public synchronized void removeApplication(ApplicationId applicationId, boolean finishApplication)
   throws IOException {
     Application application = getApplication(applicationId);
     if (application == null) {
@@ -252,11 +252,11 @@ public class FifoScheduler implements Re
     // Update metrics
     metrics.finishApp(application);
     application.finish();
-
-    // Let the cluster know that the applications are done
-    finishedApplication(applicationId, 
-        application.getAllNodesForApplication());
-    
+    if (finishApplication) {
+      // Let the cluster know that the applications are done
+      finishedApplication(applicationId, 
+          application.getAllNodesForApplication());
+    }
     // Remove the application
     applications.remove(applicationId);
   }
@@ -529,12 +529,18 @@ public class FifoScheduler implements Re
       break;
     case REMOVE:
       try {
-        
-        removeApplication(event.getAppContext().getApplicationID());
+        removeApplication(event.getAppContext().getApplicationID(), true);
       } catch(IOException ie) {
         LOG.error("Unable to remove application " + event.getAppContext().getApplicationID(),
ie);
       }
       break;  
+    case EXPIRE:
+      try {
+        removeApplication(event.getAppContext().getApplicationID(), false);
+      } catch(IOException ie) {
+        LOG.error("Unable to remove application " + event.getAppContext().getApplicationID(),
ie);
+      }
+      break;
     }
   }
   

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
Sun May  1 22:35:51 2011
@@ -75,7 +75,7 @@ public class TestAMLaunchFailure extends
     }
 
     @Override
-    public void removeApplication(ApplicationId applicationId)
+    public void removeApplication(ApplicationId applicationId, boolean finishApplication)
         throws IOException {
     }
 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
Sun May  1 22:35:51 2011
@@ -85,7 +85,7 @@ public class TestAMRMRPCResponseId exten
     }
 
     @Override
-    public void removeApplication(ApplicationId applicationId)
+    public void removeApplication(ApplicationId applicationId, boolean finishApplication)
         throws IOException {
     }
 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
Sun May  1 22:35:51 2011
@@ -156,7 +156,7 @@ public class TestAMRestart extends TestC
       case ADD:
         schedulerAddApplication++;
         break;
-      case REMOVE:
+      case EXPIRE:
         schedulerRemoveApplication++;
         LOG.info("REMOVING app : " + schedulerRemoveApplication);
         if (schedulerRemoveApplication == maxFailures) {
@@ -172,7 +172,7 @@ public class TestAMRestart extends TestC
     }
 
     @Override
-    public void removeApplication(ApplicationId applicationId)
+    public void removeApplication(ApplicationId applicationId, boolean finishApplication)
         throws IOException {
     }
     @Override

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
Sun May  1 22:35:51 2011
@@ -96,7 +96,7 @@ public class TestSchedulerNegotiator ext
     }
    
     @Override
-    public void removeApplication(ApplicationId applicationId)
+    public void removeApplication(ApplicationId applicationId, boolean finishApplication)
         throws IOException {
     }
     @Override



Mime
View raw message