hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1537561 [1/3] - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/proto/server/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/re...
Date Thu, 31 Oct 2013 17:25:38 GMT
Author: vinodkv
Date: Thu Oct 31 17:25:37 2013
New Revision: 1537561

URL: http://svn.apache.org/r1537561
Log:
YARN-891. Modified ResourceManager state-store to remember completed applications so that clients can get information about them post RM-restart. Contributed by Jian He.
svn merge --ignore-ancestry -c 1537560 ../../trunk/

Added:
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppAttemptEvent.java
      - copied unchanged from r1537560, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppAttemptEvent.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
      - copied unchanged from r1537560, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNewSavedEvent.java
      - copied unchanged from r1537560, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNewSavedEvent.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppUpdateSavedEvent.java
      - copied unchanged from r1537560, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppUpdateSavedEvent.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptNewSavedEvent.java
      - copied unchanged from r1537560, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptNewSavedEvent.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUpdateSavedEvent.java
      - copied unchanged from r1537560, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUpdateSavedEvent.java
Removed:
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppStoredEvent.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptStoredEvent.java
Modified:
    hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEventType.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptState.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUnregistrationEvent.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Thu Oct 31 17:25:37 2013
@@ -55,6 +55,10 @@ Release 2.3.0 - UNRELEASED
     YARN-1306. Clean up hadoop-sls sample-conf according to YARN-1228 (Wei Yan
     via Sandy Ryza)
 
+    YARN-891. Modified ResourceManager state-store to remember completed
+    applications so that clients can get information about them post RM-restart.
+    (Jian He via vinodkv)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto Thu Oct 31 17:25:37 2013
@@ -71,14 +71,53 @@ message GetGroupsForUserResponseProto {
 ////////////////////////////////////////////////////////////////////////
 ////// RM recovery related records /////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////
+enum RMAppAttemptStateProto {
+  RMATTEMPT_NEW = 1;
+  RMATTEMPT_SUBMITTED = 2;
+  RMATTEMPT_SCHEDULED = 3;
+  RMATTEMPT_ALLOCATED = 4;
+  RMATTEMPT_LAUNCHED = 5;
+  RMATTEMPT_FAILED = 6;
+  RMATTEMPT_RUNNING = 7;
+  RMATTEMPT_FINISHING = 8;
+  RMATTEMPT_FINISHED = 9;
+  RMATTEMPT_KILLED = 10;
+  RMATTEMPT_ALLOCATED_SAVING = 11;
+  RMATTEMPT_LAUNCHED_UNMANAGED_SAVING = 12;
+  RMATTEMPT_RECOVERED = 13;
+  RMATTEMPT_FINAL_SAVING = 14;
+}
+
+enum RMAppStateProto {
+  RMAPP_NEW = 1;
+  RMAPP_NEW_SAVING = 2;
+  RMAPP_SUBMITTED = 3;
+  RMAPP_ACCEPTED = 4;
+  RMAPP_RUNNING = 5;
+  RMAPP_FINAL_SAVING = 6;
+  RMAPP_FINISHING = 7;
+  RMAPP_FINISHED = 8;
+  RMAPP_FAILED = 9;
+  RMAPP_KILLED = 10;
+}
+
 message ApplicationStateDataProto {
   optional int64 submit_time = 1;
-  optional ApplicationSubmissionContextProto application_submission_context = 2;
-  optional string user = 3;
+  optional int64 start_time = 2;
+  optional ApplicationSubmissionContextProto application_submission_context = 3;
+  optional string user = 4;
+  optional RMAppStateProto application_state = 5;
+  optional string diagnostics = 6 [default = "N/A"];
+  optional int64 finish_time = 7;
 }
 
 message ApplicationAttemptStateDataProto {
   optional ApplicationAttemptIdProto attemptId = 1;
   optional ContainerProto master_container = 2;
   optional bytes app_attempt_tokens = 3;
+  optional RMAppAttemptStateProto app_attempt_state = 4;
+  optional string final_tracking_url = 5;
+  optional string diagnostics = 6 [default = "N/A"];
+  optional int64 start_time = 7;
+  optional FinalApplicationStatusProto final_application_status = 8;
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Thu Oct 31 17:25:37 2013
@@ -63,7 +63,6 @@ public class RMAppManager implements Eve
   private static final Log LOG = LogFactory.getLog(RMAppManager.class);
 
   private int completedAppsMax = YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS;
-  private int globalMaxAppAttempts;
   private LinkedList<ApplicationId> completedApps = new LinkedList<ApplicationId>();
 
   private final RMContext rmContext;
@@ -83,8 +82,6 @@ public class RMAppManager implements Eve
     setCompletedAppsMax(conf.getInt(
         YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS,
         YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS));
-    globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
-        YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
   }
 
   /**
@@ -302,10 +299,11 @@ public class RMAppManager implements Eve
       throw RPCUtil.getRemoteException(ie);
     }
 
-    // All done, start the RMApp
-    this.rmContext.getDispatcher().getEventHandler().handle(
-        new RMAppEvent(applicationId, isRecovered ? RMAppEventType.RECOVER:
-            RMAppEventType.START));
+    if (!isRecovered) {
+      // All done, start the RMApp
+      this.rmContext.getDispatcher().getEventHandler()
+        .handle(new RMAppEvent(applicationId, RMAppEventType.START));
+    }
   }
   
   private Credentials parseCredentials(ApplicationSubmissionContext application) 
@@ -328,53 +326,19 @@ public class RMAppManager implements Eve
     // recover applications
     Map<ApplicationId, ApplicationState> appStates = state.getApplicationState();
     LOG.info("Recovering " + appStates.size() + " applications");
-    for(ApplicationState appState : appStates.values()) {
-      boolean shouldRecover = true;
-      if(appState.getApplicationSubmissionContext().getUnmanagedAM()) {
-        // do not recover unmanaged applications since current recovery 
-        // mechanism of restarting attempts does not work for them.
-        // This will need to be changed in work preserving recovery in which 
-        // RM will re-connect with the running AM's instead of restarting them
-        LOG.info("Not recovering unmanaged application " + appState.getAppId());
-        shouldRecover = false;
-      }
-      int individualMaxAppAttempts = appState.getApplicationSubmissionContext()
-          .getMaxAppAttempts();
-      int maxAppAttempts;
-      if (individualMaxAppAttempts <= 0 ||
-          individualMaxAppAttempts > globalMaxAppAttempts) {
-        maxAppAttempts = globalMaxAppAttempts;
-        LOG.warn("The specific max attempts: " + individualMaxAppAttempts
-            + " for application: " + appState.getAppId()
-            + " is invalid, because it is out of the range [1, "
-            + globalMaxAppAttempts + "]. Use the global max attempts instead.");
-      } else {
-        maxAppAttempts = individualMaxAppAttempts;
-      }
-      // In work-preserve restart, if attemptCount == maxAttempts, the job still
-      // needs to be recovered because the last attempt may still be running.
-      if(appState.getAttemptCount() >= maxAppAttempts) {
-        LOG.info("Not recovering application " + appState.getAppId() +
-            " due to recovering attempt is beyond maxAppAttempt limit");
-        shouldRecover = false;
-      }
-
-      // re-submit the application
-      // this is going to send an app start event but since the async dispatcher
-      // has not started that event will be queued until we have completed re
-      // populating the state
-      if(shouldRecover) {
-        LOG.info("Recovering application " + appState.getAppId());
-        submitApplication(appState.getApplicationSubmissionContext(), 
-                        appState.getSubmitTime(), true, appState.getUser());
-        // re-populate attempt information in application
-        RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get(
-                                                        appState.getAppId());
-        appImpl.recover(state);
-      }
-      else {
-        store.removeApplication(appState);
-      }
+    for (ApplicationState appState : appStates.values()) {
+      LOG.info("Recovering application " + appState.getAppId());
+      submitApplication(appState.getApplicationSubmissionContext(),
+        appState.getSubmitTime(), true, appState.getUser());
+      // re-populate attempt information in application
+      RMAppImpl appImpl =
+          (RMAppImpl) rmContext.getRMApps().get(appState.getAppId());
+      appImpl.recover(state);
+      // Recover the app synchronously, as otherwise client is possible to see
+      // the application not recovered before it is actually recovered because
+      // ClientRMService is already started at this point of time.
+      appImpl.handle(new RMAppEvent(appImpl.getApplicationId(),
+        RMAppEventType.RECOVER));
     }
   }
 

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java Thu Oct 31 17:25:37 2013
@@ -133,8 +133,11 @@ public class FileSystemRMStateStore exte
                   ApplicationStateDataProto.parseFrom(childData));
             ApplicationState appState =
                 new ApplicationState(appStateData.getSubmitTime(),
+                  appStateData.getStartTime(),
                   appStateData.getApplicationSubmissionContext(),
-                  appStateData.getUser());
+                  appStateData.getUser(),
+                  appStateData.getState(),
+                  appStateData.getDiagnostics(), appStateData.getFinishTime());
             // assert child node name is same as actual applicationId
             assert appId.equals(appState.context.getApplicationId());
             rmState.appState.put(appId, appState);
@@ -156,7 +159,12 @@ public class FileSystemRMStateStore exte
             }
             ApplicationAttemptState attemptState =
                 new ApplicationAttemptState(attemptId,
-                  attemptStateData.getMasterContainer(), credentials);
+                  attemptStateData.getMasterContainer(), credentials,
+                  attemptStateData.getStartTime(),
+                  attemptStateData.getState(),
+                  attemptStateData.getFinalTrackingUrl(),
+                  attemptStateData.getDiagnostics(),
+                  attemptStateData.getFinalApplicationStatus());
 
             // assert child node name is same as application attempt id
             assert attemptId.equals(attemptState.getAttemptId());
@@ -232,7 +240,7 @@ public class FileSystemRMStateStore exte
   }
 
   @Override
-  public synchronized void storeApplicationState(String appId,
+  public synchronized void storeApplicationStateInternal(String appId,
       ApplicationStateDataPBImpl appStateDataPB) throws Exception {
     Path appDirPath = getAppDir(rmAppRoot, appId);
     fs.mkdirs(appDirPath);
@@ -251,15 +259,34 @@ public class FileSystemRMStateStore exte
   }
 
   @Override
-  public synchronized void storeApplicationAttemptState(String attemptId,
-      ApplicationAttemptStateDataPBImpl attemptStateDataPB) throws Exception {
+  public synchronized void updateApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateDataPB) throws Exception {
+    Path appDirPath = getAppDir(rmAppRoot, appId);
+    Path nodeCreatePath = getNodePath(appDirPath, appId);
+
+    LOG.info("Updating info for app: " + appId + " at: " + nodeCreatePath);
+    byte[] appStateData = appStateDataPB.getProto().toByteArray();
+    try {
+      // currently throw all exceptions. May need to respond differently for HA
+      // based on whether we have lost the right to write to FS
+      updateFile(nodeCreatePath, appStateData);
+    } catch (Exception e) {
+      LOG.info("Error updating info for app: " + appId, e);
+      throw e;
+    }
+  }
+
+  @Override
+  public synchronized void storeApplicationAttemptStateInternal(
+      String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+      throws Exception {
     ApplicationAttemptId appAttemptId =
         ConverterUtils.toApplicationAttemptId(attemptId);
     Path appDirPath =
         getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
     Path nodeCreatePath = getNodePath(appDirPath, attemptId);
-    LOG.info("Storing info for attempt: " + attemptId
-             + " at: " + nodeCreatePath);
+    LOG.info("Storing info for attempt: " + attemptId + " at: "
+        + nodeCreatePath);
     byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
     try {
       // currently throw all exceptions. May need to respond differently for HA
@@ -272,6 +299,28 @@ public class FileSystemRMStateStore exte
   }
 
   @Override
+  public synchronized void updateApplicationAttemptStateInternal(
+      String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+      throws Exception {
+    ApplicationAttemptId appAttemptId =
+        ConverterUtils.toApplicationAttemptId(attemptId);
+    Path appDirPath =
+        getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
+    Path nodeCreatePath = getNodePath(appDirPath, attemptId);
+    LOG.info("Updating info for attempt: " + attemptId + " at: "
+        + nodeCreatePath);
+    byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
+    try {
+      // currently throw all exceptions. May need to respond differently for HA
+      // based on whether we have lost the right to write to FS
+      updateFile(nodeCreatePath, attemptStateData);
+    } catch (Exception e) {
+      LOG.info("Error updating info for attempt: " + attemptId, e);
+      throw e;
+    }
+  }
+
+  @Override
   public synchronized void removeApplicationState(ApplicationState appState)
       throws Exception {
     String appId = appState.getAppId().toString();
@@ -373,12 +422,21 @@ public class FileSystemRMStateStore exte
     Path tempPath =
         new Path(outputPath.getParent(), outputPath.getName() + ".tmp");
     FSDataOutputStream fsOut = null;
-    fsOut = fs.create(tempPath, false);
+    // This file will be overwritten when app/attempt finishes for saving the
+    // final status.
+    fsOut = fs.create(tempPath, true);
     fsOut.write(data);
     fsOut.close();
     fs.rename(tempPath, outputPath);
   }
 
+  private void updateFile(Path outputPath, byte[] data) throws Exception {
+    if (fs.exists(outputPath)) {
+      deleteFile(outputPath);
+    }
+    writeFile(outputPath, data);
+  }
+
   private boolean renameFile(Path src, Path dst) throws Exception {
     return fs.rename(src, dst);
   }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java Thu Oct 31 17:25:37 2013
@@ -76,22 +76,39 @@ public class MemoryRMStateStore extends 
   }
 
   @Override
-  public void storeApplicationState(String appId, 
+  public void storeApplicationStateInternal(String appId, 
                                      ApplicationStateDataPBImpl appStateData)
       throws Exception {
-    ApplicationState appState = new ApplicationState(
-        appStateData.getSubmitTime(),
-        appStateData.getApplicationSubmissionContext(), appStateData.getUser());
-    if (state.appState.containsKey(appState.getAppId())) {
-      Exception e = new IOException("App: " + appId + " is already stored.");
-      LOG.info("Error storing info for app: " + appId, e);
-      throw e;
-    }
+    ApplicationState appState =
+        new ApplicationState(appStateData.getSubmitTime(),
+          appStateData.getStartTime(),
+          appStateData.getApplicationSubmissionContext(),
+          appStateData.getUser());
     state.appState.put(appState.getAppId(), appState);
   }
 
   @Override
-  public synchronized void storeApplicationAttemptState(String attemptIdStr, 
+  public void updateApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateData) throws Exception {
+    ApplicationState updatedAppState =
+        new ApplicationState(appStateData.getSubmitTime(),
+          appStateData.getStartTime(),
+          appStateData.getApplicationSubmissionContext(),
+          appStateData.getUser(), appStateData.getState(),
+          appStateData.getDiagnostics(), appStateData.getFinishTime());
+    LOG.info("Updating final state " + appStateData.getState() + " for app: "
+        + appId);
+    ApplicationId applicationId = updatedAppState.getAppId();
+    if (state.appState.get(applicationId) != null) {
+      // add the earlier attempts back
+      updatedAppState.attempts
+        .putAll(state.appState.get(applicationId).attempts);
+    }
+    state.appState.put(applicationId, updatedAppState);
+  }
+
+  @Override
+  public synchronized void storeApplicationAttemptStateInternal(String attemptIdStr, 
                             ApplicationAttemptStateDataPBImpl attemptStateData)
                             throws Exception {
     ApplicationAttemptId attemptId = ConverterUtils
@@ -105,22 +122,48 @@ public class MemoryRMStateStore extends 
     }
     ApplicationAttemptState attemptState =
         new ApplicationAttemptState(attemptId,
-          attemptStateData.getMasterContainer(), credentials);
+          attemptStateData.getMasterContainer(), credentials,
+          attemptStateData.getStartTime());
 
     ApplicationState appState = state.getApplicationState().get(
         attemptState.getAttemptId().getApplicationId());
     if (appState == null) {
       throw new YarnRuntimeException("Application doesn't exist");
     }
+    appState.attempts.put(attemptState.getAttemptId(), attemptState);
+  }
 
-    if (appState.attempts.containsKey(attemptState.getAttemptId())) {
-      Exception e = new IOException("Attempt: " +
-          attemptState.getAttemptId() + " is already stored.");
-      LOG.info("Error storing info for attempt: " +
-          attemptState.getAttemptId(), e);
-      throw e;
+  @Override
+  public synchronized void updateApplicationAttemptStateInternal(
+      String attemptIdStr, ApplicationAttemptStateDataPBImpl attemptStateData)
+      throws Exception {
+    ApplicationAttemptId attemptId =
+        ConverterUtils.toApplicationAttemptId(attemptIdStr);
+    Credentials credentials = null;
+    if (attemptStateData.getAppAttemptTokens() != null) {
+      DataInputByteBuffer dibb = new DataInputByteBuffer();
+      credentials = new Credentials();
+      dibb.reset(attemptStateData.getAppAttemptTokens());
+      credentials.readTokenStorageStream(dibb);
     }
-    appState.attempts.put(attemptState.getAttemptId(), attemptState);
+    ApplicationAttemptState updatedAttemptState =
+        new ApplicationAttemptState(attemptId,
+          attemptStateData.getMasterContainer(), credentials,
+          attemptStateData.getStartTime(), attemptStateData.getState(),
+          attemptStateData.getFinalTrackingUrl(),
+          attemptStateData.getDiagnostics(),
+          attemptStateData.getFinalApplicationStatus());
+
+    ApplicationState appState =
+        state.getApplicationState().get(
+          updatedAttemptState.getAttemptId().getApplicationId());
+    if (appState == null) {
+      throw new YarnRuntimeException("Application doesn't exist");
+    }
+    LOG.info("Updating final state " + updatedAttemptState.getState()
+        + " for attempt: " + updatedAttemptState.getAttemptId());
+    appState.attempts.put(updatedAttemptState.getAttemptId(),
+      updatedAttemptState);
   }
 
   @Override

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java Thu Oct 31 17:25:37 2013
@@ -50,13 +50,13 @@ public class NullRMStateStore extends RM
   }
 
   @Override
-  protected void storeApplicationState(String appId,
+  protected void storeApplicationStateInternal(String appId,
       ApplicationStateDataPBImpl appStateData) throws Exception {
     // Do nothing
   }
 
   @Override
-  protected void storeApplicationAttemptState(String attemptId,
+  protected void storeApplicationAttemptStateInternal(String attemptId,
       ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception {
     // Do nothing
   }
@@ -89,4 +89,16 @@ public class NullRMStateStore extends RM
   public void removeRMDTMasterKeyState(DelegationKey delegationKey) throws Exception {
     // Do nothing
   }
+
+  @Override
+  protected void updateApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateData) throws Exception {
+    // Do nothing 
+  }
+
+  @Override
+  protected void updateApplicationAttemptStateInternal(String attemptId,
+      ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception {
+    // Do nothing
+  }
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java Thu Oct 31 17:25:37 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
@@ -50,10 +51,14 @@ import org.apache.hadoop.yarn.security.c
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppStoredEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRemovedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNewSavedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppUpdateSavedEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
 
 @Private
 @Unstable
@@ -86,13 +91,32 @@ public abstract class RMStateStore exten
     final ApplicationAttemptId attemptId;
     final Container masterContainer;
     final Credentials appAttemptCredentials;
+    long startTime = 0;
+    // fields set when attempt completes
+    RMAppAttemptState state;
+    String finalTrackingUrl = "N/A";
+    String diagnostics;
+    FinalApplicationStatus amUnregisteredFinalStatus;
 
     public ApplicationAttemptState(ApplicationAttemptId attemptId,
-        Container masterContainer,
-        Credentials appAttemptCredentials) {
+        Container masterContainer, Credentials appAttemptCredentials,
+        long startTime) {
+      this(attemptId, masterContainer, appAttemptCredentials, startTime, null,
+        null, "", null);
+    }
+
+    public ApplicationAttemptState(ApplicationAttemptId attemptId,
+        Container masterContainer, Credentials appAttemptCredentials,
+        long startTime, RMAppAttemptState state, String finalTrackingUrl,
+        String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus) {
       this.attemptId = attemptId;
       this.masterContainer = masterContainer;
       this.appAttemptCredentials = appAttemptCredentials;
+      this.startTime = startTime;
+      this.state = state;
+      this.finalTrackingUrl = finalTrackingUrl;
+      this.diagnostics = diagnostics == null ? "" : diagnostics;
+      this.amUnregisteredFinalStatus = amUnregisteredFinalStatus;
     }
 
     public Container getMasterContainer() {
@@ -104,6 +128,21 @@ public abstract class RMStateStore exten
     public Credentials getAppAttemptCredentials() {
       return appAttemptCredentials;
     }
+    public RMAppAttemptState getState(){
+      return state;
+    }
+    public String getFinalTrackingUrl() {
+      return finalTrackingUrl;
+    }
+    public String getDiagnostics() {
+      return diagnostics;
+    }
+    public long getStartTime() {
+      return startTime;
+    }
+    public FinalApplicationStatus getFinalApplicationStatus() {
+      return amUnregisteredFinalStatus;
+    }
   }
   
   /**
@@ -112,15 +151,30 @@ public abstract class RMStateStore exten
   public static class ApplicationState {
     final ApplicationSubmissionContext context;
     final long submitTime;
+    final long startTime;
     final String user;
     Map<ApplicationAttemptId, ApplicationAttemptState> attempts =
                   new HashMap<ApplicationAttemptId, ApplicationAttemptState>();
-    
-    ApplicationState(long submitTime, ApplicationSubmissionContext context,
-        String user) {
+    // fields set when application completes.
+    RMAppState state;
+    String diagnostics;
+    long finishTime;
+
+    public ApplicationState(long submitTime,
+        long startTime, ApplicationSubmissionContext context, String user) {
+      this(submitTime, startTime, context, user, null, "", 0);
+    }
+
+    public ApplicationState(long submitTime,
+        long startTime,ApplicationSubmissionContext context,
+        String user, RMAppState state, String diagnostics, long finishTime) {
       this.submitTime = submitTime;
+      this.startTime = startTime;
       this.context = context;
       this.user = user;
+      this.state = state;
+      this.diagnostics = diagnostics == null ? "" : diagnostics;
+      this.finishTime = finishTime;
     }
 
     public ApplicationId getAppId() {
@@ -129,6 +183,9 @@ public abstract class RMStateStore exten
     public long getSubmitTime() {
       return submitTime;
     }
+    public long getStartTime() {
+      return startTime;
+    }
     public int getAttemptCount() {
       return attempts.size();
     }
@@ -141,6 +198,15 @@ public abstract class RMStateStore exten
     public String getUser() {
       return user;
     }
+    public RMAppState getState() {
+      return state;
+    }
+    public String getDiagnostics() {
+      return diagnostics;
+    }
+    public long getFinishTime() {
+      return finishTime;
+    }
   }
 
   public static class RMDTSecretManagerState {
@@ -249,23 +315,31 @@ public abstract class RMStateStore exten
    * RMAppStoredEvent will be sent on completion to notify the RMApp
    */
   @SuppressWarnings("unchecked")
-  public synchronized void storeApplication(RMApp app) {
+  public synchronized void storeNewApplication(RMApp app) {
     ApplicationSubmissionContext context = app
                                             .getApplicationSubmissionContext();
     assert context instanceof ApplicationSubmissionContextPBImpl;
-    ApplicationState appState = new ApplicationState(
-        app.getSubmitTime(), context, app.getUser());
+    ApplicationState appState =
+        new ApplicationState(app.getSubmitTime(), app.getStartTime(), context,
+          app.getUser());
     dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState));
   }
-    
+
+  @SuppressWarnings("unchecked")
+  public synchronized void updateApplicationState(ApplicationState appState) {
+    dispatcher.getEventHandler().handle(new RMStateUpdateAppEvent(appState));
+  }
+
   /**
    * Blocking API
    * Derived classes must implement this method to store the state of an 
    * application.
    */
-  protected abstract void storeApplicationState(String appId,
-                                      ApplicationStateDataPBImpl appStateData) 
-                                      throws Exception;
+  protected abstract void storeApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateData) throws Exception;
+
+  protected abstract void updateApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateData) throws Exception;
   
   @SuppressWarnings("unchecked")
   /**
@@ -274,26 +348,35 @@ public abstract class RMStateStore exten
    * This does not block the dispatcher threads
    * RMAppAttemptStoredEvent will be sent on completion to notify the RMAppAttempt
    */
-  public synchronized void storeApplicationAttempt(RMAppAttempt appAttempt) {
+  public synchronized void storeNewApplicationAttempt(RMAppAttempt appAttempt) {
     Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
 
     ApplicationAttemptState attemptState =
         new ApplicationAttemptState(appAttempt.getAppAttemptId(),
-          appAttempt.getMasterContainer(), credentials);
+          appAttempt.getMasterContainer(), credentials,
+          appAttempt.getStartTime());
 
     dispatcher.getEventHandler().handle(
       new RMStateStoreAppAttemptEvent(attemptState));
   }
-  
+
+  @SuppressWarnings("unchecked")
+  public synchronized void updateApplicationAttemptState(
+      ApplicationAttemptState attemptState) {
+    dispatcher.getEventHandler().handle(
+      new RMStateUpdateAppAttemptEvent(attemptState));
+  }
+
   /**
    * Blocking API
    * Derived classes must implement this method to store the state of an 
    * application attempt
    */
-  protected abstract void storeApplicationAttemptState(String attemptId,
-                            ApplicationAttemptStateDataPBImpl attemptStateData) 
-                            throws Exception;
+  protected abstract void storeApplicationAttemptStateInternal(String attemptId,
+      ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception;
 
+  protected abstract void updateApplicationAttemptStateInternal(String attemptId,
+      ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception;
 
   /**
    * RMDTSecretManager call this to store the state of a delegation token
@@ -372,13 +455,14 @@ public abstract class RMStateStore exten
    */
   public synchronized void removeApplication(RMApp app) {
     ApplicationState appState = new ApplicationState(
-            app.getSubmitTime(), app.getApplicationSubmissionContext(),
-            app.getUser());
+            app.getSubmitTime(), app.getStartTime(),
+            app.getApplicationSubmissionContext(), app.getUser());
     for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
       Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
       ApplicationAttemptState attemptState =
           new ApplicationAttemptState(appAttempt.getAppAttemptId(),
-            appAttempt.getMasterContainer(), credentials);
+            appAttempt.getMasterContainer(), credentials,
+            appAttempt.getStartTime());
       appState.attempts.put(attemptState.getAttemptId(), attemptState);
     }
     
@@ -409,7 +493,7 @@ public abstract class RMStateStore exten
   public static final Text AM_CLIENT_TOKEN_MASTER_KEY_NAME =
       new Text("YARN_CLIENT_TOKEN_MASTER_KEY");
   
-  private Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
+  public Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
     Credentials credentials = new Credentials();
     Token<AMRMTokenIdentifier> appToken = appAttempt.getAMRMToken();
     if(appToken != null){
@@ -427,92 +511,123 @@ public abstract class RMStateStore exten
   // Dispatcher related code
   
   private synchronized void handleStoreEvent(RMStateStoreEvent event) {
-    switch(event.getType()) {
-      case STORE_APP:
-        {
-          ApplicationState apptState =
-              ((RMStateStoreAppEvent) event).getAppState();
-          Exception storedException = null;
-          ApplicationStateDataPBImpl appStateData =
-              new ApplicationStateDataPBImpl();
-          appStateData.setSubmitTime(apptState.getSubmitTime());
-          appStateData.setApplicationSubmissionContext(
-              apptState.getApplicationSubmissionContext());
-          appStateData.setUser(apptState.getUser());
-          ApplicationId appId =
-              apptState.getApplicationSubmissionContext().getApplicationId();
-
-          LOG.info("Storing info for app: " + appId);
-          try {
-            storeApplicationState(appId.toString(), appStateData);
-          } catch (Exception e) {
-            LOG.error("Error storing app: " + appId, e);
-            storedException = e;
-          } finally {
-            notifyDoneStoringApplication(appId, storedException);
-          }
+    if (event.getType().equals(RMStateStoreEventType.STORE_APP)
+        || event.getType().equals(RMStateStoreEventType.UPDATE_APP)) {
+      ApplicationState appState = null;
+      if (event.getType().equals(RMStateStoreEventType.STORE_APP)) {
+        appState = ((RMStateStoreAppEvent) event).getAppState();
+      } else {
+        assert event.getType().equals(RMStateStoreEventType.UPDATE_APP);
+        appState = ((RMStateUpdateAppEvent) event).getAppState();
+      }
+
+      Exception storedException = null;
+      ApplicationStateDataPBImpl appStateData =
+          (ApplicationStateDataPBImpl) ApplicationStateDataPBImpl
+            .newApplicationStateData(appState.getSubmitTime(),
+              appState.getStartTime(), appState.getUser(),
+              appState.getApplicationSubmissionContext(), appState.getState(),
+              appState.getDiagnostics(), appState.getFinishTime());
+
+      ApplicationId appId =
+          appState.getApplicationSubmissionContext().getApplicationId();
+
+      LOG.info("Storing info for app: " + appId);
+      try {
+        if (event.getType().equals(RMStateStoreEventType.STORE_APP)) {
+          storeApplicationStateInternal(appId.toString(), appStateData);
+        } else {
+          assert event.getType().equals(RMStateStoreEventType.UPDATE_APP);
+          updateApplicationStateInternal(appId.toString(), appStateData);
+        }
+      } catch (Exception e) {
+        LOG.error("Error storing app: " + appId, e);
+        storedException = e;
+      } finally {
+        if (event.getType().equals(RMStateStoreEventType.STORE_APP)) {
+          notifyDoneStoringApplication(appId, storedException);
+        } else {
+          notifyDoneUpdatingApplication(appId, storedException);
+        }
+      }
+    } else if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT)
+        || event.getType().equals(RMStateStoreEventType.UPDATE_APP_ATTEMPT)) {
+
+      ApplicationAttemptState attemptState = null;
+      if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT)) {
+        attemptState =
+            ((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
+      } else {
+        assert event.getType().equals(RMStateStoreEventType.UPDATE_APP_ATTEMPT);
+        attemptState =
+            ((RMStateUpdateAppAttemptEvent) event).getAppAttemptState();
+      }
+
+      Exception storedException = null;
+      Credentials credentials = attemptState.getAppAttemptCredentials();
+      ByteBuffer appAttemptTokens = null;
+      try {
+        if (credentials != null) {
+          DataOutputBuffer dob = new DataOutputBuffer();
+          credentials.writeTokenStorageToStream(dob);
+          appAttemptTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+        }
+        ApplicationAttemptStateDataPBImpl attemptStateData =
+            (ApplicationAttemptStateDataPBImpl) ApplicationAttemptStateDataPBImpl
+              .newApplicationAttemptStateData(attemptState.getAttemptId(),
+                attemptState.getMasterContainer(), appAttemptTokens,
+                attemptState.getStartTime(), attemptState.getState(),
+                attemptState.getFinalTrackingUrl(),
+                attemptState.getDiagnostics(),
+                attemptState.getFinalApplicationStatus());
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Storing info for attempt: " + attemptState.getAttemptId());
         }
-        break;
-      case STORE_APP_ATTEMPT:
-        {
-          ApplicationAttemptState attemptState = 
-                    ((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
-          Exception storedException = null;
-
-          Credentials credentials = attemptState.getAppAttemptCredentials();
-          ByteBuffer appAttemptTokens = null;
-          try {
-            if(credentials != null){
-              DataOutputBuffer dob = new DataOutputBuffer();
-                credentials.writeTokenStorageToStream(dob);
-              appAttemptTokens =
-                  ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
-            }
-            ApplicationAttemptStateDataPBImpl attemptStateData =
-              (ApplicationAttemptStateDataPBImpl) ApplicationAttemptStateDataPBImpl
-                  .newApplicationAttemptStateData(attemptState.getAttemptId(),
-                    attemptState.getMasterContainer(), appAttemptTokens);
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("Storing info for attempt: " + attemptState.getAttemptId());
-            }
-            storeApplicationAttemptState(attemptState.getAttemptId().toString(), 
-                                         attemptStateData);
-          } catch (Exception e) {
-            LOG.error("Error storing appAttempt: " 
-                      + attemptState.getAttemptId(), e);
-            storedException = e;
-          } finally {
-            notifyDoneStoringApplicationAttempt(attemptState.getAttemptId(), 
-                                                storedException);            
-          }
+        if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT)) {
+          storeApplicationAttemptStateInternal(attemptState.getAttemptId()
+            .toString(), attemptStateData);
+        } else {
+          assert event.getType().equals(
+            RMStateStoreEventType.UPDATE_APP_ATTEMPT);
+          updateApplicationAttemptStateInternal(attemptState.getAttemptId()
+            .toString(), attemptStateData);
         }
-        break;
-      case REMOVE_APP:
-        {
-          ApplicationState appState = 
-                          ((RMStateStoreRemoveAppEvent) event).getAppState();
-          ApplicationId appId = appState.getAppId();
-          Exception removedException = null;
-          LOG.info("Removing info for app: " + appId);
-          try {
-            removeApplicationState(appState);
-          } catch (Exception e) {
-            LOG.error("Error removing app: " + appId, e);
-            removedException = e;
-          } finally {
-            notifyDoneRemovingApplcation(appId, removedException);
-          }
+      } catch (Exception e) {
+        LOG
+          .error("Error storing appAttempt: " + attemptState.getAttemptId(), e);
+        storedException = e;
+      } finally {
+        if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT)) {
+          notifyDoneStoringApplicationAttempt(attemptState.getAttemptId(),
+            storedException);
+        } else {
+          notifyDoneUpdatingApplicationAttempt(attemptState.getAttemptId(),
+            storedException);
         }
-        break;
-      default:
-        LOG.error("Unknown RMStateStoreEvent type: " + event.getType());
+      }
+    } else if (event.getType().equals(RMStateStoreEventType.REMOVE_APP)) {
+      ApplicationState appState =
+          ((RMStateStoreRemoveAppEvent) event).getAppState();
+      ApplicationId appId = appState.getAppId();
+      Exception removedException = null;
+      LOG.info("Removing info for app: " + appId);
+      try {
+        removeApplicationState(appState);
+      } catch (Exception e) {
+        LOG.error("Error removing app: " + appId, e);
+        removedException = e;
+      } finally {
+        notifyDoneRemovingApplcation(appId, removedException);
+      }
+    } else {
+      LOG.error("Unknown RMStateStoreEvent type: " + event.getType());
     }
   }
 
   @SuppressWarnings("unchecked")
   /**
    * In (@link handleStoreEvent}, this method is called to notify the
-   * application about operation completion
+   * application that new application is stored in state store
    * @param appId id of the application that has been saved
    * @param storedException the exception that is thrown when storing the
    * application
@@ -520,19 +635,33 @@ public abstract class RMStateStore exten
   private void notifyDoneStoringApplication(ApplicationId appId,
                                                   Exception storedException) {
     rmDispatcher.getEventHandler().handle(
-        new RMAppStoredEvent(appId, storedException));
+        new RMAppNewSavedEvent(appId, storedException));
   }
-  
+
+  @SuppressWarnings("unchecked")
+  private void notifyDoneUpdatingApplication(ApplicationId appId,
+      Exception storedException) {
+    rmDispatcher.getEventHandler().handle(
+      new RMAppUpdateSavedEvent(appId, storedException));
+  }
+
   @SuppressWarnings("unchecked")
   /**
    * In (@link handleStoreEvent}, this method is called to notify the
-   * application attempt about operation completion
+   * application attempt that new attempt is stored in state store
    * @param appAttempt attempt that has been saved
    */
   private void notifyDoneStoringApplicationAttempt(ApplicationAttemptId attemptId,
                                                   Exception storedException) {
     rmDispatcher.getEventHandler().handle(
-        new RMAppAttemptStoredEvent(attemptId, storedException));
+        new RMAppAttemptNewSavedEvent(attemptId, storedException));
+  }
+
+  @SuppressWarnings("unchecked")
+  private void notifyDoneUpdatingApplicationAttempt(ApplicationAttemptId attemptId,
+      Exception updatedException) {
+    rmDispatcher.getEventHandler().handle(
+      new RMAppAttemptUpdateSavedEvent(attemptId, updatedException));
   }
 
   @SuppressWarnings("unchecked")

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java Thu Oct 31 17:25:37 2013
@@ -21,5 +21,7 @@ package org.apache.hadoop.yarn.server.re
 public enum RMStateStoreEventType {
   STORE_APP_ATTEMPT,
   STORE_APP,
+  UPDATE_APP,
+  UPDATE_APP_ATTEMPT,
   REMOVE_APP
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java Thu Oct 31 17:25:37 2013
@@ -18,7 +18,14 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.recovery;
 
-import com.google.common.annotations.VisibleForTesting;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -27,6 +34,8 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.io.DataInputByteBuffer;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.ZKUtil;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -37,9 +46,6 @@ import org.apache.hadoop.yarn.security.c
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.ZKUtil;
-
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.Code;
@@ -51,13 +57,7 @@ import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import com.google.common.annotations.VisibleForTesting;
 
 @Private
 @Unstable
@@ -224,8 +224,11 @@ public class ZKRMStateStore extends RMSt
                 ApplicationStateDataProto.parseFrom(childData));
         ApplicationState appState =
             new ApplicationState(appStateData.getSubmitTime(),
-                appStateData.getApplicationSubmissionContext(),
-                appStateData.getUser());
+              appStateData.getStartTime(),
+              appStateData.getApplicationSubmissionContext(),
+              appStateData.getUser(),
+              appStateData.getState(),
+              appStateData.getDiagnostics(), appStateData.getFinishTime());
         if (!appId.equals(appState.context.getApplicationId())) {
           throw new YarnRuntimeException("The child node name is different " +
               "from the application id");
@@ -249,7 +252,12 @@ public class ZKRMStateStore extends RMSt
         }
         ApplicationAttemptState attemptState =
             new ApplicationAttemptState(attemptId,
-                attemptStateData.getMasterContainer(), credentials);
+              attemptStateData.getMasterContainer(), credentials,
+              attemptStateData.getStartTime(),
+              attemptStateData.getState(),
+              attemptStateData.getFinalTrackingUrl(),
+              attemptStateData.getDiagnostics(),
+              attemptStateData.getFinalApplicationStatus());
         if (!attemptId.equals(attemptState.getAttemptId())) {
           throw new YarnRuntimeException("The child node name is different " +
               "from the application attempt id");
@@ -280,21 +288,34 @@ public class ZKRMStateStore extends RMSt
   }
 
   @Override
-  public synchronized void storeApplicationState(
-      String appId, ApplicationStateDataPBImpl appStateDataPB) throws
-      Exception {
+  public synchronized void storeApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateDataPB) throws Exception {
     String nodeCreatePath = getNodePath(rmAppRoot, appId);
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("Storing info for app: " + appId + " at: " + nodeCreatePath);
     }
     byte[] appStateData = appStateDataPB.getProto().toByteArray();
-    createWithRetries(
-        nodeCreatePath, appStateData, zkAcl, CreateMode.PERSISTENT);
+    createWithRetries(nodeCreatePath, appStateData, zkAcl,
+      CreateMode.PERSISTENT);
+
+  }
+
+  @Override
+  public synchronized void updateApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateDataPB) throws Exception {
+    String nodeCreatePath = getNodePath(rmAppRoot, appId);
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Storing final state info for app: " + appId + " at: "
+          + nodeCreatePath);
+    }
+    byte[] appStateData = appStateDataPB.getProto().toByteArray();
+    setDataWithRetries(nodeCreatePath, appStateData, 0);
   }
 
   @Override
-  public synchronized void storeApplicationAttemptState(
+  public synchronized void storeApplicationAttemptStateInternal(
       String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
       throws Exception {
     String nodeCreatePath = getNodePath(rmAppRoot, attemptId);
@@ -304,7 +325,20 @@ public class ZKRMStateStore extends RMSt
     }
     byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
     createWithRetries(nodeCreatePath, attemptStateData, zkAcl,
-        CreateMode.PERSISTENT);
+      CreateMode.PERSISTENT);
+  }
+
+  @Override
+  public synchronized void updateApplicationAttemptStateInternal(
+      String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+      throws Exception {
+    String nodeCreatePath = getNodePath(rmAppRoot, attemptId);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Storing final state info for attempt: " + attemptId + " at: "
+          + nodeCreatePath);
+    }
+    byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
+    setDataWithRetries(nodeCreatePath, attemptStateData, 0);
   }
 
   @Override

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java Thu Oct 31 17:25:37 2013
@@ -24,6 +24,8 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 
 /*
  * Contains the state data that needs to be persisted for an ApplicationAttempt
@@ -61,4 +63,50 @@ public interface ApplicationAttemptState
   public ByteBuffer getAppAttemptTokens();
 
   public void setAppAttemptTokens(ByteBuffer attemptTokens);
+
+  /**
+   * Get the final state of the application attempt.
+   * @return the final state of the application attempt.
+   */
+  public RMAppAttemptState getState();
+
+  public void setState(RMAppAttemptState state);
+
+  /**
+   * Get the original not-proxied <em>final tracking url</em> for the
+   * application. This is intended to only be used by the proxy itself.
+   * 
+   * @return the original not-proxied <em>final tracking url</em> for the
+   *         application
+   */
+  public String getFinalTrackingUrl();
+
+  /**
+   * Set the final tracking Url of the AM.
+   * @param url
+   */
+  public void setFinalTrackingUrl(String url);
+  /**
+   * Get the <em>diagnositic information</em> of the attempt 
+   * @return <em>diagnositic information</em> of the attempt
+   */
+  public String getDiagnostics();
+
+  public void setDiagnostics(String diagnostics);
+
+  /**
+   * Get the <em>start time</em> of the application.
+   * @return <em>start time</em> of the application
+   */
+  public long getStartTime();
+
+  public void setStartTime(long startTime);
+
+  /**
+   * Get the <em>final finish status</em> of the application.
+   * @return <em>final finish status</em> of the application
+   */
+  public FinalApplicationStatus getFinalApplicationStatus();
+
+  public void setFinalApplicationStatus(FinalApplicationStatus finishState);
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java Thu Oct 31 17:25:37 2013
@@ -18,10 +18,13 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.recovery.records;
 
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 
 /**
  * Contains all the state data that needs to be stored persistently 
@@ -42,7 +45,19 @@ public interface ApplicationStateData {
   @Public
   @Unstable
   public void setSubmitTime(long submitTime);
-  
+
+  /**
+   * Get the <em>start time</em> of the application.
+   * @return <em>start time</em> of the application
+   */
+  @Public
+  @Stable
+  public abstract long getStartTime();
+
+  @Private
+  @Unstable
+  public abstract void setStartTime(long startTime);
+
   /**
    * The application submitter
    */
@@ -66,6 +81,29 @@ public interface ApplicationStateData {
   @Public
   @Unstable
   public void setApplicationSubmissionContext(
-                                          ApplicationSubmissionContext context);
+      ApplicationSubmissionContext context);
+
+  /**
+   * Get the final state of the application.
+   * @return the final state of the application.
+   */
+  public RMAppState getState();
+
+  public void setState(RMAppState state);
+
+  /**
+   * Get the diagnostics information for the application master.
+   * @return the diagnostics information for the application master.
+   */
+  public String getDiagnostics();
+
+  public void setDiagnostics(String diagnostics);
+
+  /**
+   * The finish time of the application.
+   * @return the finish time of the application.,
+   */
+  public long getFinishTime();
 
+  public void setFinishTime(long finishTime);
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java Thu Oct 31 17:25:37 2013
@@ -22,14 +22,19 @@ import java.nio.ByteBuffer;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMAppAttemptStateProto;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 
 public class ApplicationAttemptStateDataPBImpl
 extends ProtoBase<ApplicationAttemptStateDataProto> 
@@ -156,14 +161,125 @@ implements ApplicationAttemptStateData {
     this.appAttemptTokens = attemptTokens;
   }
 
+  @Override
+  public RMAppAttemptState getState() {
+    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasAppAttemptState()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getAppAttemptState());
+  }
+
+  @Override
+  public void setState(RMAppAttemptState state) {
+    maybeInitBuilder();
+    if (state == null) {
+      builder.clearAppAttemptState();
+      return;
+    }
+    builder.setAppAttemptState(convertToProtoFormat(state));
+  }
+
+  @Override
+  public String getFinalTrackingUrl() {
+    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasFinalTrackingUrl()) {
+      return null;
+    }
+    return p.getFinalTrackingUrl();
+  }
+
+  @Override
+  public void setFinalTrackingUrl(String url) {
+    maybeInitBuilder();
+    if (url == null) {
+      builder.clearFinalTrackingUrl();
+      return;
+    }
+    builder.setFinalTrackingUrl(url);
+  }
+
+  @Override
+  public String getDiagnostics() {
+    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasDiagnostics()) {
+      return null;
+    }
+    return p.getDiagnostics();
+  }
+
+  @Override
+  public void setDiagnostics(String diagnostics) {
+    maybeInitBuilder();
+    if (diagnostics == null) {
+      builder.clearDiagnostics();
+      return;
+    }
+    builder.setDiagnostics(diagnostics);
+  }
+
+  @Override
+  public long getStartTime() {
+    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getStartTime();
+  }
+
+  @Override
+  public void setStartTime(long startTime) {
+    maybeInitBuilder();
+    builder.setStartTime(startTime);
+  }
+
+  @Override
+  public FinalApplicationStatus getFinalApplicationStatus() {
+    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasFinalApplicationStatus()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getFinalApplicationStatus());
+  }
+
+  @Override
+  public void setFinalApplicationStatus(FinalApplicationStatus finishState) {
+    maybeInitBuilder();
+    if (finishState == null) {
+      builder.clearFinalApplicationStatus();
+      return;
+    }
+    builder.setFinalApplicationStatus(convertToProtoFormat(finishState));
+  }
+
   public static ApplicationAttemptStateData newApplicationAttemptStateData(
       ApplicationAttemptId attemptId, Container container,
-      ByteBuffer attemptTokens) {
+      ByteBuffer attemptTokens, long startTime, RMAppAttemptState finalState,
+      String finalTrackingUrl, String diagnostics,
+      FinalApplicationStatus amUnregisteredFinalStatus) {
     ApplicationAttemptStateData attemptStateData =
         recordFactory.newRecordInstance(ApplicationAttemptStateData.class);
     attemptStateData.setAttemptId(attemptId);
     attemptStateData.setMasterContainer(container);
     attemptStateData.setAppAttemptTokens(attemptTokens);
+    attemptStateData.setState(finalState);
+    attemptStateData.setFinalTrackingUrl(finalTrackingUrl);
+    attemptStateData.setDiagnostics(diagnostics);
+    attemptStateData.setStartTime(startTime);
+    attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus);
     return attemptStateData;
   }
+
+  private static String RM_APP_ATTEMPT_PREFIX = "RMATTEMPT_";
+  public static RMAppAttemptStateProto convertToProtoFormat(RMAppAttemptState e) {
+    return RMAppAttemptStateProto.valueOf(RM_APP_ATTEMPT_PREFIX + e.name());
+  }
+  public static RMAppAttemptState convertFromProtoFormat(RMAppAttemptStateProto e) {
+    return RMAppAttemptState.valueOf(e.name().replace(RM_APP_ATTEMPT_PREFIX, ""));
+  }
+
+  private FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus s) {
+    return ProtoUtils.convertToProtoFormat(s);
+  }
+  private FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto s) {
+    return ProtoUtils.convertFromProtoFormat(s);
+  }
+
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java Thu Oct 31 17:25:37 2013
@@ -21,14 +21,20 @@ package org.apache.hadoop.yarn.server.re
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMAppStateProto;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 
 public class ApplicationStateDataPBImpl 
 extends ProtoBase<ApplicationStateDataProto> 
 implements ApplicationStateData {
-  
+  private static final RecordFactory recordFactory = RecordFactoryProvider
+      .getRecordFactory(null);
+
   ApplicationStateDataProto proto = 
             ApplicationStateDataProto.getDefaultInstance();
   ApplicationStateDataProto.Builder builder = null;
@@ -92,6 +98,18 @@ implements ApplicationStateData {
   }
 
   @Override
+  public long getStartTime() {
+    ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getStartTime();
+  }
+
+  @Override
+  public void setStartTime(long startTime) {
+    maybeInitBuilder();
+    builder.setStartTime(startTime);
+  }
+
+  @Override
   public String getUser() {
     ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
     if (!p.hasUser()) {
@@ -132,4 +150,78 @@ implements ApplicationStateData {
     this.applicationSubmissionContext = context;
   }
 
+  @Override
+  public RMAppState getState() {
+    ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasApplicationState()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getApplicationState());
+  }
+
+  @Override
+  public void setState(RMAppState finalState) {
+    maybeInitBuilder();
+    if (finalState == null) {
+      builder.clearApplicationState();
+      return;
+    }
+    builder.setApplicationState(convertToProtoFormat(finalState));
+  }
+
+  @Override
+  public String getDiagnostics() {
+    ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasDiagnostics()) {
+      return null;
+    }
+    return p.getDiagnostics();
+  }
+
+  @Override
+  public void setDiagnostics(String diagnostics) {
+    maybeInitBuilder();
+    if (diagnostics == null) {
+      builder.clearDiagnostics();
+      return;
+    }
+    builder.setDiagnostics(diagnostics);
+  }
+
+  @Override
+  public long getFinishTime() {
+    ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getFinishTime();
+  }
+
+  @Override
+  public void setFinishTime(long finishTime) {
+    maybeInitBuilder();
+    builder.setFinishTime(finishTime);
+  }
+
+  public static ApplicationStateData newApplicationStateData(long submitTime,
+      long startTime, String user,
+      ApplicationSubmissionContext submissionContext, RMAppState state,
+      String diagnostics, long finishTime) {
+
+    ApplicationStateData appState =
+        recordFactory.newRecordInstance(ApplicationStateData.class);
+    appState.setSubmitTime(submitTime);
+    appState.setStartTime(startTime);
+    appState.setUser(user);
+    appState.setApplicationSubmissionContext(submissionContext);
+    appState.setState(state);
+    appState.setDiagnostics(diagnostics);
+    appState.setFinishTime(finishTime);
+    return appState;
+  }
+
+  private static String RM_APP_PREFIX = "RMAPP_";
+  public static RMAppStateProto convertToProtoFormat(RMAppState e) {
+    return RMAppStateProto.valueOf(RM_APP_PREFIX + e.name());
+  }
+  public static RMAppState convertFromProtoFormat(RMAppStateProto e) {
+    return RMAppState.valueOf(e.name().replace(RM_APP_PREFIX, ""));
+  }
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java?rev=1537561&r1=1537560&r2=1537561&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java Thu Oct 31 17:25:37 2013
@@ -35,6 +35,7 @@ public enum RMAppEventType {
   NODE_UPDATE,
 
   // Source: RMStateStore
-  APP_SAVED,
+  APP_NEW_SAVED,
+  APP_UPDATE_SAVED,
   APP_REMOVED
 }



Mime
View raw message