hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjs...@apache.org
Subject svn commit: r1586548 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ hadoop-yarn/hadoop-yarn-server/ha...
Date Fri, 11 Apr 2014 03:38:27 GMT
Author: zjshen
Date: Fri Apr 11 03:38:27 2014
New Revision: 1586548

URL: http://svn.apache.org/r1586548
Log:
YARN-1924. Made ZKRMStateStore updateApplication(Attempt)StateInternal work when Application(Attempt)
state hasn't been stored before. Contributed by Jian He.
svn merge --ignore-ancestry -c 1586547 ../../trunk/

Modified:
    hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1586548&r1=1586547&r2=1586548&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Apr 11 03:38:27 2014
@@ -92,6 +92,10 @@ Release 2.4.1 - UNRELEASED
     YARN-1903. Set exit code and diagnostics when container is killed at
     NEW/LOCALIZING state. (Zhijie Shen via jianhe)
 
+    YARN-1924. Made ZKRMStateStore updateApplication(Attempt)StateInternal work
+    when Application(Attempt) state hasn't been stored before. (Jian He via
+    zjshen)
+
 Release 2.4.0 - 2014-04-07 
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1586548&r1=1586547&r2=1586548&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
Fri Apr 11 03:38:27 2014
@@ -538,6 +538,8 @@ public class FileSystemRMStateStore exte
   protected void replaceFile(Path srcPath, Path dstPath) throws Exception {
     if (fs.exists(dstPath)) {
       deleteFile(dstPath);
+    } else {
+      LOG.info("File doesn't exist. Skip deleting the file " + dstPath);
     }
     fs.rename(srcPath, dstPath);
   }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1586548&r1=1586547&r2=1586548&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
Fri Apr 11 03:38:27 2014
@@ -628,7 +628,7 @@ public abstract class RMStateStore exten
           notifyDoneUpdatingApplication(appId, storedException);
         }
       } catch (Exception e) {
-        LOG.error("Error storing app: " + appId, e);
+        LOG.error("Error storing/updating app: " + appId, e);
         notifyStoreOperationFailed(e);
       }
     } else if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT)
@@ -679,7 +679,7 @@ public abstract class RMStateStore exten
         }
       } catch (Exception e) {
         LOG.error(
-            "Error storing appAttempt: " + attemptState.getAttemptId(), e);
+            "Error storing/updating appAttempt: " + attemptState.getAttemptId(), e);
         notifyStoreOperationFailed(e);
       }
     } else if (event.getType().equals(RMStateStoreEventType.REMOVE_APP)) {

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1586548&r1=1586547&r2=1586548&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
Fri Apr 11 03:38:27 2014
@@ -566,7 +566,15 @@ public class ZKRMStateStore extends RMSt
           + nodeUpdatePath);
     }
     byte[] appStateData = appStateDataPB.getProto().toByteArray();
-    setDataWithRetries(nodeUpdatePath, appStateData, 0);
+
+    if (zkClient.exists(nodeUpdatePath, true) != null) {
+      setDataWithRetries(nodeUpdatePath, appStateData, -1);
+    } else {
+      createWithRetries(nodeUpdatePath, appStateData, zkAcl,
+        CreateMode.PERSISTENT);
+      LOG.info(appId + " znode didn't exist. Created a new znode to"
+          + " update the application state.");
+    }
   }
 
   @Override
@@ -601,7 +609,15 @@ public class ZKRMStateStore extends RMSt
           + " at: " + nodeUpdatePath);
     }
     byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
-    setDataWithRetries(nodeUpdatePath, attemptStateData, 0);
+
+    if (zkClient.exists(nodeUpdatePath, true) != null) {
+      setDataWithRetries(nodeUpdatePath, attemptStateData, -1);
+    } else {
+      createWithRetries(nodeUpdatePath, attemptStateData, zkAcl,
+        CreateMode.PERSISTENT);
+      LOG.info(appAttemptId + " znode didn't exist. Created a new znode to"
+          + " update the application attempt state.");
+    }
   }
 
   @Override
@@ -961,6 +977,7 @@ public class ZKRMStateStore extends RMSt
             Thread.sleep(zkRetryInterval);
             continue;
           }
+          LOG.error("Error while doing ZK operation.", ke);
           throw ke;
         }
       }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java?rev=1586548&r1=1586547&r2=1586548&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
Fri Apr 11 03:38:27 2014
@@ -310,6 +310,30 @@ public class RMStateStoreTestBase extend
           "myTrackingUrl", "attemptDiagnostics",
           FinalApplicationStatus.SUCCEEDED);
     store.updateApplicationAttemptState(newAttemptState);
+
+    // test updating the state of an app/attempt whose initial state was not
+    // saved.
+    ApplicationId dummyAppId = ApplicationId.newInstance(1234, 10);
+    ApplicationSubmissionContext dummyContext =
+        new ApplicationSubmissionContextPBImpl();
+    dummyContext.setApplicationId(dummyAppId);
+    ApplicationState dummyApp =
+        new ApplicationState(appState.submitTime, appState.startTime,
+          dummyContext, appState.user, RMAppState.FINISHED, "appDiagnostics",
+          1234);
+    store.updateApplicationState(dummyApp);
+
+    ApplicationAttemptId dummyAttemptId =
+        ApplicationAttemptId.newInstance(dummyAppId, 6);
+    ApplicationAttemptState dummyAttempt =
+        new ApplicationAttemptState(dummyAttemptId,
+          oldAttemptState.getMasterContainer(),
+          oldAttemptState.getAppAttemptCredentials(),
+          oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
+          "myTrackingUrl", "attemptDiagnostics",
+          FinalApplicationStatus.SUCCEEDED);
+    store.updateApplicationAttemptState(dummyAttempt);
+
     // let things settle down
     Thread.sleep(1000);
     store.close();
@@ -320,6 +344,7 @@ public class RMStateStoreTestBase extend
     RMState newRMState = store.loadState();
     Map<ApplicationId, ApplicationState> newRMAppState =
         newRMState.getApplicationState();
+    assertNotNull(newRMAppState.get(dummyApp.getAppId()));
     ApplicationState updatedAppState = newRMAppState.get(appId1);
     assertEquals(appState.getAppId(),updatedAppState.getAppId());
     assertEquals(appState.getSubmitTime(), updatedAppState.getSubmitTime());
@@ -331,6 +356,8 @@ public class RMStateStoreTestBase extend
     assertEquals(1234, updatedAppState.getFinishTime());
 
     // check updated attempt state
+    assertNotNull(newRMAppState.get(dummyApp.getAppId()).getAttempt(
+      dummyAttemptId));
     ApplicationAttemptState updatedAttemptState =
         updatedAppState.getAttempt(newAttemptState.getAttemptId());
     assertEquals(oldAttemptState.getAttemptId(),



Mime
View raw message