Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6B12010D97 for ; Fri, 11 Apr 2014 03:38:54 +0000 (UTC) Received: (qmail 71743 invoked by uid 500); 11 Apr 2014 03:38:53 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 71697 invoked by uid 500); 11 Apr 2014 03:38:52 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 71689 invoked by uid 99); 11 Apr 2014 03:38:52 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 11 Apr 2014 03:38:52 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 11 Apr 2014 03:38:50 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id BDE32238890B; Fri, 11 Apr 2014 03:38:27 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1586548 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ hadoop-yarn/hadoop-yarn-server/ha... Date: Fri, 11 Apr 2014 03:38:27 -0000 To: yarn-commits@hadoop.apache.org From: zjshen@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140411033827.BDE32238890B@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: zjshen Date: Fri Apr 11 03:38:27 2014 New Revision: 1586548 URL: http://svn.apache.org/r1586548 Log: YARN-1924. Made ZKRMStateStore updateApplication(Attempt)StateInternal work when Application(Attempt) state hasn't been stored before. Contributed by Jian He. svn merge --ignore-ancestry -c 1586547 ../../trunk/ Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1586548&r1=1586547&r2=1586548&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Apr 11 03:38:27 2014 @@ -92,6 +92,10 @@ Release 2.4.1 - UNRELEASED YARN-1903. Set exit code and diagnostics when container is killed at NEW/LOCALIZING state. (Zhijie Shen via jianhe) + YARN-1924. Made ZKRMStateStore updateApplication(Attempt)StateInternal work + when Application(Attempt) state hasn't been stored before. (Jian He via + zjshen) + Release 2.4.0 - 2014-04-07 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1586548&r1=1586547&r2=1586548&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java Fri Apr 11 03:38:27 2014 @@ -538,6 +538,8 @@ public class FileSystemRMStateStore exte protected void replaceFile(Path srcPath, Path dstPath) throws Exception { if (fs.exists(dstPath)) { deleteFile(dstPath); + } else { + LOG.info("File doesn't exist. Skip deleting the file " + dstPath); } fs.rename(srcPath, dstPath); } Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1586548&r1=1586547&r2=1586548&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java Fri Apr 11 03:38:27 2014 @@ -628,7 +628,7 @@ public abstract class RMStateStore exten notifyDoneUpdatingApplication(appId, storedException); } } catch (Exception e) { - LOG.error("Error storing app: " + appId, e); + LOG.error("Error storing/updating app: " + appId, e); notifyStoreOperationFailed(e); } } else if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT) @@ -679,7 +679,7 @@ public abstract class RMStateStore exten } } catch (Exception e) { LOG.error( - "Error storing appAttempt: " + attemptState.getAttemptId(), e); + "Error storing/updating appAttempt: " + attemptState.getAttemptId(), e); notifyStoreOperationFailed(e); } } else if (event.getType().equals(RMStateStoreEventType.REMOVE_APP)) { Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1586548&r1=1586547&r2=1586548&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java Fri Apr 11 03:38:27 2014 @@ -566,7 +566,15 @@ public class ZKRMStateStore extends RMSt + nodeUpdatePath); } byte[] appStateData = appStateDataPB.getProto().toByteArray(); - setDataWithRetries(nodeUpdatePath, appStateData, 0); + + if (zkClient.exists(nodeUpdatePath, true) != null) { + setDataWithRetries(nodeUpdatePath, appStateData, -1); + } else { + createWithRetries(nodeUpdatePath, appStateData, zkAcl, + CreateMode.PERSISTENT); + LOG.info(appId + " znode didn't exist. Created a new znode to" + + " update the application state."); + } } @Override @@ -601,7 +609,15 @@ public class ZKRMStateStore extends RMSt + " at: " + nodeUpdatePath); } byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray(); - setDataWithRetries(nodeUpdatePath, attemptStateData, 0); + + if (zkClient.exists(nodeUpdatePath, true) != null) { + setDataWithRetries(nodeUpdatePath, attemptStateData, -1); + } else { + createWithRetries(nodeUpdatePath, attemptStateData, zkAcl, + CreateMode.PERSISTENT); + LOG.info(appAttemptId + " znode didn't exist. Created a new znode to" + + " update the application attempt state."); + } } @Override @@ -961,6 +977,7 @@ public class ZKRMStateStore extends RMSt Thread.sleep(zkRetryInterval); continue; } + LOG.error("Error while doing ZK operation.", ke); throw ke; } } Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java?rev=1586548&r1=1586547&r2=1586548&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java Fri Apr 11 03:38:27 2014 @@ -310,6 +310,30 @@ public class RMStateStoreTestBase extend "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED); store.updateApplicationAttemptState(newAttemptState); + + // test updating the state of an app/attempt whose initial state was not + // saved. + ApplicationId dummyAppId = ApplicationId.newInstance(1234, 10); + ApplicationSubmissionContext dummyContext = + new ApplicationSubmissionContextPBImpl(); + dummyContext.setApplicationId(dummyAppId); + ApplicationState dummyApp = + new ApplicationState(appState.submitTime, appState.startTime, + dummyContext, appState.user, RMAppState.FINISHED, "appDiagnostics", + 1234); + store.updateApplicationState(dummyApp); + + ApplicationAttemptId dummyAttemptId = + ApplicationAttemptId.newInstance(dummyAppId, 6); + ApplicationAttemptState dummyAttempt = + new ApplicationAttemptState(dummyAttemptId, + oldAttemptState.getMasterContainer(), + oldAttemptState.getAppAttemptCredentials(), + oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, + "myTrackingUrl", "attemptDiagnostics", + FinalApplicationStatus.SUCCEEDED); + store.updateApplicationAttemptState(dummyAttempt); + // let things settle down Thread.sleep(1000); store.close(); @@ -320,6 +344,7 @@ public class RMStateStoreTestBase extend RMState newRMState = store.loadState(); Map newRMAppState = newRMState.getApplicationState(); + assertNotNull(newRMAppState.get(dummyApp.getAppId())); ApplicationState updatedAppState = newRMAppState.get(appId1); assertEquals(appState.getAppId(),updatedAppState.getAppId()); assertEquals(appState.getSubmitTime(), updatedAppState.getSubmitTime()); @@ -331,6 +356,8 @@ public class RMStateStoreTestBase extend assertEquals(1234, updatedAppState.getFinishTime()); // check updated attempt state + assertNotNull(newRMAppState.get(dummyApp.getAppId()).getAttempt( + dummyAttemptId)); ApplicationAttemptState updatedAttemptState = updatedAppState.getAttempt(newAttemptState.getAttemptId()); assertEquals(oldAttemptState.getAttemptId(),