Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5B04311872 for ; Fri, 18 Apr 2014 16:34:24 +0000 (UTC) Received: (qmail 73835 invoked by uid 500); 18 Apr 2014 16:34:05 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 73670 invoked by uid 500); 18 Apr 2014 16:34:01 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 73589 invoked by uid 99); 18 Apr 2014 16:33:58 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Apr 2014 16:33:58 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Apr 2014 16:33:41 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 43B6D2388C2C; Fri, 18 Apr 2014 16:32:59 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1588509 [1/2] - in /hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop... Date: Fri, 18 Apr 2014 16:32:56 -0000 To: yarn-commits@hadoop.apache.org From: cmccabe@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140418163259.43B6D2388C2C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cmccabe Date: Fri Apr 18 16:32:35 2014 New Revision: 1588509 URL: http://svn.apache.org/r1588509 Log: Merge trunk into the HADOOP-10388 branch Added: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java - copied unchanged from r1588387, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm - copied unchanged from r1588387, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRestart.apt.vm - copied unchanged from r1588387, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRestart.apt.vm Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YARN.apt.vm Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt Fri Apr 18 16:32:35 2014 @@ -21,6 +21,8 @@ Release 2.5.0 - UNRELEASED NEW FEATURES + YARN-1757. NM Recovery. Auxiliary service support. (Jason Lowe via kasha) + IMPROVEMENTS YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via @@ -36,9 +38,6 @@ Release 2.5.0 - UNRELEASED YARN-1429. *nix: Allow a way for users to augment classpath of YARN daemons. (Jarek Jarcec Cecho via kasha) - YARN-1833. TestRMAdminService Fails in trunk and branch-2 (Mit Desais via - jeagles) - YARN-1520. update capacity scheduler docs to include necessary parameters (Chen He via jeagles) @@ -48,6 +47,15 @@ Release 2.5.0 - UNRELEASED YARN-1136. Replace junit.framework.Assert with org.junit.Assert (Chen He via jeagles) + YARN-1889. In Fair Scheduler, avoid creating objects on each call to + AppSchedulable comparator (Hong Zhiguo via Sandy Ryza) + + YARN-1923. Make Fair Scheduler resource ratio calculations terminate faster + (Anubhav Dhoot via Sandy Ryza) + + YARN-1870. FileInputStream is not closed in ProcfsBasedProcessTree#constructProcessSMAPInfo. + (Fengdong Yu via junping_du) + OPTIMIZATIONS BUG FIXES @@ -57,7 +65,86 @@ Release 2.5.0 - UNRELEASED YARN-1790. Fair Scheduler UI not showing apps table (bc Wong via Sandy Ryza) -Release 2.4.0 - UNRELEASED + YARN-1784. TestContainerAllocation assumes CapacityScheduler. + (Robert Kanter via kasha) + +Release 2.4.1 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + YARN-1892. Improved some logs in the scheduler. (Jian He via zjshen) + + OPTIMIZATIONS + + BUG FIXES + + YARN-1898. Made Standby RM links conf, stacks, logLevel, metrics, jmx, logs + and static not be redirected to Active RM. (Xuan Gong via zjshen) + + YARN-1837. Fixed TestMoveApplication#testMoveRejectedByScheduler failure. + (Hong Zhiguo via jianhe) + + YARN-1905. TestProcfsBasedProcessTree must only run on Linux. (cnauroth) + + YARN-1883. TestRMAdminService fails due to inconsistent entries in + UserGroups (Mit Desai via jeagles) + + YARN-1908. Fixed DistributedShell to not fail in secure clusters. (Vinod + Kumar Vavilapalli and Jian He via vinodkv) + + YARN-1910. Fixed a race condition in TestAMRMTokens that causes the test to + fail more often on Windows. (Xuan Gong via vinodkv) + + YARN-1920. Fixed TestFileSystemApplicationHistoryStore failure on windows. + (Vinod Kumar Vavilapalli via zjshen) + + YARN-1914. Fixed resource-download on NodeManagers to skip permission + verification of public cache files in Windows+local file-system environment. + (Varun Vasudev via vinodkv) + + YARN-1903. Set exit code and diagnostics when container is killed at + NEW/LOCALIZING state. (Zhijie Shen via jianhe) + + YARN-1924. Made ZKRMStateStore updateApplication(Attempt)StateInternal work + when Application(Attempt) state hasn't been stored before. (Jian He via + zjshen) + + YARN-1926. Changed DistributedShell to use appIDs as unique identifiers for + HDFS paths and thus avoid test failures on Windows. (Varun Vasudev via + vinodkv) + + YARN-1833. TestRMAdminService Fails in trunk and branch-2 (Mit Desai via + jeagles) + + YARN-1907. TestRMApplicationHistoryWriter#testRMWritingMassiveHistory + intermittently fails. (Mit Desai via kihwal) + + YARN-1933. Fixed test issues with TestAMRestart and TestNodeHealthService. + (Jian He via vinodkv) + + YARN-1928. Fixed a race condition in TestAMRMRPCNodeUpdates which caused it + to fail occassionally. (Zhijie Shen via vinodkv) + + YARN-1934. Fixed a potential NPE in ZKRMStateStore caused by handling + Disconnected event from ZK. (Karthik Kambatla via jianhe) + + YARN-1931. Private API change in YARN-1824 in 2.4 broke compatibility + with previous releases (Sandy Ryza via tgraves) + + YARN-1750. TestNodeStatusUpdater#testNMRegistration is incorrect in test + case. (Wangda Tan via junping_du) + + YARN-1947. TestRMDelegationTokens#testRMDTMasterKeyStateOnRollingMasterKey + is failing intermittently. (Jian He via junping_du) + + YARN-1281. Fixed TestZKRMStateStoreZKClientConnections to not fail + intermittently due to ZK-client timeouts. (Tsuyoshi Ozawa via vinodkv) + +Release 2.4.0 - 2014-04-07 INCOMPATIBLE CHANGES @@ -329,6 +416,11 @@ Release 2.4.0 - UNRELEASED YARN-1452. Added documentation about the configuration and usage of generic application history and the timeline data service. (Zhijie Shen via vinodkv) + YARN-1891. Added documentation for NodeManager health-monitoring. (Varun + Vasudev via vinodkv) + + YARN-1017. Added documentation for ResourceManager Restart.(jianhe) + OPTIMIZATIONS YARN-1771. Reduce the number of NameNode operations during localization of @@ -588,6 +680,9 @@ Release 2.4.0 - UNRELEASED YARN-1873. Fixed TestDistributedShell failure when the test cases are out of order. (Mit Desai via zjshen) + YARN-1893. Mark AtMostOnce annotation to ApplicationMasterProtocol#allocate. + (Xuan Gong via jianhe) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java Fri Apr 18 16:32:35 2014 @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.io.retry.AtMostOnce; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; @@ -162,6 +163,7 @@ public interface ApplicationMasterProtoc */ @Public @Stable + @AtMostOnce public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Apr 18 16:32:35 2014 @@ -884,6 +884,13 @@ public class YarnConfiguration extends C public static final String DEFAULT_NM_USER_HOME_DIR= "/home/"; + public static final String NM_RECOVERY_PREFIX = NM_PREFIX + "recovery."; + public static final String NM_RECOVERY_ENABLED = + NM_RECOVERY_PREFIX + "enabled"; + public static final boolean DEFAULT_NM_RECOVERY_ENABLED = false; + + public static final String NM_RECOVERY_DIR = NM_RECOVERY_PREFIX + "dir"; + //////////////////////////////// // Web Proxy Configs //////////////////////////////// Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java Fri Apr 18 16:32:35 2014 @@ -22,6 +22,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; @@ -38,11 +39,22 @@ import org.apache.hadoop.yarn.conf.YarnC @Evolving public abstract class AuxiliaryService extends AbstractService { + private Path recoveryPath = null; + protected AuxiliaryService(String name) { super(name); } /** + * Get the path specific to this auxiliary service to use for recovery. + * + * @return state storage path or null if recovery is not enabled + */ + protected Path getRecoveryPath() { + return recoveryPath; + } + + /** * A new application is started on this NodeManager. This is a signal to * this {@link AuxiliaryService} about the application initialization. * @@ -102,4 +114,13 @@ public abstract class AuxiliaryService e public void stopContainer(ContainerTerminationContext stopContainerContext) { } + /** + * Set the path for this auxiliary service to use for storing state + * that will be used during recovery. + * + * @param recoveryPath where recoverable state should be stored + */ + public void setRecoveryPath(Path recoveryPath) { + this.recoveryPath = recoveryPath; + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Fri Apr 18 16:32:35 2014 @@ -27,6 +27,7 @@ import java.io.StringReader; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -91,7 +92,6 @@ import org.apache.hadoop.yarn.client.api import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; @@ -185,6 +185,9 @@ public class ApplicationMaster { @SuppressWarnings("rawtypes") private AMRMClientAsync amRMClient; + // In both secure and non-secure modes, this points to the job-submitter. + private UserGroupInformation appSubmitterUgi; + // Handle to communicate with the Node Manager private NMClientAsync nmClientAsync; // Listen to process the response from the Node Manager @@ -236,7 +239,7 @@ public class ApplicationMaster { // Location of shell script ( obtained from info set in env ) // Shell script path in fs - private String shellScriptPath = ""; + private String scriptPath = ""; // Timestamp needed for creating a local resource private long shellScriptPathTimestamp = 0; // File length needed for local resource @@ -451,7 +454,7 @@ public class ApplicationMaster { } if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) { - shellScriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION); + scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION); if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) { shellScriptPathTimestamp = Long.valueOf(envs @@ -462,10 +465,10 @@ public class ApplicationMaster { .get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)); } - if (!shellScriptPath.isEmpty() + if (!scriptPath.isEmpty() && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) { LOG.error("Illegal values in env for shell script path" + ", path=" - + shellScriptPath + ", len=" + shellScriptPathLen + ", timestamp=" + + scriptPath + ", len=" + shellScriptPathLen + ", timestamp=" + shellScriptPathTimestamp); throw new IllegalArgumentException( "Illegal values in env for shell script path"); @@ -525,14 +528,23 @@ public class ApplicationMaster { credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator> iter = credentials.getAllTokens().iterator(); + LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token token = iter.next(); + LOG.info(token); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + // Create appSubmitterUgi and add original tokens to it + String appSubmitterUserName = + System.getenv(ApplicationConstants.Environment.USER.name()); + appSubmitterUgi = + UserGroupInformation.createRemoteUser(appSubmitterUserName); + appSubmitterUgi.addCredentials(credentials); + AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); @@ -901,19 +913,26 @@ public class ApplicationMaster { // resources too. // In this scenario, if a shell script is specified, we need to have it // copied and made available to the container. - if (!shellScriptPath.isEmpty()) { - Path renamedSchellScriptPath = null; + if (!scriptPath.isEmpty()) { + Path renamedScriptPath = null; if (Shell.WINDOWS) { - renamedSchellScriptPath = new Path(shellScriptPath + ".bat"); + renamedScriptPath = new Path(scriptPath + ".bat"); } else { - renamedSchellScriptPath = new Path(shellScriptPath + ".sh"); + renamedScriptPath = new Path(scriptPath + ".sh"); } + try { - FileSystem fs = renamedSchellScriptPath.getFileSystem(conf); - fs.rename(new Path(shellScriptPath), renamedSchellScriptPath); - } catch (IOException e) { - LOG.warn("Not able to add suffix (.bat/.sh) to the shell script filename"); - throw new YarnRuntimeException(e); + // rename the script file based on the underlying OS syntax. + renameScriptFile(renamedScriptPath); + } catch (Exception e) { + LOG.error( + "Not able to add suffix (.bat/.sh) to the shell script filename", + e); + // We know we cannot continue launching the container + // so we should release it. + numCompletedContainers.incrementAndGet(); + numFailedContainers.incrementAndGet(); + return; } LocalResource shellRsrc = Records.newRecord(LocalResource.class); @@ -921,11 +940,10 @@ public class ApplicationMaster { shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION); try { shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI( - renamedSchellScriptPath.toString()))); + renamedScriptPath.toString()))); } catch (URISyntaxException e) { LOG.error("Error when trying to use shell script path specified" - + " in env, path=" + renamedSchellScriptPath); - e.printStackTrace(); + + " in env, path=" + renamedScriptPath, e); // A failure scenario on bad input such as invalid shell script path // We know we cannot continue launching the container @@ -949,7 +967,7 @@ public class ApplicationMaster { // Set executable command vargs.add(shellCommand); // Set shell script path - if (!shellScriptPath.isEmpty()) { + if (!scriptPath.isEmpty()) { vargs.add(Shell.WINDOWS ? ExecBatScripStringtPath : ExecShellStringPath); } @@ -983,6 +1001,20 @@ public class ApplicationMaster { } } + private void renameScriptFile(final Path renamedScriptPath) + throws IOException, InterruptedException { + appSubmitterUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException { + FileSystem fs = renamedScriptPath.getFileSystem(conf); + fs.rename(new Path(scriptPath), renamedScriptPath); + return null; + } + }); + LOG.info("User " + appSubmitterUgi.getUserName() + + " added suffix(.sh/.bat) to script file as " + renamedScriptPath); + } + /** * Setup the request that will be sent to the RM for the container ask. * Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Fri Apr 18 16:32:35 2014 @@ -468,12 +468,12 @@ public class Client { // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); - addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(), + addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { - addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(), + addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null); } @@ -489,7 +489,7 @@ public class Client { if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); String shellPathSuffix = - appName + "/" + appId.getId() + "/" + SCRIPT_PATH; + appName + "/" + appId.toString() + "/" + SCRIPT_PATH; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); @@ -500,12 +500,12 @@ public class Client { } if (!shellCommand.isEmpty()) { - addToLocalResources(fs, null, shellCommandPath, appId.getId(), + addToLocalResources(fs, null, shellCommandPath, appId.toString(), localResources, shellCommand); } if (shellArgs.length > 0) { - addToLocalResources(fs, null, shellArgsPath, appId.getId(), + addToLocalResources(fs, null, shellArgsPath, appId.toString(), localResources, StringUtils.join(shellArgs, " ")); } // Set local resource info into app master container launch context @@ -737,7 +737,7 @@ public class Client { } private void addToLocalResources(FileSystem fs, String fileSrcPath, - String fileDstPath, int appId, Map localResources, + String fileDstPath, String appId, Map localResources, String resources) throws IOException { String suffix = appName + "/" + appId + "/" + fileDstPath; Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Fri Apr 18 16:32:35 2014 @@ -219,7 +219,7 @@ public class TestDistributedShell { "--num_containers", "1", "--shell_command", - Shell.WINDOWS ? "timeout 8" : "sleep 8", + "sleep 8", "--master_memory", "512", "--container_memory", Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java Fri Apr 18 16:32:35 2014 @@ -33,6 +33,8 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.ClientBaseWithFixes; import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; @@ -67,14 +69,18 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; @@ -82,6 +88,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -96,6 +103,7 @@ import org.apache.hadoop.yarn.server.api import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; +import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; import org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManager; @@ -257,11 +265,13 @@ public abstract class ProtocolHATestBase } protected void startHACluster(int numOfNMs, boolean overrideClientRMService, - boolean overrideRTS) throws Exception { + boolean overrideRTS, boolean overrideApplicationMasterService) + throws Exception { conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster = new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2, - numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS); + numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS, + overrideApplicationMasterService); cluster.resetStartFailoverFlag(false); cluster.init(conf); cluster.start(); @@ -285,17 +295,19 @@ public abstract class ProtocolHATestBase private boolean overrideClientRMService; private boolean overrideRTS; + private boolean overrideApplicationMasterService; private final AtomicBoolean startFailover = new AtomicBoolean(false); private final AtomicBoolean failoverTriggered = new AtomicBoolean(false); public MiniYARNClusterForHATesting(String testName, int numResourceManagers, int numNodeManagers, int numLocalDirs, int numLogDirs, boolean enableAHS, boolean overrideClientRMService, - boolean overrideRTS) { + boolean overrideRTS, boolean overrideApplicationMasterService) { super(testName, numResourceManagers, numNodeManagers, numLocalDirs, numLogDirs, enableAHS); this.overrideClientRMService = overrideClientRMService; this.overrideRTS = overrideRTS; + this.overrideApplicationMasterService = overrideApplicationMasterService; } public boolean getStartFailoverFlag() { @@ -324,6 +336,11 @@ public abstract class ProtocolHATestBase if (count >= maximumWaittingTime) { return false; } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + // DO NOTHING + } return true; } @@ -354,6 +371,14 @@ public abstract class ProtocolHATestBase } return super.createResourceTrackerService(); } + @Override + protected ApplicationMasterService createApplicationMasterService() { + if (overrideApplicationMasterService) { + return new CustomedApplicationMasterService(this.rmContext, + this.scheduler); + } + return super.createApplicationMasterService(); + } }; } @@ -717,5 +742,31 @@ public abstract class ProtocolHATestBase return super.nodeHeartbeat(request); } } + + private class CustomedApplicationMasterService extends + ApplicationMasterService { + public CustomedApplicationMasterService(RMContext rmContext, + YarnScheduler scheduler) { + super(rmContext, scheduler); + } + + @Override + public AllocateResponse allocate(AllocateRequest request) + throws YarnException, IOException { + resetStartFailoverFlag(true); + // make sure failover has been triggered + Assert.assertTrue(waittingForFailOver()); + return createFakeAllocateResponse(); + } + + } + + public AllocateResponse createFakeAllocateResponse() { + return AllocateResponse.newInstance(-1, + new ArrayList(), + new ArrayList(), new ArrayList(), + Resource.newInstance(1024, 2), AMCommand.AM_RESYNC, 1, + null, new ArrayList()); + } } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java Fri Apr 18 16:32:35 2014 @@ -51,7 +51,7 @@ public class TestApplicationClientProtoc @Before public void initiate() throws Exception { - startHACluster(1, true, false); + startHACluster(1, true, false, false); Configuration conf = new YarnConfiguration(this.conf); client = createAndStartYarnClient(conf); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java Fri Apr 18 16:32:35 2014 @@ -270,9 +270,33 @@ public class TestRMFailover extends Clie String header = getHeader("Refresh", rm2Url); assertTrue(header.contains("; url=" + rm1Url)); + header = getHeader("Refresh", rm2Url + "/metrics"); + assertTrue(header.contains("; url=" + rm1Url)); + + header = getHeader("Refresh", rm2Url + "/jmx"); + assertTrue(header.contains("; url=" + rm1Url)); + + // standby RM links /conf, /stacks, /logLevel, /static, /logs, + // /cluster/cluster as well as webService + // /ws/v1/cluster/info should not be redirected to active RM header = getHeader("Refresh", rm2Url + "/cluster/cluster"); assertEquals(null, header); + header = getHeader("Refresh", rm2Url + "/conf"); + assertEquals(null, header); + + header = getHeader("Refresh", rm2Url + "/stacks"); + assertEquals(null, header); + + header = getHeader("Refresh", rm2Url + "/logLevel"); + assertEquals(null, header); + + header = getHeader("Refresh", rm2Url + "/static"); + assertEquals(null, header); + + header = getHeader("Refresh", rm2Url + "/logs"); + assertEquals(null, header); + header = getHeader("Refresh", rm2Url + "/ws/v1/cluster/info"); assertEquals(null, header); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java Fri Apr 18 16:32:35 2014 @@ -41,7 +41,7 @@ public class TestResourceTrackerOnHA ext @Before public void initiate() throws Exception { - startHACluster(0, false, true); + startHACluster(0, false, true, false); this.resourceTracker = getRMClient(); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Fri Apr 18 16:32:35 2014 @@ -343,9 +343,10 @@ public class TestNMClient { // getContainerStatus can be called after stopContainer try { // O is possible if CLEANUP_CONTAINER is executed too late + // 137 is possible if the container is not terminated but killed testGetContainerStatus(container, i, ContainerState.COMPLETE, "Container killed by the ApplicationMaster.", Arrays.asList( - new Integer[] {143, 0})); + new Integer[] {137, 143, 0})); } catch (YarnException e) { // The exception is possible because, after the container is stopped, // it may be removed from NM's context. Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java Fri Apr 18 16:32:35 2014 @@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; /** - * Yarn application related utilities + * Yarn internal application-related utilities */ @Private public class Apps { @@ -97,6 +97,17 @@ public class Apps { } } } + + /** + * This older version of this method is kept around for compatibility + * because downstream frameworks like Spark and Tez have been using it. + * Downstream frameworks are expected to move off of it. + */ + @Deprecated + public static void setEnvFromInputString(Map env, + String envString) { + setEnvFromInputString(env, envString, File.pathSeparator); + } @Public @Unstable @@ -112,6 +123,18 @@ public class Apps { environment.put(StringInterner.weakIntern(variable), StringInterner.weakIntern(val)); } + + /** + * This older version of this method is kept around for compatibility + * because downstream frameworks like Spark and Tez have been using it. + * Downstream frameworks are expected to move off of it. + */ + @Deprecated + public static void addToEnvironment( + Map environment, + String variable, String value) { + addToEnvironment(environment, variable, value, File.pathSeparator); + } public static String crossPlatformify(String var) { return ApplicationConstants.PARAMETER_EXPANSION_LEFT + var Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java Fri Apr 18 16:32:35 2014 @@ -36,12 +36,14 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.RunJar; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -139,6 +141,16 @@ public class FSDownload implements Calla if (!checkPublicPermsForAll(fs, sStat, FsAction.READ_EXECUTE, FsAction.READ)) { return false; } + + if (Shell.WINDOWS && fs instanceof LocalFileSystem) { + // Relax the requirement for public cache on LFS on Windows since default + // permissions are "700" all the way up to the drive letter. In this + // model, the only requirement for a user is to give EVERYONE group + // permission on the file and the file will be considered public. + // This code path is only hit when fs.default.name is file:/// (mainly + // in tests). + return true; + } return ancestorsHaveExecutePermissions(fs, current.getParent(), statCache); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java Fri Apr 18 16:32:35 2014 @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.util; import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; @@ -736,7 +735,7 @@ public class ProcfsBasedProcessTree exte fReader = new FileReader(file); in = new BufferedReader(fReader); ProcessSmapMemoryInfo memoryMappingInfo = null; - List lines = IOUtils.readLines(new FileInputStream(file)); + List lines = IOUtils.readLines(in); for (String line : lines) { line = line.trim(); try { Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java Fri Apr 18 16:32:35 2014 @@ -43,8 +43,7 @@ public class TimelineUtils { mapper = new ObjectMapper(); AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); mapper.setAnnotationIntrospector(introspector); - mapper.getSerializationConfig() - .setSerializationInclusion(Inclusion.NON_NULL); + mapper.setSerializationInclusion(Inclusion.NON_NULL); } /** @@ -77,7 +76,7 @@ public class TimelineUtils { public static String dumpTimelineRecordtoJSON(Object o, boolean pretty) throws JsonGenerationException, JsonMappingException, IOException { if (pretty) { - return mapper.defaultPrettyPrintingWriter().writeValueAsString(o); + return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(o); } else { return mapper.writeValueAsString(o); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java Fri Apr 18 16:32:35 2014 @@ -51,8 +51,7 @@ public class YarnJacksonJaxbJsonProvider ObjectMapper mapper = super.locateMapper(type, mediaType); AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); mapper.setAnnotationIntrospector(introspector); - mapper.getSerializationConfig() - .setSerializationInclusion(Inclusion.NON_NULL); + mapper.setSerializationInclusion(Inclusion.NON_NULL); return mapper; } } \ No newline at end of file Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Apr 18 16:32:35 2014 @@ -1019,6 +1019,19 @@ 500 + + Enable the node manager to recover after starting + yarn.nodemanager.recovery.enabled + false + + + + The local filesystem directory in which the node manager will + store state when recovery is enabled. + yarn.nodemanager.recovery.dir + ${hadoop.tmp.dir}/yarn-nm-recovery + + yarn.nodemanager.aux-services.mapreduce_shuffle.class Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java Fri Apr 18 16:32:35 2014 @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.util; import static org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.KB_TO_BYTES; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -109,19 +110,13 @@ public class TestProcfsBasedProcessTree @Before public void setup() throws IOException { + assumeTrue(Shell.LINUX); FileContext.getLocalFSFileContext().delete( new Path(TEST_ROOT_DIR.getAbsolutePath()), true); } @Test(timeout = 30000) public void testProcessTree() throws Exception { - - if (!Shell.LINUX) { - System.out - .println("ProcfsBasedProcessTree is not available on this system. Not testing"); - return; - - } try { Assert.assertTrue(ProcfsBasedProcessTree.isAvailable()); } catch (Exception e) { Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java Fri Apr 18 16:32:35 2014 @@ -179,7 +179,7 @@ public class FileSystemApplicationHistor LOG.info("Completed reading history information of application " + appId); return historyData; } catch (IOException e) { - LOG.error("Error when reading history file of application " + appId); + LOG.error("Error when reading history file of application " + appId, e); throw e; } finally { hfReader.close(); @@ -296,7 +296,7 @@ public class FileSystemApplicationHistor return historyData; } catch (IOException e) { LOG.error("Error when reading history file of application attempt" - + appAttemptId); + + appAttemptId, e); throw e; } finally { hfReader.close(); @@ -344,7 +344,7 @@ public class FileSystemApplicationHistor + containerId); return historyData; } catch (IOException e) { - LOG.error("Error when reading history file of container " + containerId); + LOG.error("Error when reading history file of container " + containerId, e); throw e; } finally { hfReader.close(); @@ -420,7 +420,7 @@ public class FileSystemApplicationHistor + appStart.getApplicationId()); } catch (IOException e) { LOG.error("Error when openning history file of application " - + appStart.getApplicationId()); + + appStart.getApplicationId(), e); throw e; } outstandingWriters.put(appStart.getApplicationId(), hfWriter); @@ -437,7 +437,7 @@ public class FileSystemApplicationHistor + appStart.getApplicationId() + " is written"); } catch (IOException e) { LOG.error("Error when writing start information of application " - + appStart.getApplicationId()); + + appStart.getApplicationId(), e); throw e; } } @@ -456,7 +456,7 @@ public class FileSystemApplicationHistor + appFinish.getApplicationId() + " is written"); } catch (IOException e) { LOG.error("Error when writing finish information of application " - + appFinish.getApplicationId()); + + appFinish.getApplicationId(), e); throw e; } finally { hfWriter.close(); @@ -480,7 +480,7 @@ public class FileSystemApplicationHistor + appAttemptStart.getApplicationAttemptId() + " is written"); } catch (IOException e) { LOG.error("Error when writing start information of application attempt " - + appAttemptStart.getApplicationAttemptId()); + + appAttemptStart.getApplicationAttemptId(), e); throw e; } } @@ -501,7 +501,7 @@ public class FileSystemApplicationHistor + appAttemptFinish.getApplicationAttemptId() + " is written"); } catch (IOException e) { LOG.error("Error when writing finish information of application attempt " - + appAttemptFinish.getApplicationAttemptId()); + + appAttemptFinish.getApplicationAttemptId(), e); throw e; } } @@ -521,7 +521,7 @@ public class FileSystemApplicationHistor + containerStart.getContainerId() + " is written"); } catch (IOException e) { LOG.error("Error when writing start information of container " - + containerStart.getContainerId()); + + containerStart.getContainerId(), e); throw e; } } @@ -541,7 +541,7 @@ public class FileSystemApplicationHistor + containerFinish.getContainerId() + " is written"); } catch (IOException e) { LOG.error("Error when writing finish information of container " - + containerFinish.getContainerId()); + + containerFinish.getContainerId(), e); } } @@ -676,9 +676,10 @@ public class FileSystemApplicationHistor private TFile.Reader reader; private TFile.Reader.Scanner scanner; + FSDataInputStream fsdis; public HistoryFileReader(Path historyFile) throws IOException { - FSDataInputStream fsdis = fs.open(historyFile); + fsdis = fs.open(historyFile); reader = new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(), getConfig()); @@ -707,7 +708,7 @@ public class FileSystemApplicationHistor } public void close() { - IOUtils.cleanup(LOG, scanner, reader); + IOUtils.cleanup(LOG, scanner, reader, fsdis); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java Fri Apr 18 16:32:35 2014 @@ -23,6 +23,8 @@ import java.net.URI; import org.junit.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -42,6 +44,9 @@ import org.junit.Test; public class TestFileSystemApplicationHistoryStore extends ApplicationHistoryStoreTestUtils { + private static Log LOG = LogFactory + .getLog(TestFileSystemApplicationHistoryStore.class.getName()); + private FileSystem fs; private Path fsWorkingPath; @@ -50,9 +55,12 @@ public class TestFileSystemApplicationHi fs = new RawLocalFileSystem(); Configuration conf = new Configuration(); fs.initialize(new URI("/"), conf); - fsWorkingPath = new Path("Test"); + fsWorkingPath = + new Path("target", + TestFileSystemApplicationHistoryStore.class.getSimpleName()); fs.delete(fsWorkingPath, true); - conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString()); + conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, + fsWorkingPath.toString()); store = new FileSystemApplicationHistoryStore(); store.init(conf); store.start(); @@ -67,6 +75,7 @@ public class TestFileSystemApplicationHi @Test public void testReadWriteHistoryData() throws IOException { + LOG.info("Starting testReadWriteHistoryData"); testWriteHistoryData(5); testReadHistoryData(5); } @@ -167,6 +176,7 @@ public class TestFileSystemApplicationHi @Test public void testWriteAfterApplicationFinish() throws IOException { + LOG.info("Starting testWriteAfterApplicationFinish"); ApplicationId appId = ApplicationId.newInstance(0, 1); writeApplicationStartData(appId); writeApplicationFinishData(appId); @@ -203,6 +213,7 @@ public class TestFileSystemApplicationHi @Test public void testMassiveWriteContainerHistoryData() throws IOException { + LOG.info("Starting testMassiveWriteContainerHistoryData"); long mb = 1024 * 1024; long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb; ApplicationId appId = ApplicationId.newInstance(0, 1); @@ -221,12 +232,14 @@ public class TestFileSystemApplicationHi @Test public void testMissingContainerHistoryData() throws IOException { + LOG.info("Starting testMissingContainerHistoryData"); testWriteHistoryData(3, true, false); testReadHistoryData(3, true, false); } @Test public void testMissingApplicationAttemptHistoryData() throws IOException { + LOG.info("Starting testMissingApplicationAttemptHistoryData"); testWriteHistoryData(3, false, true); testReadHistoryData(3, false, true); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Fri Apr 18 16:32:35 2014 @@ -28,6 +28,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; @@ -127,6 +130,20 @@ public class NodeManager extends Composi conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); + boolean recoveryEnabled = conf.getBoolean( + YarnConfiguration.NM_RECOVERY_ENABLED, + YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED); + if (recoveryEnabled) { + FileSystem recoveryFs = FileSystem.getLocal(conf); + String recoveryDirName = conf.get(YarnConfiguration.NM_RECOVERY_DIR); + if (recoveryDirName == null) { + throw new IllegalArgumentException("Recovery is enabled but " + + YarnConfiguration.NM_RECOVERY_DIR + " is not set."); + } + Path recoveryRoot = new Path(recoveryDirName); + recoveryFs.mkdirs(recoveryRoot, new FsPermission((short)0700)); + } + NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager(conf); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Fri Apr 18 16:32:35 2014 @@ -343,10 +343,6 @@ public class NodeStatusUpdaterImpl exten + ", " + nodeHealthStatus.getHealthReport()); } List containersStatuses = getContainerStatuses(); - if (LOG.isDebugEnabled()) { - LOG.debug(this.nodeId + " sending out status for " - + containersStatuses.size() + " containers"); - } NodeStatus nodeStatus = NodeStatus.newInstance(nodeId, responseId, containersStatuses, createKeepAliveApplicationList(), nodeHealthStatus); @@ -373,7 +369,8 @@ public class NodeStatusUpdaterImpl exten } } if (LOG.isDebugEnabled()) { - LOG.debug("Sending out container statuses: " + containerStatuses); + LOG.debug("Sending out " + containerStatuses.size() + + " container statuses: " + containerStatuses); } return containerStatuses; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java Fri Apr 18 16:32:35 2014 @@ -29,15 +29,18 @@ import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.Service; import org.apache.hadoop.service.ServiceStateChangeListener; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; import org.apache.hadoop.yarn.server.api.AuxiliaryService; -import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; @@ -46,6 +49,8 @@ import com.google.common.base.Preconditi public class AuxServices extends AbstractService implements ServiceStateChangeListener, EventHandler { + static final String STATE_STORE_ROOT_NAME = "nm-aux-services"; + private static final Log LOG = LogFactory.getLog(AuxServices.class); protected final Map serviceMap; @@ -91,6 +96,17 @@ public class AuxServices extends Abstrac @Override public void serviceInit(Configuration conf) throws Exception { + final FsPermission storeDirPerms = new FsPermission((short)0700); + Path stateStoreRoot = null; + FileSystem stateStoreFs = null; + boolean recoveryEnabled = conf.getBoolean( + YarnConfiguration.NM_RECOVERY_ENABLED, + YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED); + if (recoveryEnabled) { + stateStoreRoot = new Path(conf.get(YarnConfiguration.NM_RECOVERY_DIR), + STATE_STORE_ROOT_NAME); + stateStoreFs = FileSystem.getLocal(conf); + } Collection auxNames = conf.getStringCollection( YarnConfiguration.NM_AUX_SERVICES); for (final String sName : auxNames) { @@ -119,6 +135,11 @@ public class AuxServices extends Abstrac +"the name in the config."); } addService(sName, s); + if (recoveryEnabled) { + Path storePath = new Path(stateStoreRoot, sName); + stateStoreFs.mkdirs(storePath, storeDirPerms); + s.setRecoveryPath(storePath); + } s.init(conf); } catch (RuntimeException e) { LOG.fatal("Failed to initialize " + sName, e); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Fri Apr 18 16:32:35 2014 @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.conf.YarnC import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent; @@ -141,7 +142,7 @@ public class ContainerImpl implements Co ContainerEventType.UPDATE_DIAGNOSTICS_MSG, UPDATE_DIAGNOSTICS_TRANSITION) .addTransition(ContainerState.NEW, ContainerState.DONE, - ContainerEventType.KILL_CONTAINER, CONTAINER_DONE_TRANSITION) + ContainerEventType.KILL_CONTAINER, new KillOnNewTransition()) // From LOCALIZING State .addTransition(ContainerState.LOCALIZING, @@ -760,7 +761,9 @@ public class ContainerImpl implements Co container.cleanup(); container.metrics.endInitingContainer(); ContainerKillEvent killEvent = (ContainerKillEvent) event; + container.exitCode = ExitCode.TERMINATED.getExitCode(); container.diagnostics.append(killEvent.getDiagnostic()).append("\n"); + container.diagnostics.append("Container is killed before being launched.\n"); } } @@ -828,7 +831,6 @@ public class ContainerImpl implements Co /** * Handle the following transitions: - * - NEW -> DONE upon KILL_CONTAINER * - {LOCALIZATION_FAILED, EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE, * KILLING, CONTAINER_CLEANEDUP_AFTER_KILL} * -> DONE upon CONTAINER_RESOURCES_CLEANEDUP @@ -850,6 +852,21 @@ public class ContainerImpl implements Co } /** + * Handle the following transition: + * - NEW -> DONE upon KILL_CONTAINER + */ + static class KillOnNewTransition extends ContainerDoneTransition { + @Override + public void transition(ContainerImpl container, ContainerEvent event) { + ContainerKillEvent killEvent = (ContainerKillEvent) event; + container.exitCode = ExitCode.TERMINATED.getExitCode(); + container.diagnostics.append(killEvent.getDiagnostic()).append("\n"); + container.diagnostics.append("Container is killed before being launched.\n"); + super.transition(container, event); + } + } + + /** * Update diagnostics, staying in the same state. */ static class ContainerDiagnosticsUpdateTransition implements Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java Fri Apr 18 16:32:35 2014 @@ -78,11 +78,17 @@ public class TestNodeHealthService { } private void writeNodeHealthScriptFile(String scriptStr, boolean setExecutable) - throws IOException { - PrintWriter pw = new PrintWriter(new FileOutputStream(nodeHealthscriptFile)); - pw.println(scriptStr); - pw.flush(); - pw.close(); + throws IOException { + PrintWriter pw = null; + try { + FileUtil.setWritable(nodeHealthscriptFile, true); + FileUtil.setReadable(nodeHealthscriptFile, true); + pw = new PrintWriter(new FileOutputStream(nodeHealthscriptFile)); + pw.println(scriptStr); + pw.flush(); + } finally { + pw.close(); + } FileUtil.setExecutable(nodeHealthscriptFile, setExecutable); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1588509&r1=1588508&r2=1588509&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Fri Apr 18 16:32:35 2014 @@ -261,10 +261,12 @@ public class TestNodeStatusUpdater { this.context.getContainers().put(secondContainerID, container); } else if (heartBeatID == 3) { // Checks on the RM end - Assert.assertEquals("Number of applications should only be one!", 1, + Assert.assertEquals("Number of applications should have two!", 2, appToContainers.size()); - Assert.assertEquals("Number of container for the app should be two!", - 2, appToContainers.get(appId2).size()); + Assert.assertEquals("Number of container for the app-1 should be only one!", + 1, appToContainers.get(appId1).size()); + Assert.assertEquals("Number of container for the app-2 should be only one!", + 1, appToContainers.get(appId2).size()); // Checks on the NM end ConcurrentMap activeContainers =