hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject svn commit: r1588509 [1/2] - in /hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop...
Date Fri, 18 Apr 2014 16:32:56 GMT
Author: cmccabe
Date: Fri Apr 18 16:32:35 2014
New Revision: 1588509

URL: http://svn.apache.org/r1588509
Log:
Merge trunk into the HADOOP-10388 branch

Added:
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
      - copied unchanged from r1588387, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm
      - copied unchanged from r1588387, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRestart.apt.vm
      - copied unchanged from r1588387, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRestart.apt.vm
Modified:
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
    hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YARN.apt.vm

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt Fri Apr 18 16:32:35 2014
@@ -21,6 +21,8 @@ Release 2.5.0 - UNRELEASED
 
   NEW FEATURES
 
+    YARN-1757. NM Recovery. Auxiliary service support. (Jason Lowe via kasha)
+
   IMPROVEMENTS
 
     YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
@@ -36,9 +38,6 @@ Release 2.5.0 - UNRELEASED
     YARN-1429. *nix: Allow a way for users to augment classpath of YARN daemons.
     (Jarek Jarcec Cecho via kasha)
 
-    YARN-1833. TestRMAdminService Fails in trunk and branch-2 (Mit Desais via
-    jeagles)
-
     YARN-1520. update capacity scheduler docs to include necessary parameters
     (Chen He via jeagles)
 
@@ -48,6 +47,15 @@ Release 2.5.0 - UNRELEASED
     YARN-1136. Replace junit.framework.Assert with org.junit.Assert (Chen He
     via jeagles)
 
+    YARN-1889. In Fair Scheduler, avoid creating objects on each call to
+    AppSchedulable comparator (Hong Zhiguo via Sandy Ryza)
+
+    YARN-1923. Make Fair Scheduler resource ratio calculations terminate faster
+    (Anubhav Dhoot via Sandy Ryza)
+
+    YARN-1870. FileInputStream is not closed in ProcfsBasedProcessTree#constructProcessSMAPInfo. 
+    (Fengdong Yu via junping_du)
+
   OPTIMIZATIONS
 
   BUG FIXES 
@@ -57,7 +65,86 @@ Release 2.5.0 - UNRELEASED
 
     YARN-1790. Fair Scheduler UI not showing apps table (bc Wong via Sandy Ryza)
 
-Release 2.4.0 - UNRELEASED
+    YARN-1784. TestContainerAllocation assumes CapacityScheduler.
+    (Robert Kanter via kasha)
+
+Release 2.4.1 - UNRELEASED
+
+  INCOMPATIBLE CHANGES
+
+  NEW FEATURES
+
+  IMPROVEMENTS
+
+    YARN-1892. Improved some logs in the scheduler. (Jian He via zjshen)
+
+  OPTIMIZATIONS
+
+  BUG FIXES
+
+    YARN-1898. Made Standby RM links conf, stacks, logLevel, metrics, jmx, logs
+    and static not be redirected to Active RM. (Xuan Gong via zjshen)
+
+    YARN-1837. Fixed TestMoveApplication#testMoveRejectedByScheduler failure.
+    (Hong Zhiguo via jianhe)
+
+    YARN-1905. TestProcfsBasedProcessTree must only run on Linux. (cnauroth)
+
+    YARN-1883. TestRMAdminService fails due to inconsistent entries in
+    UserGroups (Mit Desai via jeagles)
+
+    YARN-1908. Fixed DistributedShell to not fail in secure clusters. (Vinod
+    Kumar Vavilapalli and Jian He via vinodkv)
+
+    YARN-1910. Fixed a race condition in TestAMRMTokens that causes the test to
+    fail more often on Windows. (Xuan Gong via vinodkv)
+
+    YARN-1920. Fixed TestFileSystemApplicationHistoryStore failure on windows.
+    (Vinod Kumar Vavilapalli via zjshen)
+
+    YARN-1914. Fixed resource-download on NodeManagers to skip permission
+    verification of public cache files in Windows+local file-system environment.
+    (Varun Vasudev via vinodkv)
+
+    YARN-1903. Set exit code and diagnostics when container is killed at
+    NEW/LOCALIZING state. (Zhijie Shen via jianhe)
+
+    YARN-1924. Made ZKRMStateStore updateApplication(Attempt)StateInternal work
+    when Application(Attempt) state hasn't been stored before. (Jian He via
+    zjshen)
+
+    YARN-1926. Changed DistributedShell to use appIDs as unique identifiers for
+    HDFS paths and thus avoid test failures on Windows. (Varun Vasudev via
+    vinodkv)
+
+    YARN-1833. TestRMAdminService Fails in trunk and branch-2 (Mit Desai via
+    jeagles)
+
+    YARN-1907. TestRMApplicationHistoryWriter#testRMWritingMassiveHistory 
+    intermittently fails. (Mit Desai via kihwal)
+
+    YARN-1933. Fixed test issues with TestAMRestart and TestNodeHealthService.
+    (Jian He via vinodkv)
+
+    YARN-1928. Fixed a race condition in TestAMRMRPCNodeUpdates which caused it
+    to fail occassionally. (Zhijie Shen via vinodkv)
+
+    YARN-1934. Fixed a potential NPE in ZKRMStateStore caused by handling
+    Disconnected event from ZK. (Karthik Kambatla via jianhe)
+
+    YARN-1931. Private API change in YARN-1824 in 2.4 broke compatibility 
+    with previous releases (Sandy Ryza via tgraves)
+
+    YARN-1750. TestNodeStatusUpdater#testNMRegistration is incorrect in test 
+    case. (Wangda Tan via junping_du)
+
+    YARN-1947. TestRMDelegationTokens#testRMDTMasterKeyStateOnRollingMasterKey 
+    is failing intermittently. (Jian He via junping_du)
+
+    YARN-1281. Fixed TestZKRMStateStoreZKClientConnections to not fail
+    intermittently due to ZK-client timeouts. (Tsuyoshi Ozawa via vinodkv) 
+
+Release 2.4.0 - 2014-04-07 
 
   INCOMPATIBLE CHANGES
 
@@ -329,6 +416,11 @@ Release 2.4.0 - UNRELEASED
     YARN-1452. Added documentation about the configuration and usage of generic
     application history and the timeline data service. (Zhijie Shen via vinodkv)
 
+    YARN-1891. Added documentation for NodeManager health-monitoring. (Varun
+    Vasudev via vinodkv)
+
+    YARN-1017. Added documentation for ResourceManager Restart.(jianhe)
+
   OPTIMIZATIONS
 
     YARN-1771. Reduce the number of NameNode operations during localization of
@@ -588,6 +680,9 @@ Release 2.4.0 - UNRELEASED
     YARN-1873. Fixed TestDistributedShell failure when the test cases are out of
     order. (Mit Desai via zjshen)
 
+    YARN-1893. Mark AtMostOnce annotation to ApplicationMasterProtocol#allocate.
+    (Xuan Gong via jianhe)
+
 Release 2.3.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java Fri Apr 18 16:32:35 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.io.retry.AtMostOnce;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
@@ -162,6 +163,7 @@ public interface ApplicationMasterProtoc
    */
   @Public
   @Stable
+  @AtMostOnce
   public AllocateResponse allocate(AllocateRequest request) 
   throws YarnException, IOException;
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Apr 18 16:32:35 2014
@@ -884,6 +884,13 @@ public class YarnConfiguration extends C
   
   public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
 
+  public static final String NM_RECOVERY_PREFIX = NM_PREFIX + "recovery.";
+  public static final String NM_RECOVERY_ENABLED =
+      NM_RECOVERY_PREFIX + "enabled";
+  public static final boolean DEFAULT_NM_RECOVERY_ENABLED = false;
+
+  public static final String NM_RECOVERY_DIR = NM_RECOVERY_PREFIX + "dir";
+
   ////////////////////////////////
   // Web Proxy Configs
   ////////////////////////////////

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java Fri Apr 18 16:32:35 2014
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -38,11 +39,22 @@ import org.apache.hadoop.yarn.conf.YarnC
 @Evolving
 public abstract class AuxiliaryService extends AbstractService {
 
+  private Path recoveryPath = null;
+
   protected AuxiliaryService(String name) {
     super(name);
   }
 
   /**
+   * Get the path specific to this auxiliary service to use for recovery.
+   *
+   * @return state storage path or null if recovery is not enabled
+   */
+  protected Path getRecoveryPath() {
+    return recoveryPath;
+  }
+
+  /**
    * A new application is started on this NodeManager. This is a signal to
    * this {@link AuxiliaryService} about the application initialization.
    * 
@@ -102,4 +114,13 @@ public abstract class AuxiliaryService e
   public void stopContainer(ContainerTerminationContext stopContainerContext) {
   }
 
+  /**
+   * Set the path for this auxiliary service to use for storing state
+   * that will be used during recovery.
+   *
+   * @param recoveryPath where recoverable state should be stored
+   */
+  public void setRecoveryPath(Path recoveryPath) {
+    this.recoveryPath = recoveryPath;
+  }
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Fri Apr 18 16:32:35 2014
@@ -27,6 +27,7 @@ import java.io.StringReader;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -91,7 +92,6 @@ import org.apache.hadoop.yarn.client.api
 import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
@@ -185,6 +185,9 @@ public class ApplicationMaster {
   @SuppressWarnings("rawtypes")
   private AMRMClientAsync amRMClient;
 
+  // In both secure and non-secure modes, this points to the job-submitter.
+  private UserGroupInformation appSubmitterUgi;
+
   // Handle to communicate with the Node Manager
   private NMClientAsync nmClientAsync;
   // Listen to process the response from the Node Manager
@@ -236,7 +239,7 @@ public class ApplicationMaster {
 
   // Location of shell script ( obtained from info set in env )
   // Shell script path in fs
-  private String shellScriptPath = "";
+  private String scriptPath = "";
   // Timestamp needed for creating a local resource
   private long shellScriptPathTimestamp = 0;
   // File length needed for local resource
@@ -451,7 +454,7 @@ public class ApplicationMaster {
     }
 
     if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) {
-      shellScriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION);
+      scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION);
 
       if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) {
         shellScriptPathTimestamp = Long.valueOf(envs
@@ -462,10 +465,10 @@ public class ApplicationMaster {
             .get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
       }
 
-      if (!shellScriptPath.isEmpty()
+      if (!scriptPath.isEmpty()
           && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) {
         LOG.error("Illegal values in env for shell script path" + ", path="
-            + shellScriptPath + ", len=" + shellScriptPathLen + ", timestamp="
+            + scriptPath + ", len=" + shellScriptPathLen + ", timestamp="
             + shellScriptPathTimestamp);
         throw new IllegalArgumentException(
             "Illegal values in env for shell script path");
@@ -525,14 +528,23 @@ public class ApplicationMaster {
     credentials.writeTokenStorageToStream(dob);
     // Now remove the AM->RM token so that containers cannot access it.
     Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
+    LOG.info("Executing with tokens:");
     while (iter.hasNext()) {
       Token<?> token = iter.next();
+      LOG.info(token);
       if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
         iter.remove();
       }
     }
     allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
 
+    // Create appSubmitterUgi and add original tokens to it
+    String appSubmitterUserName =
+        System.getenv(ApplicationConstants.Environment.USER.name());
+    appSubmitterUgi =
+        UserGroupInformation.createRemoteUser(appSubmitterUserName);
+    appSubmitterUgi.addCredentials(credentials);
+
     AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
     amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
     amRMClient.init(conf);
@@ -901,19 +913,26 @@ public class ApplicationMaster {
       // resources too.
       // In this scenario, if a shell script is specified, we need to have it
       // copied and made available to the container.
-      if (!shellScriptPath.isEmpty()) {
-        Path renamedSchellScriptPath = null;
+      if (!scriptPath.isEmpty()) {
+        Path renamedScriptPath = null;
         if (Shell.WINDOWS) {
-          renamedSchellScriptPath = new Path(shellScriptPath + ".bat");
+          renamedScriptPath = new Path(scriptPath + ".bat");
         } else {
-          renamedSchellScriptPath = new Path(shellScriptPath + ".sh");
+          renamedScriptPath = new Path(scriptPath + ".sh");
         }
+
         try {
-          FileSystem fs = renamedSchellScriptPath.getFileSystem(conf);
-          fs.rename(new Path(shellScriptPath), renamedSchellScriptPath);
-        } catch (IOException e) {
-          LOG.warn("Not able to add suffix (.bat/.sh) to the shell script filename");
-          throw new YarnRuntimeException(e);
+          // rename the script file based on the underlying OS syntax.
+          renameScriptFile(renamedScriptPath);
+        } catch (Exception e) {
+          LOG.error(
+              "Not able to add suffix (.bat/.sh) to the shell script filename",
+              e);
+          // We know we cannot continue launching the container
+          // so we should release it.
+          numCompletedContainers.incrementAndGet();
+          numFailedContainers.incrementAndGet();
+          return;
         }
 
         LocalResource shellRsrc = Records.newRecord(LocalResource.class);
@@ -921,11 +940,10 @@ public class ApplicationMaster {
         shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
         try {
           shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI(
-            renamedSchellScriptPath.toString())));
+            renamedScriptPath.toString())));
         } catch (URISyntaxException e) {
           LOG.error("Error when trying to use shell script path specified"
-              + " in env, path=" + renamedSchellScriptPath);
-          e.printStackTrace();
+              + " in env, path=" + renamedScriptPath, e);
 
           // A failure scenario on bad input such as invalid shell script path
           // We know we cannot continue launching the container
@@ -949,7 +967,7 @@ public class ApplicationMaster {
       // Set executable command
       vargs.add(shellCommand);
       // Set shell script path
-      if (!shellScriptPath.isEmpty()) {
+      if (!scriptPath.isEmpty()) {
         vargs.add(Shell.WINDOWS ? ExecBatScripStringtPath
             : ExecShellStringPath);
       }
@@ -983,6 +1001,20 @@ public class ApplicationMaster {
     }
   }
 
+  private void renameScriptFile(final Path renamedScriptPath)
+      throws IOException, InterruptedException {
+    appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() {
+      @Override
+      public Void run() throws IOException {
+        FileSystem fs = renamedScriptPath.getFileSystem(conf);
+        fs.rename(new Path(scriptPath), renamedScriptPath);
+        return null;
+      }
+    });
+    LOG.info("User " + appSubmitterUgi.getUserName()
+        + " added suffix(.sh/.bat) to script file as " + renamedScriptPath);
+  }
+
   /**
    * Setup the request that will be sent to the RM for the container ask.
    *

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Fri Apr 18 16:32:35 2014
@@ -468,12 +468,12 @@ public class Client {
     // Copy the application master jar to the filesystem 
     // Create a local resource to point to the destination jar path 
     FileSystem fs = FileSystem.get(conf);
-    addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(),
+    addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(),
         localResources, null);
 
     // Set the log4j properties if needed 
     if (!log4jPropFile.isEmpty()) {
-      addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(),
+      addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(),
           localResources, null);
     }			
 
@@ -489,7 +489,7 @@ public class Client {
     if (!shellScriptPath.isEmpty()) {
       Path shellSrc = new Path(shellScriptPath);
       String shellPathSuffix =
-          appName + "/" + appId.getId() + "/" + SCRIPT_PATH;
+          appName + "/" + appId.toString() + "/" + SCRIPT_PATH;
       Path shellDst =
           new Path(fs.getHomeDirectory(), shellPathSuffix);
       fs.copyFromLocalFile(false, true, shellSrc, shellDst);
@@ -500,12 +500,12 @@ public class Client {
     }
 
     if (!shellCommand.isEmpty()) {
-      addToLocalResources(fs, null, shellCommandPath, appId.getId(),
+      addToLocalResources(fs, null, shellCommandPath, appId.toString(),
           localResources, shellCommand);
     }
 
     if (shellArgs.length > 0) {
-      addToLocalResources(fs, null, shellArgsPath, appId.getId(),
+      addToLocalResources(fs, null, shellArgsPath, appId.toString(),
           localResources, StringUtils.join(shellArgs, " "));
     }
     // Set local resource info into app master container launch context
@@ -737,7 +737,7 @@ public class Client {
   }
 
   private void addToLocalResources(FileSystem fs, String fileSrcPath,
-      String fileDstPath, int appId, Map<String, LocalResource> localResources,
+      String fileDstPath, String appId, Map<String, LocalResource> localResources,
       String resources) throws IOException {
     String suffix =
         appName + "/" + appId + "/" + fileDstPath;

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Fri Apr 18 16:32:35 2014
@@ -219,7 +219,7 @@ public class TestDistributedShell {
         "--num_containers",
         "1",
         "--shell_command",
-        Shell.WINDOWS ? "timeout 8" : "sleep 8",
+        "sleep 8",
         "--master_memory",
         "512",
         "--container_memory",

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java Fri Apr 18 16:32:35 2014
@@ -33,6 +33,8 @@ import junit.framework.Assert;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ha.ClientBaseWithFixes;
 import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
@@ -67,14 +69,18 @@ import org.apache.hadoop.yarn.api.protoc
 import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
+import org.apache.hadoop.yarn.api.records.AMCommand;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
@@ -82,6 +88,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -96,6 +103,7 @@ import org.apache.hadoop.yarn.server.api
 import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
 import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
+import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
 import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
 import org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor;
 import org.apache.hadoop.yarn.server.resourcemanager.NodesListManager;
@@ -257,11 +265,13 @@ public abstract class ProtocolHATestBase
   }
 
   protected void startHACluster(int numOfNMs, boolean overrideClientRMService,
-      boolean overrideRTS) throws Exception {
+      boolean overrideRTS, boolean overrideApplicationMasterService)
+      throws Exception {
     conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     cluster =
         new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2,
-            numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS);
+            numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS,
+            overrideApplicationMasterService);
     cluster.resetStartFailoverFlag(false);
     cluster.init(conf);
     cluster.start();
@@ -285,17 +295,19 @@ public abstract class ProtocolHATestBase
 
     private boolean overrideClientRMService;
     private boolean overrideRTS;
+    private boolean overrideApplicationMasterService;
     private final AtomicBoolean startFailover = new AtomicBoolean(false);
     private final AtomicBoolean failoverTriggered = new AtomicBoolean(false);
 
     public MiniYARNClusterForHATesting(String testName,
         int numResourceManagers, int numNodeManagers, int numLocalDirs,
         int numLogDirs, boolean enableAHS, boolean overrideClientRMService,
-        boolean overrideRTS) {
+        boolean overrideRTS, boolean overrideApplicationMasterService) {
       super(testName, numResourceManagers, numNodeManagers, numLocalDirs,
           numLogDirs, enableAHS);
       this.overrideClientRMService = overrideClientRMService;
       this.overrideRTS = overrideRTS;
+      this.overrideApplicationMasterService = overrideApplicationMasterService;
     }
 
     public boolean getStartFailoverFlag() {
@@ -324,6 +336,11 @@ public abstract class ProtocolHATestBase
       if (count >= maximumWaittingTime) {
         return false;
       }
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        // DO NOTHING
+      }
       return true;
     }
 
@@ -354,6 +371,14 @@ public abstract class ProtocolHATestBase
           }
           return super.createResourceTrackerService();
         }
+        @Override
+        protected ApplicationMasterService createApplicationMasterService() {
+          if (overrideApplicationMasterService) {
+            return new CustomedApplicationMasterService(this.rmContext,
+                this.scheduler);
+          }
+          return super.createApplicationMasterService();
+        }
       };
     }
 
@@ -717,5 +742,31 @@ public abstract class ProtocolHATestBase
         return super.nodeHeartbeat(request);
       }
     }
+
+    private class CustomedApplicationMasterService extends
+        ApplicationMasterService {
+      public CustomedApplicationMasterService(RMContext rmContext,
+          YarnScheduler scheduler) {
+        super(rmContext, scheduler);
+      }
+
+      @Override
+      public AllocateResponse allocate(AllocateRequest request)
+          throws YarnException, IOException {
+        resetStartFailoverFlag(true);
+        // make sure failover has been triggered
+        Assert.assertTrue(waittingForFailOver());
+        return createFakeAllocateResponse();
+      }
+
+    }
+
+    public AllocateResponse createFakeAllocateResponse() {
+      return AllocateResponse.newInstance(-1,
+          new ArrayList<ContainerStatus>(),
+          new ArrayList<Container>(), new ArrayList<NodeReport>(),
+          Resource.newInstance(1024, 2), AMCommand.AM_RESYNC, 1,
+          null, new ArrayList<NMToken>());
+    }
   }
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java Fri Apr 18 16:32:35 2014
@@ -51,7 +51,7 @@ public class TestApplicationClientProtoc
 
   @Before
   public void initiate() throws Exception {
-    startHACluster(1, true, false);
+    startHACluster(1, true, false, false);
     Configuration conf = new YarnConfiguration(this.conf);
     client = createAndStartYarnClient(conf);
   }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java Fri Apr 18 16:32:35 2014
@@ -270,9 +270,33 @@ public class TestRMFailover extends Clie
     String header = getHeader("Refresh", rm2Url);
     assertTrue(header.contains("; url=" + rm1Url));
 
+    header = getHeader("Refresh", rm2Url + "/metrics");
+    assertTrue(header.contains("; url=" + rm1Url));
+
+    header = getHeader("Refresh", rm2Url + "/jmx");
+    assertTrue(header.contains("; url=" + rm1Url));
+
+    // standby RM links /conf, /stacks, /logLevel, /static, /logs,
+    // /cluster/cluster as well as webService
+    // /ws/v1/cluster/info should not be redirected to active RM
     header = getHeader("Refresh", rm2Url + "/cluster/cluster");
     assertEquals(null, header);
 
+    header = getHeader("Refresh", rm2Url + "/conf");
+    assertEquals(null, header);
+
+    header = getHeader("Refresh", rm2Url + "/stacks");
+    assertEquals(null, header);
+
+    header = getHeader("Refresh", rm2Url + "/logLevel");
+    assertEquals(null, header);
+
+    header = getHeader("Refresh", rm2Url + "/static");
+    assertEquals(null, header);
+
+    header = getHeader("Refresh", rm2Url + "/logs");
+    assertEquals(null, header);
+
     header = getHeader("Refresh", rm2Url + "/ws/v1/cluster/info");
     assertEquals(null, header);
 

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java Fri Apr 18 16:32:35 2014
@@ -41,7 +41,7 @@ public class TestResourceTrackerOnHA ext
 
   @Before
   public void initiate() throws Exception {
-    startHACluster(0, false, true);
+    startHACluster(0, false, true, false);
     this.resourceTracker = getRMClient();
   }
 

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Fri Apr 18 16:32:35 2014
@@ -343,9 +343,10 @@ public class TestNMClient {
         // getContainerStatus can be called after stopContainer
         try {
           // O is possible if CLEANUP_CONTAINER is executed too late
+          // 137 is possible if the container is not terminated but killed
           testGetContainerStatus(container, i, ContainerState.COMPLETE,
               "Container killed by the ApplicationMaster.", Arrays.asList(
-                  new Integer[] {143, 0}));
+                  new Integer[] {137, 143, 0}));
         } catch (YarnException e) {
           // The exception is possible because, after the container is stopped,
           // it may be removed from NM's context.

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java Fri Apr 18 16:32:35 2014
@@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 
 /**
- * Yarn application related utilities
+ * Yarn internal application-related utilities
  */
 @Private
 public class Apps {
@@ -97,6 +97,17 @@ public class Apps {
       }
     }
   }
+  
+  /**
+   * This older version of this method is kept around for compatibility
+   * because downstream frameworks like Spark and Tez have been using it.
+   * Downstream frameworks are expected to move off of it.
+   */
+  @Deprecated
+  public static void setEnvFromInputString(Map<String, String> env,
+      String envString) {
+    setEnvFromInputString(env, envString, File.pathSeparator);
+  }
 
   @Public
   @Unstable
@@ -112,6 +123,18 @@ public class Apps {
     environment.put(StringInterner.weakIntern(variable), 
         StringInterner.weakIntern(val));
   }
+  
+  /**
+   * This older version of this method is kept around for compatibility
+   * because downstream frameworks like Spark and Tez have been using it.
+   * Downstream frameworks are expected to move off of it.
+   */
+  @Deprecated
+  public static void addToEnvironment(
+      Map<String, String> environment,
+      String variable, String value) {
+    addToEnvironment(environment, variable, value, File.pathSeparator);
+  }
 
   public static String crossPlatformify(String var) {
     return ApplicationConstants.PARAMETER_EXPANSION_LEFT + var

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java Fri Apr 18 16:32:35 2014
@@ -36,12 +36,14 @@ import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.RunJar;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 
@@ -139,6 +141,16 @@ public class FSDownload implements Calla
     if (!checkPublicPermsForAll(fs, sStat, FsAction.READ_EXECUTE, FsAction.READ)) {
       return false;
     }
+
+    if (Shell.WINDOWS && fs instanceof LocalFileSystem) {
+      // Relax the requirement for public cache on LFS on Windows since default
+      // permissions are "700" all the way up to the drive letter. In this
+      // model, the only requirement for a user is to give EVERYONE group
+      // permission on the file and the file will be considered public.
+      // This code path is only hit when fs.default.name is file:/// (mainly
+      // in tests).
+      return true;
+    }
     return ancestorsHaveExecutePermissions(fs, current.getParent(), statCache);
   }
 

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java Fri Apr 18 16:32:35 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.util;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
@@ -736,7 +735,7 @@ public class ProcfsBasedProcessTree exte
       fReader = new FileReader(file);
       in = new BufferedReader(fReader);
       ProcessSmapMemoryInfo memoryMappingInfo = null;
-      List<String> lines = IOUtils.readLines(new FileInputStream(file));
+      List<String> lines = IOUtils.readLines(in);
       for (String line : lines) {
         line = line.trim();
         try {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java Fri Apr 18 16:32:35 2014
@@ -43,8 +43,7 @@ public class TimelineUtils {
     mapper = new ObjectMapper();
     AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
     mapper.setAnnotationIntrospector(introspector);
-    mapper.getSerializationConfig()
-        .setSerializationInclusion(Inclusion.NON_NULL);
+    mapper.setSerializationInclusion(Inclusion.NON_NULL);
   }
 
   /**
@@ -77,7 +76,7 @@ public class TimelineUtils {
   public static String dumpTimelineRecordtoJSON(Object o, boolean pretty)
       throws JsonGenerationException, JsonMappingException, IOException {
     if (pretty) {
-      return mapper.defaultPrettyPrintingWriter().writeValueAsString(o);
+      return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(o);
     } else {
       return mapper.writeValueAsString(o);
     }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java Fri Apr 18 16:32:35 2014
@@ -51,8 +51,7 @@ public class YarnJacksonJaxbJsonProvider
     ObjectMapper mapper = super.locateMapper(type, mediaType);
     AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
     mapper.setAnnotationIntrospector(introspector);
-    mapper.getSerializationConfig()
-        .setSerializationInclusion(Inclusion.NON_NULL);
+    mapper.setSerializationInclusion(Inclusion.NON_NULL);
     return mapper;
   }
 }
\ No newline at end of file

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Apr 18 16:32:35 2014
@@ -1019,6 +1019,19 @@
   	<value>500</value>
   </property>
   
+  <property>
+    <description>Enable the node manager to recover after starting</description>
+    <name>yarn.nodemanager.recovery.enabled</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <description>The local filesystem directory in which the node manager will
+    store state when recovery is enabled.</description>
+    <name>yarn.nodemanager.recovery.dir</name>
+    <value>${hadoop.tmp.dir}/yarn-nm-recovery</value>
+  </property>
+
   <!--Map Reduce configuration-->
   <property>
     <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java Fri Apr 18 16:32:35 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.util;
 
 import static org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.KB_TO_BYTES;
 import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
 
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
@@ -109,19 +110,13 @@ public class TestProcfsBasedProcessTree 
 
   @Before
   public void setup() throws IOException {
+    assumeTrue(Shell.LINUX);
     FileContext.getLocalFSFileContext().delete(
       new Path(TEST_ROOT_DIR.getAbsolutePath()), true);
   }
 
   @Test(timeout = 30000)
   public void testProcessTree() throws Exception {
-
-    if (!Shell.LINUX) {
-      System.out
-        .println("ProcfsBasedProcessTree is not available on this system. Not testing");
-      return;
-
-    }
     try {
       Assert.assertTrue(ProcfsBasedProcessTree.isAvailable());
     } catch (Exception e) {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java Fri Apr 18 16:32:35 2014
@@ -179,7 +179,7 @@ public class FileSystemApplicationHistor
       LOG.info("Completed reading history information of application " + appId);
       return historyData;
     } catch (IOException e) {
-      LOG.error("Error when reading history file of application " + appId);
+      LOG.error("Error when reading history file of application " + appId, e);
       throw e;
     } finally {
       hfReader.close();
@@ -296,7 +296,7 @@ public class FileSystemApplicationHistor
       return historyData;
     } catch (IOException e) {
       LOG.error("Error when reading history file of application attempt"
-          + appAttemptId);
+          + appAttemptId, e);
       throw e;
     } finally {
       hfReader.close();
@@ -344,7 +344,7 @@ public class FileSystemApplicationHistor
           + containerId);
       return historyData;
     } catch (IOException e) {
-      LOG.error("Error when reading history file of container " + containerId);
+      LOG.error("Error when reading history file of container " + containerId, e);
       throw e;
     } finally {
       hfReader.close();
@@ -420,7 +420,7 @@ public class FileSystemApplicationHistor
             + appStart.getApplicationId());
       } catch (IOException e) {
         LOG.error("Error when openning history file of application "
-            + appStart.getApplicationId());
+            + appStart.getApplicationId(), e);
         throw e;
       }
       outstandingWriters.put(appStart.getApplicationId(), hfWriter);
@@ -437,7 +437,7 @@ public class FileSystemApplicationHistor
           + appStart.getApplicationId() + " is written");
     } catch (IOException e) {
       LOG.error("Error when writing start information of application "
-          + appStart.getApplicationId());
+          + appStart.getApplicationId(), e);
       throw e;
     }
   }
@@ -456,7 +456,7 @@ public class FileSystemApplicationHistor
           + appFinish.getApplicationId() + " is written");
     } catch (IOException e) {
       LOG.error("Error when writing finish information of application "
-          + appFinish.getApplicationId());
+          + appFinish.getApplicationId(), e);
       throw e;
     } finally {
       hfWriter.close();
@@ -480,7 +480,7 @@ public class FileSystemApplicationHistor
           + appAttemptStart.getApplicationAttemptId() + " is written");
     } catch (IOException e) {
       LOG.error("Error when writing start information of application attempt "
-          + appAttemptStart.getApplicationAttemptId());
+          + appAttemptStart.getApplicationAttemptId(), e);
       throw e;
     }
   }
@@ -501,7 +501,7 @@ public class FileSystemApplicationHistor
           + appAttemptFinish.getApplicationAttemptId() + " is written");
     } catch (IOException e) {
       LOG.error("Error when writing finish information of application attempt "
-          + appAttemptFinish.getApplicationAttemptId());
+          + appAttemptFinish.getApplicationAttemptId(), e);
       throw e;
     }
   }
@@ -521,7 +521,7 @@ public class FileSystemApplicationHistor
           + containerStart.getContainerId() + " is written");
     } catch (IOException e) {
       LOG.error("Error when writing start information of container "
-          + containerStart.getContainerId());
+          + containerStart.getContainerId(), e);
       throw e;
     }
   }
@@ -541,7 +541,7 @@ public class FileSystemApplicationHistor
           + containerFinish.getContainerId() + " is written");
     } catch (IOException e) {
       LOG.error("Error when writing finish information of container "
-          + containerFinish.getContainerId());
+          + containerFinish.getContainerId(), e);
     }
   }
 
@@ -676,9 +676,10 @@ public class FileSystemApplicationHistor
 
     private TFile.Reader reader;
     private TFile.Reader.Scanner scanner;
+    FSDataInputStream fsdis;
 
     public HistoryFileReader(Path historyFile) throws IOException {
-      FSDataInputStream fsdis = fs.open(historyFile);
+      fsdis = fs.open(historyFile);
       reader =
           new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(),
             getConfig());
@@ -707,7 +708,7 @@ public class FileSystemApplicationHistor
     }
 
     public void close() {
-      IOUtils.cleanup(LOG, scanner, reader);
+      IOUtils.cleanup(LOG, scanner, reader, fsdis);
     }
 
   }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java Fri Apr 18 16:32:35 2014
@@ -23,6 +23,8 @@ import java.net.URI;
 
 import org.junit.Assert;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -42,6 +44,9 @@ import org.junit.Test;
 public class TestFileSystemApplicationHistoryStore extends
     ApplicationHistoryStoreTestUtils {
 
+  private static Log LOG = LogFactory
+    .getLog(TestFileSystemApplicationHistoryStore.class.getName());
+
   private FileSystem fs;
   private Path fsWorkingPath;
 
@@ -50,9 +55,12 @@ public class TestFileSystemApplicationHi
     fs = new RawLocalFileSystem();
     Configuration conf = new Configuration();
     fs.initialize(new URI("/"), conf);
-    fsWorkingPath = new Path("Test");
+    fsWorkingPath =
+        new Path("target",
+          TestFileSystemApplicationHistoryStore.class.getSimpleName());
     fs.delete(fsWorkingPath, true);
-    conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString());
+    conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI,
+      fsWorkingPath.toString());
     store = new FileSystemApplicationHistoryStore();
     store.init(conf);
     store.start();
@@ -67,6 +75,7 @@ public class TestFileSystemApplicationHi
 
   @Test
   public void testReadWriteHistoryData() throws IOException {
+    LOG.info("Starting testReadWriteHistoryData");
     testWriteHistoryData(5);
     testReadHistoryData(5);
   }
@@ -167,6 +176,7 @@ public class TestFileSystemApplicationHi
 
   @Test
   public void testWriteAfterApplicationFinish() throws IOException {
+    LOG.info("Starting testWriteAfterApplicationFinish");
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     writeApplicationStartData(appId);
     writeApplicationFinishData(appId);
@@ -203,6 +213,7 @@ public class TestFileSystemApplicationHi
 
   @Test
   public void testMassiveWriteContainerHistoryData() throws IOException {
+    LOG.info("Starting testMassiveWriteContainerHistoryData");
     long mb = 1024 * 1024;
     long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
     ApplicationId appId = ApplicationId.newInstance(0, 1);
@@ -221,12 +232,14 @@ public class TestFileSystemApplicationHi
 
   @Test
   public void testMissingContainerHistoryData() throws IOException {
+    LOG.info("Starting testMissingContainerHistoryData");
     testWriteHistoryData(3, true, false);
     testReadHistoryData(3, true, false);
   }
   
   @Test
   public void testMissingApplicationAttemptHistoryData() throws IOException {
+    LOG.info("Starting testMissingApplicationAttemptHistoryData");
     testWriteHistoryData(3, false, true);
     testReadHistoryData(3, false, true);
   }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Fri Apr 18 16:32:35 2014
@@ -28,6 +28,9 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.service.CompositeService;
@@ -127,6 +130,20 @@ public class NodeManager extends Composi
 
     conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
 
+    boolean recoveryEnabled = conf.getBoolean(
+        YarnConfiguration.NM_RECOVERY_ENABLED,
+        YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED);
+    if (recoveryEnabled) {
+      FileSystem recoveryFs = FileSystem.getLocal(conf);
+      String recoveryDirName = conf.get(YarnConfiguration.NM_RECOVERY_DIR);
+      if (recoveryDirName == null) {
+        throw new IllegalArgumentException("Recovery is enabled but " +
+            YarnConfiguration.NM_RECOVERY_DIR + " is not set.");
+      }
+      Path recoveryRoot = new Path(recoveryDirName);
+      recoveryFs.mkdirs(recoveryRoot, new FsPermission((short)0700));
+    }
+
     NMContainerTokenSecretManager containerTokenSecretManager =
         new NMContainerTokenSecretManager(conf);
 

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Fri Apr 18 16:32:35 2014
@@ -343,10 +343,6 @@ public class NodeStatusUpdaterImpl exten
           + ", " + nodeHealthStatus.getHealthReport());
     }
     List<ContainerStatus> containersStatuses = getContainerStatuses();
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(this.nodeId + " sending out status for "
-          + containersStatuses.size() + " containers");
-    }
     NodeStatus nodeStatus =
         NodeStatus.newInstance(nodeId, responseId, containersStatuses,
           createKeepAliveApplicationList(), nodeHealthStatus);
@@ -373,7 +369,8 @@ public class NodeStatusUpdaterImpl exten
       }
     }
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Sending out container statuses: " + containerStatuses);
+      LOG.debug("Sending out " + containerStatuses.size()
+          + " container statuses: " + containerStatuses);
     }
     return containerStatuses;
   }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java Fri Apr 18 16:32:35 2014
@@ -29,15 +29,18 @@ import java.util.regex.Pattern;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.service.ServiceStateChangeListener;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
 import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
 import org.apache.hadoop.yarn.server.api.AuxiliaryService;
-import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
 import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
 import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
 
@@ -46,6 +49,8 @@ import com.google.common.base.Preconditi
 public class AuxServices extends AbstractService
     implements ServiceStateChangeListener, EventHandler<AuxServicesEvent> {
 
+  static final String STATE_STORE_ROOT_NAME = "nm-aux-services";
+
   private static final Log LOG = LogFactory.getLog(AuxServices.class);
 
   protected final Map<String,AuxiliaryService> serviceMap;
@@ -91,6 +96,17 @@ public class AuxServices extends Abstrac
 
   @Override
   public void serviceInit(Configuration conf) throws Exception {
+    final FsPermission storeDirPerms = new FsPermission((short)0700);
+    Path stateStoreRoot = null;
+    FileSystem stateStoreFs = null;
+    boolean recoveryEnabled = conf.getBoolean(
+        YarnConfiguration.NM_RECOVERY_ENABLED,
+        YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED);
+    if (recoveryEnabled) {
+      stateStoreRoot = new Path(conf.get(YarnConfiguration.NM_RECOVERY_DIR),
+          STATE_STORE_ROOT_NAME);
+      stateStoreFs = FileSystem.getLocal(conf);
+    }
     Collection<String> auxNames = conf.getStringCollection(
         YarnConfiguration.NM_AUX_SERVICES);
     for (final String sName : auxNames) {
@@ -119,6 +135,11 @@ public class AuxServices extends Abstrac
                   +"the name in the config.");
         }
         addService(sName, s);
+        if (recoveryEnabled) {
+          Path storePath = new Path(stateStoreRoot, sName);
+          stateStoreFs.mkdirs(storePath, storeDirPerms);
+          s.setRecoveryPath(storePath);
+        }
         s.init(conf);
       } catch (RuntimeException e) {
         LOG.fatal("Failed to initialize " + sName, e);

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Fri Apr 18 16:32:35 2014
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.conf.YarnC
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
 import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
 import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@@ -141,7 +142,7 @@ public class ContainerImpl implements Co
         ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
         UPDATE_DIAGNOSTICS_TRANSITION)
     .addTransition(ContainerState.NEW, ContainerState.DONE,
-        ContainerEventType.KILL_CONTAINER, CONTAINER_DONE_TRANSITION)
+        ContainerEventType.KILL_CONTAINER, new KillOnNewTransition())
 
     // From LOCALIZING State
     .addTransition(ContainerState.LOCALIZING,
@@ -760,7 +761,9 @@ public class ContainerImpl implements Co
       container.cleanup();
       container.metrics.endInitingContainer();
       ContainerKillEvent killEvent = (ContainerKillEvent) event;
+      container.exitCode = ExitCode.TERMINATED.getExitCode();
       container.diagnostics.append(killEvent.getDiagnostic()).append("\n");
+      container.diagnostics.append("Container is killed before being launched.\n");
     }
   }
 
@@ -828,7 +831,6 @@ public class ContainerImpl implements Co
 
   /**
    * Handle the following transitions:
-   * - NEW -> DONE upon KILL_CONTAINER
    * - {LOCALIZATION_FAILED, EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE,
    *    KILLING, CONTAINER_CLEANEDUP_AFTER_KILL}
    *   -> DONE upon CONTAINER_RESOURCES_CLEANEDUP
@@ -850,6 +852,21 @@ public class ContainerImpl implements Co
   }
 
   /**
+   * Handle the following transition:
+   * - NEW -> DONE upon KILL_CONTAINER
+   */
+  static class KillOnNewTransition extends ContainerDoneTransition {
+    @Override
+    public void transition(ContainerImpl container, ContainerEvent event) {
+      ContainerKillEvent killEvent = (ContainerKillEvent) event;
+      container.exitCode = ExitCode.TERMINATED.getExitCode();
+      container.diagnostics.append(killEvent.getDiagnostic()).append("\n");
+      container.diagnostics.append("Container is killed before being launched.\n");
+      super.transition(container, event);
+    }
+  }
+
+  /**
    * Update diagnostics, staying in the same state.
    */
   static class ContainerDiagnosticsUpdateTransition implements

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java Fri Apr 18 16:32:35 2014
@@ -78,11 +78,17 @@ public class TestNodeHealthService {
   }
 
   private void writeNodeHealthScriptFile(String scriptStr, boolean setExecutable)
-      throws IOException {
-    PrintWriter pw = new PrintWriter(new FileOutputStream(nodeHealthscriptFile));
-    pw.println(scriptStr);
-    pw.flush();
-    pw.close();
+          throws IOException {
+    PrintWriter pw = null;
+    try {
+      FileUtil.setWritable(nodeHealthscriptFile, true);
+      FileUtil.setReadable(nodeHealthscriptFile, true);
+      pw = new PrintWriter(new FileOutputStream(nodeHealthscriptFile));
+      pw.println(scriptStr);
+      pw.flush();
+    } finally {
+      pw.close();
+    }
     FileUtil.setExecutable(nodeHealthscriptFile, setExecutable);
   }
 

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Fri Apr 18 16:32:35 2014
@@ -261,10 +261,12 @@ public class TestNodeStatusUpdater {
         this.context.getContainers().put(secondContainerID, container);
       } else if (heartBeatID == 3) {
         // Checks on the RM end
-        Assert.assertEquals("Number of applications should only be one!", 1,
+        Assert.assertEquals("Number of applications should have two!", 2,
             appToContainers.size());
-        Assert.assertEquals("Number of container for the app should be two!",
-            2, appToContainers.get(appId2).size());
+        Assert.assertEquals("Number of container for the app-1 should be only one!",
+            1, appToContainers.get(appId1).size());
+        Assert.assertEquals("Number of container for the app-2 should be only one!",
+            1, appToContainers.get(appId2).size());
 
         // Checks on the NM end
         ConcurrentMap<ContainerId, Container> activeContainers =



Mime
View raw message