hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1540535 [1/5] - in /hadoop/common/branches/YARN-321/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/ha...
Date Sun, 10 Nov 2013 20:09:16 GMT
Author: vinodkv
Date: Sun Nov 10 20:09:09 2013
New Revision: 1540535

URL: http://svn.apache.org/r1540535
Log:
Forwarding YARN-321 branch to latest branch-2.
svn merge ../branch-2

Added:
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppAttemptEvent.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppAttemptEvent.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNewSavedEvent.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNewSavedEvent.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppUpdateSavedEvent.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppUpdateSavedEvent.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptNewSavedEvent.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptNewSavedEvent.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUpdateSavedEvent.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUpdateSavedEvent.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
      - copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
Removed:
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppStoredEvent.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptStoredEvent.java
Modified:
    hadoop/common/branches/YARN-321/hadoop-yarn-project/   (props changed)
    hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEventType.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptState.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUnregistrationEvent.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm

Propchange: hadoop/common/branches/YARN-321/hadoop-yarn-project/
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-2/hadoop-yarn-project:r1537327-1540532

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt Sun Nov 10 20:09:09 2013
@@ -19,6 +19,9 @@ Release 2.3.0 - UNRELEASED
     YARN-1068. Add admin support for HA operations (Karthik Kambatla via
     bikas)
 
+    YARN-311. RM/scheduler support for dynamic resource configuration.
+    (Junping Du via llu)
+
   IMPROVEMENTS
 
     YARN-7. Support CPU resource for DistributedShell. (Junping Du via llu)
@@ -55,6 +58,21 @@ Release 2.3.0 - UNRELEASED
     YARN-1306. Clean up hadoop-sls sample-conf according to YARN-1228 (Wei Yan
     via Sandy Ryza)
 
+    YARN-891. Modified ResourceManager state-store to remember completed
+    applications so that clients can get information about them post RM-restart.
+    (Jian He via vinodkv)
+
+    YARN-1290. Let continuous scheduling achieve more balanced task assignment
+    (Wei Yan via Sandy Ryza)
+
+    YARN-786. Expose application resource usage in RM REST API (Sandy Ryza)
+
+    YARN-1323. Set HTTPS webapp address along with other RPC addresses in HAUtil
+    (Karthik Kambatla via Sandy Ryza)
+
+    YARN-1121. Changed ResourceManager's state-store to drain all events on
+    shut-down. (Jian He via vinodkv)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -79,6 +97,9 @@ Release 2.3.0 - UNRELEASED
     YARN-1305. RMHAProtocolService#serviceInit should handle HAUtil's
     IllegalArgumentException (Tsuyoshi Ozawa via bikas)
 
+    YARN-1374. Changed ResourceManager to start the preemption policy monitors
+    as active services. (Karthik Kambatla via vinodkv)
+
 Release 2.2.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -105,6 +126,12 @@ Release 2.2.1 - UNRELEASED
     YARN-1109. Demote NodeManager "Sending out status for container" logs to
     debug (haosdent via Sandy Ryza)
 
+    YARN-1321. Changed NMTokenCache to support both singleton and an instance
+    usage. (Alejandro Abdelnur via vinodkv) 
+
+    YARN-1388. Fair Scheduler page always displays blank fair share (Liyin Liang
+    via Sandy Ryza)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -153,6 +180,12 @@ Release 2.2.1 - UNRELEASED
     YARN-1358. TestYarnCLI fails on Windows due to line endings. (Chuan Liu via
     cnauroth)
 
+    YARN-1343. NodeManagers additions/restarts are not reported as node updates 
+    in AllocateResponse responses to AMs. (tucu)
+
+    YARN-1381. Same relaxLocality appears twice in exception message of
+    AMRMClientImpl#checkLocalityRelaxationConflict() (Ted Yu via Sandy Ryza)
+
 Release 2.2.0 - 2013-10-13
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java Sun Nov 10 20:09:09 2013
@@ -42,6 +42,7 @@ public class HAUtil {
           YarnConfiguration.RM_ADMIN_ADDRESS,
           YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
           YarnConfiguration.RM_WEBAPP_ADDRESS,
+          YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
           // TODO Remove after YARN-1318
           YarnConfiguration.RM_HA_ADMIN_ADDRESS));
 

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto Sun Nov 10 20:09:09 2013
@@ -71,14 +71,53 @@ message GetGroupsForUserResponseProto {
 ////////////////////////////////////////////////////////////////////////
 ////// RM recovery related records /////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////
+enum RMAppAttemptStateProto {
+  RMATTEMPT_NEW = 1;
+  RMATTEMPT_SUBMITTED = 2;
+  RMATTEMPT_SCHEDULED = 3;
+  RMATTEMPT_ALLOCATED = 4;
+  RMATTEMPT_LAUNCHED = 5;
+  RMATTEMPT_FAILED = 6;
+  RMATTEMPT_RUNNING = 7;
+  RMATTEMPT_FINISHING = 8;
+  RMATTEMPT_FINISHED = 9;
+  RMATTEMPT_KILLED = 10;
+  RMATTEMPT_ALLOCATED_SAVING = 11;
+  RMATTEMPT_LAUNCHED_UNMANAGED_SAVING = 12;
+  RMATTEMPT_RECOVERED = 13;
+  RMATTEMPT_FINAL_SAVING = 14;
+}
+
+enum RMAppStateProto {
+  RMAPP_NEW = 1;
+  RMAPP_NEW_SAVING = 2;
+  RMAPP_SUBMITTED = 3;
+  RMAPP_ACCEPTED = 4;
+  RMAPP_RUNNING = 5;
+  RMAPP_FINAL_SAVING = 6;
+  RMAPP_FINISHING = 7;
+  RMAPP_FINISHED = 8;
+  RMAPP_FAILED = 9;
+  RMAPP_KILLED = 10;
+}
+
 message ApplicationStateDataProto {
   optional int64 submit_time = 1;
-  optional ApplicationSubmissionContextProto application_submission_context = 2;
-  optional string user = 3;
+  optional int64 start_time = 2;
+  optional ApplicationSubmissionContextProto application_submission_context = 3;
+  optional string user = 4;
+  optional RMAppStateProto application_state = 5;
+  optional string diagnostics = 6 [default = "N/A"];
+  optional int64 finish_time = 7;
 }
 
 message ApplicationAttemptStateDataProto {
   optional ApplicationAttemptIdProto attemptId = 1;
   optional ContainerProto master_container = 2;
   optional bytes app_attempt_tokens = 3;
+  optional RMAppAttemptStateProto app_attempt_state = 4;
+  optional string final_tracking_url = 5;
+  optional string diagnostics = 6 [default = "N/A"];
+  optional int64 start_time = 7;
+  optional FinalApplicationStatusProto final_application_status = 8;
 }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Sun Nov 10 20:09:09 2013
@@ -58,6 +58,11 @@ message ResourceProto {
   optional int32 virtual_cores = 2;
 }
 
+message ResourceOptionProto {
+  optional ResourceProto resource = 1;
+  optional int32 over_commit_timeout = 2;
+}
+
 message PriorityProto {
   optional int32 priority = 1;
 }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java Sun Nov 10 20:09:09 2013
@@ -59,9 +59,12 @@ public abstract class AMRMClient<T exten
     return client;
   }
 
+  private NMTokenCache nmTokenCache;
+
   @Private
   protected AMRMClient(String name) {
     super(name);
+    nmTokenCache = NMTokenCache.getSingleton();
   }
 
   /**
@@ -297,4 +300,33 @@ public abstract class AMRMClient<T exten
    */
   public abstract void updateBlacklist(List<String> blacklistAdditions,
       List<String> blacklistRemovals);
+
+  /**
+   * Set the NM token cache for the <code>AMRMClient</code>. This cache must
+   * be shared with the {@link NMClient} used to manage containers for the
+   * <code>AMRMClient</code>
+   * <p/>
+   * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+   * singleton instance will be used.
+   *
+   * @param nmTokenCache the NM token cache to use.
+   */
+  public void setNMTokenCache(NMTokenCache nmTokenCache) {
+    this.nmTokenCache = nmTokenCache;
+  }
+
+  /**
+   * Get the NM token cache of the <code>AMRMClient</code>. This cache must be
+   * shared with the {@link NMClient} used to manage containers for the
+   * <code>AMRMClient</code>.
+   * <p/>
+   * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+   * singleton instance will be used.
+   *
+   * @return the NM token cache.
+   */
+  public NMTokenCache getNMTokenCache() {
+    return nmTokenCache;
+  }
+
 }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java Sun Nov 10 20:09:09 2013
@@ -58,6 +58,8 @@ public abstract class NMClient extends A
     return client;
   }
 
+  private NMTokenCache nmTokenCache = NMTokenCache.getSingleton();
+
   @Private
   protected NMClient(String name) {
     super(name);
@@ -118,4 +120,33 @@ public abstract class NMClient extends A
    * @param enabled whether the feature is enabled or not
    */
   public abstract void cleanupRunningContainersOnStop(boolean enabled);
+
+  /**
+   * Set the NM Token cache of the <code>NMClient</code>. This cache must be
+   * shared with the {@link AMRMClient} that requested the containers managed
+   * by this <code>NMClient</code>
+   * <p/>
+   * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+   * singleton instance will be used.
+   *
+   * @param nmTokenCache the NM token cache to use.
+   */
+  public void setNMTokenCache(NMTokenCache nmTokenCache) {
+    this.nmTokenCache = nmTokenCache;
+  }
+
+  /**
+   * Get the NM token cache of the <code>NMClient</code>. This cache must be
+   * shared with the {@link AMRMClient} that requested the containers managed
+   * by this <code>NMClient</code>
+   * <p/>
+   * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+   * singleton instance will be used.
+   *
+   * @return the NM token cache
+   */
+  public NMTokenCache getNMTokenCache() {
+    return nmTokenCache;
+  }
+
 }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java Sun Nov 10 20:09:09 2013
@@ -23,21 +23,139 @@ import java.util.concurrent.ConcurrentHa
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
 
 import com.google.common.annotations.VisibleForTesting;
 
 /**
- * It manages NMTokens required for communicating with Node manager. Its a
- * static token cache.
+ * NMTokenCache manages NMTokens required for an Application Master
+ * communicating with individual NodeManagers.
+ * <p/>
+ * By default Yarn client libraries {@link AMRMClient} and {@link NMClient} use
+ * {@link #getSingleton()} instance of the cache.
+ * <ul>
+ * <li>Using the singleton instance of the cache is appropriate when running a
+ * single ApplicationMaster in the same JVM.</li>
+ * <li>When using the singleton, users don't need to do anything special,
+ * {@link AMRMClient} and {@link NMClient} are already set up to use the default
+ * singleton {@link NMTokenCache}</li>
+ * </ul>
+ * <p/>
+ * If running multiple Application Masters in the same JVM, a different cache
+ * instance should be used for each Application Master.
+ * <p/>
+ * <ul>
+ * <li>
+ * If using the {@link AMRMClient} and the {@link NMClient}, setting up and using
+ * an instance cache is as follows:
+ * <p/>
+ * 
+ * <pre>
+ *   NMTokenCache nmTokenCache = new NMTokenCache();
+ *   AMRMClient rmClient = AMRMClient.createAMRMClient();
+ *   NMClient nmClient = NMClient.createNMClient();
+ *   nmClient.setNMTokenCache(nmTokenCache);
+ *   ...
+ * </pre>
+ * </li>
+ * <li>
+ * If using the {@link AMRMClientAsync} and the {@link NMClientAsync}, setting up
+ * and using an instance cache is as follows:
+ * <p/>
+ * 
+ * <pre>
+ *   NMTokenCache nmTokenCache = new NMTokenCache();
+ *   AMRMClient rmClient = AMRMClient.createAMRMClient();
+ *   NMClient nmClient = NMClient.createNMClient();
+ *   nmClient.setNMTokenCache(nmTokenCache);
+ *   AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+ *   NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+ *   ...
+ * </pre>
+ * </li>
+ * <li>
+ * If using {@link ApplicationMasterProtocol} and
+ * {@link ContainerManagementProtocol} directly, setting up and using an
+ * instance cache is as follows:
+ * <p/>
+ * 
+ * <pre>
+ *   NMTokenCache nmTokenCache = new NMTokenCache();
+ *   ...
+ *   ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+ *   ...
+ *   AllocateRequest allocateRequest = ...
+ *   ...
+ *   AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+ *   for (NMToken token : allocateResponse.getNMTokens()) {
+ *     nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+ *   }
+ *   ...
+ *   ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+ *   ...
+ *   nmPro.startContainer(container, containerContext);
+ *   ...
+ * </pre>
+ * </li>
+ * </ul>
+ * It is also possible to mix the usage of a client (<code>AMRMClient</code> or
+ * <code>NMClient</code>, or the async versions of them) with a protocol proxy (
+ * <code>ContainerManagementProtocolProxy</code> or
+ * <code>ApplicationMasterProtocol</code>).
  */
 @Public
 @Evolving
 public class NMTokenCache {
-  private static ConcurrentHashMap<String, Token> nmTokens;
+  private static final NMTokenCache NM_TOKEN_CACHE = new NMTokenCache();
   
+  /**
+   * Returns the singleton NM token cache.
+   *
+   * @return the singleton NM token cache.
+   */
+  public static NMTokenCache getSingleton() {
+    return NM_TOKEN_CACHE;
+  }
   
-  static {
+  /**
+   * Returns NMToken, null if absent. Only the singleton obtained from
+   * {@link #getSingleton()} is looked at for the tokens. If you are using your
+   * own NMTokenCache that is different from the singleton, use
+   * {@link #getToken(String) }
+   * 
+   * @param nodeAddr
+   * @return {@link Token} NMToken required for communicating with node manager
+   */
+  @Public
+  public static Token getNMToken(String nodeAddr) {
+    return NM_TOKEN_CACHE.getToken(nodeAddr);
+  }
+  
+  /**
+   * Sets the NMToken for node address only in the singleton obtained from
+   * {@link #getSingleton()}. If you are using your own NMTokenCache that is
+   * different from the singleton, use {@link #setToken(String, Token) }
+   * 
+   * @param nodeAddr
+   *          node address (host:port)
+   * @param token
+   *          NMToken
+   */
+  @Public
+  public static void setNMToken(String nodeAddr, Token token) {
+    NM_TOKEN_CACHE.setToken(nodeAddr, token);
+  }
+
+  private ConcurrentHashMap<String, Token> nmTokens;
+
+  /**
+   * Creates a NM token cache instance.
+   */
+  public NMTokenCache() {
     nmTokens = new ConcurrentHashMap<String, Token>();
   }
   
@@ -45,11 +163,11 @@ public class NMTokenCache {
    * Returns NMToken, null if absent
    * @param nodeAddr
    * @return {@link Token} NMToken required for communicating with node
-   * manager
+   *         manager
    */
   @Public
   @Evolving
-  public static Token getNMToken(String nodeAddr) {
+  public Token getToken(String nodeAddr) {
     return nmTokens.get(nodeAddr);
   }
   
@@ -60,7 +178,7 @@ public class NMTokenCache {
    */
   @Public
   @Evolving
-  public static void setNMToken(String nodeAddr, Token token) {
+  public void setToken(String nodeAddr, Token token) {
     nmTokens.put(nodeAddr, token);
   }
   
@@ -69,7 +187,7 @@ public class NMTokenCache {
    */
   @Private
   @VisibleForTesting
-  public static boolean containsNMToken(String nodeAddr) {
+  public boolean containsToken(String nodeAddr) {
     return nmTokens.containsKey(nodeAddr);
   }
   
@@ -78,7 +196,7 @@ public class NMTokenCache {
    */
   @Private
   @VisibleForTesting
-  public static int numberOfNMTokensInCache() {
+  public int numberOfTokensInCache() {
     return nmTokens.size();
   }
   
@@ -88,7 +206,7 @@ public class NMTokenCache {
    */
   @Private
   @VisibleForTesting
-  public static void removeNMToken(String nodeAddr) {
+  public void removeToken(String nodeAddr) {
     nmTokens.remove(nodeAddr);
   }
   
@@ -97,7 +215,7 @@ public class NMTokenCache {
    */
   @Private
   @VisibleForTesting
-  public static void clearCache() {
+  public void clearCache() {
     nmTokens.clear();
   }
 }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java Sun Nov 10 20:09:09 2013
@@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.client.Cli
 import org.apache.hadoop.yarn.client.api.AMRMClient;
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
 import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
-import org.apache.hadoop.yarn.client.api.NMTokenCache;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -288,12 +287,12 @@ public class AMRMClientImpl<T extends Co
   protected void populateNMTokens(AllocateResponse allocateResponse) {
     for (NMToken token : allocateResponse.getNMTokens()) {
       String nodeId = token.getNodeId().toString();
-      if (NMTokenCache.containsNMToken(nodeId)) {
+      if (getNMTokenCache().containsToken(nodeId)) {
         LOG.debug("Replacing token for : " + nodeId);
       } else {
         LOG.debug("Received new token for : " + nodeId);
       }
-      NMTokenCache.setNMToken(nodeId, token.getToken());
+      getNMTokenCache().setToken(nodeId, token.getToken());
     }
   }
 
@@ -500,13 +499,15 @@ public class AMRMClientImpl<T extends Co
     for (String location : locations) {
         TreeMap<Resource, ResourceRequestInfo> reqs =
             remoteRequests.get(location);
-        if (reqs != null && !reqs.isEmpty()
-            && reqs.values().iterator().next().remoteRequest.getRelaxLocality()
-            != relaxLocality) {
-          throw new InvalidContainerRequestException("Cannot submit a "
-              + "ContainerRequest asking for location " + location
-              + " with locality relaxation " + relaxLocality + " when it has "
-              + "already been requested with locality relaxation " + relaxLocality);
+        if (reqs != null && !reqs.isEmpty()) {
+          boolean existingRelaxLocality =
+              reqs.values().iterator().next().remoteRequest.getRelaxLocality();
+          if (relaxLocality != existingRelaxLocality) {
+            throw new InvalidContainerRequestException("Cannot submit a "
+                + "ContainerRequest asking for location " + location
+                + " with locality relaxation " + relaxLocality + " when it has "
+                + "already been requested with locality relaxation " + existingRelaxLocality);
+          }
         }
       }
   }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java Sun Nov 10 20:09:09 2013
@@ -56,9 +56,16 @@ public class ContainerManagementProtocol
   private final LinkedHashMap<String, ContainerManagementProtocolProxyData> cmProxy;
   private final Configuration conf;
   private final YarnRPC rpc;
+  private NMTokenCache nmTokenCache;
   
   public ContainerManagementProtocolProxy(Configuration conf) {
+    this(conf, NMTokenCache.getSingleton());
+  }
+
+  public ContainerManagementProtocolProxy(Configuration conf,
+      NMTokenCache nmTokenCache) {
     this.conf = conf;
+    this.nmTokenCache = nmTokenCache;
 
     maxConnectedNMs =
         conf.getInt(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES,
@@ -86,7 +93,7 @@ public class ContainerManagementProtocol
 
     while (proxy != null
         && !proxy.token.getIdentifier().equals(
-            NMTokenCache.getNMToken(containerManagerBindAddr).getIdentifier())) {
+            nmTokenCache.getToken(containerManagerBindAddr).getIdentifier())) {
       LOG.info("Refreshing proxy as NMToken got updated for node : "
           + containerManagerBindAddr);
       // Token is updated. check if anyone has already tried closing it.
@@ -109,7 +116,7 @@ public class ContainerManagementProtocol
     if (proxy == null) {
       proxy =
           new ContainerManagementProtocolProxyData(rpc, containerManagerBindAddr,
-              containerId, NMTokenCache.getNMToken(containerManagerBindAddr));
+              containerId, nmTokenCache.getToken(containerManagerBindAddr));
       if (cmProxy.size() > maxConnectedNMs) {
         // Number of existing proxy exceed the limit.
         String cmAddr = cmProxy.keySet().iterator().next();

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java Sun Nov 10 20:09:09 2013
@@ -130,7 +130,10 @@ public class NMClientImpl extends NMClie
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
     super.serviceInit(conf);
-    cmProxy = new ContainerManagementProtocolProxy(conf);
+    if (getNMTokenCache() == null) {
+      throw new IllegalStateException("NMTokenCache has not been set");
+    }
+    cmProxy = new ContainerManagementProtocolProxy(conf, getNMTokenCache());
   }
   
   @Override

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Sun Nov 10 20:09:09 2013
@@ -626,6 +626,13 @@ public class TestAMRMClient {
     try {
       // start am rm client
       amClient = AMRMClient.<ContainerRequest>createAMRMClient();
+
+      //setting an instance NMTokenCache
+      amClient.setNMTokenCache(new NMTokenCache());
+      //asserting we are not using the singleton instance cache
+      Assert.assertNotSame(NMTokenCache.getSingleton(), 
+          amClient.getNMTokenCache());
+
       amClient.init(conf);
       amClient.start();
 
@@ -681,8 +688,8 @@ public class TestAMRMClient {
     int iterationsLeft = 3;
     Set<ContainerId> releases = new TreeSet<ContainerId>();
     
-    NMTokenCache.clearCache();
-    Assert.assertEquals(0, NMTokenCache.numberOfNMTokensInCache());
+    amClient.getNMTokenCache().clearCache();
+    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
     HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>();
     
     while (allocatedContainerCount < containersRequestedAny

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Sun Nov 10 20:09:09 2013
@@ -78,6 +78,7 @@ public class TestNMClient {
   List<NodeReport> nodeReports = null;
   ApplicationAttemptId attemptId = null;
   int nodeCount = 3;
+  NMTokenCache nmTokenCache = null;
   
   @Before
   public void setup() throws YarnException, IOException {
@@ -155,10 +156,16 @@ public class TestNMClient {
       .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
     UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
 
+    //creating an instance NMTokenCase
+    nmTokenCache = new NMTokenCache();
+    
     // start am rm client
     rmClient =
         (AMRMClientImpl<ContainerRequest>) AMRMClient
           .<ContainerRequest> createAMRMClient();
+
+    //setting an instance NMTokenCase
+    rmClient.setNMTokenCache(nmTokenCache);
     rmClient.init(conf);
     rmClient.start();
     assertNotNull(rmClient);
@@ -166,6 +173,9 @@ public class TestNMClient {
 
     // start am nm client
     nmClient = (NMClientImpl) NMClient.createNMClient();
+    
+    //propagating the AMRMClient NMTokenCache instance
+    nmClient.setNMTokenCache(rmClient.getNMTokenCache());
     nmClient.init(conf);
     nmClient.start();
     assertNotNull(nmClient);
@@ -258,7 +268,7 @@ public class TestNMClient {
       }
       if (!allocResponse.getNMTokens().isEmpty()) {
         for (NMToken token : allocResponse.getNMTokens()) {
-          NMTokenCache.setNMToken(token.getNodeId().toString(),
+          rmClient.getNMTokenCache().setToken(token.getNodeId().toString(),
               token.getToken());
         }
       }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java Sun Nov 10 20:09:09 2013
@@ -49,6 +49,19 @@ public class AsyncDispatcher extends Abs
   private final BlockingQueue<Event> eventQueue;
   private volatile boolean stopped = false;
 
+  // Configuration flag for enabling/disabling draining dispatcher's events on
+  // stop functionality.
+  private volatile boolean drainEventsOnStop = false;
+
+  // Indicates all the remaining dispatcher's events on stop have been drained
+  // and processed.
+  private volatile boolean drained = true;
+
+  // For drainEventsOnStop enabled only, block newly coming events into the
+  // queue while stopping.
+  private volatile boolean blockNewEvents = false;
+  private EventHandler handlerInstance = null;
+
   private Thread eventHandlingThread;
   protected final Map<Class<? extends Enum>, EventHandler> eventDispatchers;
   private boolean exitOnDispatchException;
@@ -68,6 +81,7 @@ public class AsyncDispatcher extends Abs
       @Override
       public void run() {
         while (!stopped && !Thread.currentThread().isInterrupted()) {
+          drained = eventQueue.isEmpty();
           Event event;
           try {
             event = eventQueue.take();
@@ -102,8 +116,19 @@ public class AsyncDispatcher extends Abs
     eventHandlingThread.start();
   }
 
+  public void setDrainEventsOnStop() {
+    drainEventsOnStop = true;
+  }
+
   @Override
   protected void serviceStop() throws Exception {
+    if (drainEventsOnStop) {
+      blockNewEvents = true;
+      LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
+      while(!drained) {
+        Thread.yield();
+      }
+    }
     stopped = true;
     if (eventHandlingThread != null) {
       eventHandlingThread.interrupt();
@@ -173,11 +198,19 @@ public class AsyncDispatcher extends Abs
 
   @Override
   public EventHandler getEventHandler() {
-    return new GenericEventHandler();
+    if (handlerInstance == null) {
+      handlerInstance = new GenericEventHandler();
+    }
+    return handlerInstance;
   }
 
   class GenericEventHandler implements EventHandler<Event> {
     public void handle(Event event) {
+      if (blockNewEvents) {
+        return;
+      }
+      drained = false;
+
       /* all this method does is enqueue all the events onto the queue */
       int qSize = eventQueue.size();
       if (qSize !=0 && qSize %1000 == 0) {

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java Sun Nov 10 20:09:09 2013
@@ -160,17 +160,14 @@ public class NodesListManager extends Ab
       if (unusableRMNodesConcurrentSet.contains(eventNode)) {
         LOG.debug(eventNode + " reported usable");
         unusableRMNodesConcurrentSet.remove(eventNode);
-        for (RMApp app : rmContext.getRMApps().values()) {
-          this.rmContext
-              .getDispatcher()
-              .getEventHandler()
-              .handle(
-                  new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
-                      RMAppNodeUpdateType.NODE_USABLE));
-        }
-      } else {
-        LOG.warn(eventNode
-            + " reported usable without first reporting unusable");
+      }
+      for (RMApp app : rmContext.getRMApps().values()) {
+        this.rmContext
+            .getDispatcher()
+            .getEventHandler()
+            .handle(
+                new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
+                    RMAppNodeUpdateType.NODE_USABLE));
       }
       break;
     default:

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Sun Nov 10 20:09:09 2013
@@ -63,7 +63,6 @@ public class RMAppManager implements Eve
   private static final Log LOG = LogFactory.getLog(RMAppManager.class);
 
   private int completedAppsMax = YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS;
-  private int globalMaxAppAttempts;
   private LinkedList<ApplicationId> completedApps = new LinkedList<ApplicationId>();
 
   private final RMContext rmContext;
@@ -83,8 +82,6 @@ public class RMAppManager implements Eve
     setCompletedAppsMax(conf.getInt(
         YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS,
         YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS));
-    globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
-        YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
   }
 
   /**
@@ -302,10 +299,11 @@ public class RMAppManager implements Eve
       throw RPCUtil.getRemoteException(ie);
     }
 
-    // All done, start the RMApp
-    this.rmContext.getDispatcher().getEventHandler().handle(
-        new RMAppEvent(applicationId, isRecovered ? RMAppEventType.RECOVER:
-            RMAppEventType.START));
+    if (!isRecovered) {
+      // All done, start the RMApp
+      this.rmContext.getDispatcher().getEventHandler()
+        .handle(new RMAppEvent(applicationId, RMAppEventType.START));
+    }
   }
   
   private Credentials parseCredentials(ApplicationSubmissionContext application) 
@@ -328,53 +326,19 @@ public class RMAppManager implements Eve
     // recover applications
     Map<ApplicationId, ApplicationState> appStates = state.getApplicationState();
     LOG.info("Recovering " + appStates.size() + " applications");
-    for(ApplicationState appState : appStates.values()) {
-      boolean shouldRecover = true;
-      if(appState.getApplicationSubmissionContext().getUnmanagedAM()) {
-        // do not recover unmanaged applications since current recovery 
-        // mechanism of restarting attempts does not work for them.
-        // This will need to be changed in work preserving recovery in which 
-        // RM will re-connect with the running AM's instead of restarting them
-        LOG.info("Not recovering unmanaged application " + appState.getAppId());
-        shouldRecover = false;
-      }
-      int individualMaxAppAttempts = appState.getApplicationSubmissionContext()
-          .getMaxAppAttempts();
-      int maxAppAttempts;
-      if (individualMaxAppAttempts <= 0 ||
-          individualMaxAppAttempts > globalMaxAppAttempts) {
-        maxAppAttempts = globalMaxAppAttempts;
-        LOG.warn("The specific max attempts: " + individualMaxAppAttempts
-            + " for application: " + appState.getAppId()
-            + " is invalid, because it is out of the range [1, "
-            + globalMaxAppAttempts + "]. Use the global max attempts instead.");
-      } else {
-        maxAppAttempts = individualMaxAppAttempts;
-      }
-      // In work-preserve restart, if attemptCount == maxAttempts, the job still
-      // needs to be recovered because the last attempt may still be running.
-      if(appState.getAttemptCount() >= maxAppAttempts) {
-        LOG.info("Not recovering application " + appState.getAppId() +
-            " due to recovering attempt is beyond maxAppAttempt limit");
-        shouldRecover = false;
-      }
-
-      // re-submit the application
-      // this is going to send an app start event but since the async dispatcher
-      // has not started that event will be queued until we have completed re
-      // populating the state
-      if(shouldRecover) {
-        LOG.info("Recovering application " + appState.getAppId());
-        submitApplication(appState.getApplicationSubmissionContext(), 
-                        appState.getSubmitTime(), true, appState.getUser());
-        // re-populate attempt information in application
-        RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get(
-                                                        appState.getAppId());
-        appImpl.recover(state);
-      }
-      else {
-        store.removeApplication(appState);
-      }
+    for (ApplicationState appState : appStates.values()) {
+      LOG.info("Recovering application " + appState.getAppId());
+      submitApplication(appState.getApplicationSubmissionContext(),
+        appState.getSubmitTime(), true, appState.getUser());
+      // re-populate attempt information in application
+      RMAppImpl appImpl =
+          (RMAppImpl) rmContext.getRMApps().get(appState.getAppId());
+      appImpl.recover(state);
+      // Recover the app synchronously, as otherwise client is possible to see
+      // the application not recovered before it is actually recovered because
+      // ClientRMService is already started at this point of time.
+      appImpl.handle(new RMAppEvent(appImpl.getApplicationId(),
+        RMAppEventType.RECOVER));
     }
   }
 

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Sun Nov 10 20:09:09 2013
@@ -501,6 +501,36 @@ public class ResourceManager extends Com
 
       super.serviceStop();
     }
+
+    protected void createPolicyMonitors() {
+      if (scheduler instanceof PreemptableResourceScheduler
+          && conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
+          YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS)) {
+        LOG.info("Loading policy monitors");
+        List<SchedulingEditPolicy> policies = conf.getInstances(
+            YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+            SchedulingEditPolicy.class);
+        if (policies.size() > 0) {
+          rmDispatcher.register(ContainerPreemptEventType.class,
+              new RMContainerPreemptEventDispatcher(
+                  (PreemptableResourceScheduler) scheduler));
+          for (SchedulingEditPolicy policy : policies) {
+            LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
+            policy.init(conf, rmContext.getDispatcher().getEventHandler(),
+                (PreemptableResourceScheduler) scheduler);
+            // periodically check whether we need to take action to guarantee
+            // constraints
+            SchedulingMonitor mon = new SchedulingMonitor(policy);
+            addService(mon);
+          }
+        } else {
+          LOG.warn("Policy monitors configured (" +
+              YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS +
+              ") but none specified (" +
+              YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES + ")");
+        }
+      }
+    }
   }
 
   @Private
@@ -829,37 +859,6 @@ public class ResourceManager extends Com
     return new ApplicationMasterService(this.rmContext, scheduler);
   }
 
-  protected void createPolicyMonitors() {
-    if (scheduler instanceof PreemptableResourceScheduler
-        && conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
-          YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS)) {
-      LOG.info("Loading policy monitors");
-      List<SchedulingEditPolicy> policies = conf.getInstances(
-              YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
-              SchedulingEditPolicy.class);
-      if (policies.size() > 0) {
-        this.rmDispatcher.register(ContainerPreemptEventType.class,
-          new RMContainerPreemptEventDispatcher(
-            (PreemptableResourceScheduler) scheduler));
-        for (SchedulingEditPolicy policy : policies) {
-          LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
-          policy.init(conf, this.rmContext.getDispatcher().getEventHandler(),
-              (PreemptableResourceScheduler) scheduler);
-          // periodically check whether we need to take action to guarantee
-          // constraints
-          SchedulingMonitor mon = new SchedulingMonitor(policy);
-          addService(mon);
-
-        }
-      } else {
-        LOG.warn("Policy monitors configured (" +
-            YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS +
-            ") but none specified (" +
-            YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES + ")");
-      }
-    }
-  }
-
   protected AdminService createAdminService(
       ClientRMService clientRMService, 
       ApplicationMasterService applicationMasterService,

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Sun Nov 10 20:09:09 2013
@@ -31,6 +31,7 @@ import org.apache.hadoop.service.Abstrac
 import org.apache.hadoop.util.VersionUtil;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -234,7 +235,8 @@ public class ResourceTrackerService exte
         .getCurrentKey());    
 
     RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort,
-        resolve(host), capability, nodeManagerVersion);
+        resolve(host), ResourceOption.newInstance(capability, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT),
+        nodeManagerVersion);
 
     RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode);
     if (oldNode == null) {

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java Sun Nov 10 20:09:09 2013
@@ -133,8 +133,11 @@ public class FileSystemRMStateStore exte
                   ApplicationStateDataProto.parseFrom(childData));
             ApplicationState appState =
                 new ApplicationState(appStateData.getSubmitTime(),
+                  appStateData.getStartTime(),
                   appStateData.getApplicationSubmissionContext(),
-                  appStateData.getUser());
+                  appStateData.getUser(),
+                  appStateData.getState(),
+                  appStateData.getDiagnostics(), appStateData.getFinishTime());
             // assert child node name is same as actual applicationId
             assert appId.equals(appState.context.getApplicationId());
             rmState.appState.put(appId, appState);
@@ -156,7 +159,12 @@ public class FileSystemRMStateStore exte
             }
             ApplicationAttemptState attemptState =
                 new ApplicationAttemptState(attemptId,
-                  attemptStateData.getMasterContainer(), credentials);
+                  attemptStateData.getMasterContainer(), credentials,
+                  attemptStateData.getStartTime(),
+                  attemptStateData.getState(),
+                  attemptStateData.getFinalTrackingUrl(),
+                  attemptStateData.getDiagnostics(),
+                  attemptStateData.getFinalApplicationStatus());
 
             // assert child node name is same as application attempt id
             assert attemptId.equals(attemptState.getAttemptId());
@@ -232,7 +240,7 @@ public class FileSystemRMStateStore exte
   }
 
   @Override
-  public synchronized void storeApplicationState(String appId,
+  public synchronized void storeApplicationStateInternal(String appId,
       ApplicationStateDataPBImpl appStateDataPB) throws Exception {
     Path appDirPath = getAppDir(rmAppRoot, appId);
     fs.mkdirs(appDirPath);
@@ -251,15 +259,34 @@ public class FileSystemRMStateStore exte
   }
 
   @Override
-  public synchronized void storeApplicationAttemptState(String attemptId,
-      ApplicationAttemptStateDataPBImpl attemptStateDataPB) throws Exception {
+  public synchronized void updateApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateDataPB) throws Exception {
+    Path appDirPath = getAppDir(rmAppRoot, appId);
+    Path nodeCreatePath = getNodePath(appDirPath, appId);
+
+    LOG.info("Updating info for app: " + appId + " at: " + nodeCreatePath);
+    byte[] appStateData = appStateDataPB.getProto().toByteArray();
+    try {
+      // currently throw all exceptions. May need to respond differently for HA
+      // based on whether we have lost the right to write to FS
+      updateFile(nodeCreatePath, appStateData);
+    } catch (Exception e) {
+      LOG.info("Error updating info for app: " + appId, e);
+      throw e;
+    }
+  }
+
+  @Override
+  public synchronized void storeApplicationAttemptStateInternal(
+      String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+      throws Exception {
     ApplicationAttemptId appAttemptId =
         ConverterUtils.toApplicationAttemptId(attemptId);
     Path appDirPath =
         getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
     Path nodeCreatePath = getNodePath(appDirPath, attemptId);
-    LOG.info("Storing info for attempt: " + attemptId
-             + " at: " + nodeCreatePath);
+    LOG.info("Storing info for attempt: " + attemptId + " at: "
+        + nodeCreatePath);
     byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
     try {
       // currently throw all exceptions. May need to respond differently for HA
@@ -272,6 +299,28 @@ public class FileSystemRMStateStore exte
   }
 
   @Override
+  public synchronized void updateApplicationAttemptStateInternal(
+      String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+      throws Exception {
+    ApplicationAttemptId appAttemptId =
+        ConverterUtils.toApplicationAttemptId(attemptId);
+    Path appDirPath =
+        getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
+    Path nodeCreatePath = getNodePath(appDirPath, attemptId);
+    LOG.info("Updating info for attempt: " + attemptId + " at: "
+        + nodeCreatePath);
+    byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
+    try {
+      // currently throw all exceptions. May need to respond differently for HA
+      // based on whether we have lost the right to write to FS
+      updateFile(nodeCreatePath, attemptStateData);
+    } catch (Exception e) {
+      LOG.info("Error updating info for attempt: " + attemptId, e);
+      throw e;
+    }
+  }
+
+  @Override
   public synchronized void removeApplicationState(ApplicationState appState)
       throws Exception {
     String appId = appState.getAppId().toString();
@@ -373,12 +422,21 @@ public class FileSystemRMStateStore exte
     Path tempPath =
         new Path(outputPath.getParent(), outputPath.getName() + ".tmp");
     FSDataOutputStream fsOut = null;
-    fsOut = fs.create(tempPath, false);
+    // This file will be overwritten when app/attempt finishes for saving the
+    // final status.
+    fsOut = fs.create(tempPath, true);
     fsOut.write(data);
     fsOut.close();
     fs.rename(tempPath, outputPath);
   }
 
+  private void updateFile(Path outputPath, byte[] data) throws Exception {
+    if (fs.exists(outputPath)) {
+      deleteFile(outputPath);
+    }
+    writeFile(outputPath, data);
+  }
+
   private boolean renameFile(Path src, Path dst) throws Exception {
     return fs.rename(src, dst);
   }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java Sun Nov 10 20:09:09 2013
@@ -76,22 +76,39 @@ public class MemoryRMStateStore extends 
   }
 
   @Override
-  public void storeApplicationState(String appId, 
+  public void storeApplicationStateInternal(String appId, 
                                      ApplicationStateDataPBImpl appStateData)
       throws Exception {
-    ApplicationState appState = new ApplicationState(
-        appStateData.getSubmitTime(),
-        appStateData.getApplicationSubmissionContext(), appStateData.getUser());
-    if (state.appState.containsKey(appState.getAppId())) {
-      Exception e = new IOException("App: " + appId + " is already stored.");
-      LOG.info("Error storing info for app: " + appId, e);
-      throw e;
-    }
+    ApplicationState appState =
+        new ApplicationState(appStateData.getSubmitTime(),
+          appStateData.getStartTime(),
+          appStateData.getApplicationSubmissionContext(),
+          appStateData.getUser());
     state.appState.put(appState.getAppId(), appState);
   }
 
   @Override
-  public synchronized void storeApplicationAttemptState(String attemptIdStr, 
+  public void updateApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateData) throws Exception {
+    ApplicationState updatedAppState =
+        new ApplicationState(appStateData.getSubmitTime(),
+          appStateData.getStartTime(),
+          appStateData.getApplicationSubmissionContext(),
+          appStateData.getUser(), appStateData.getState(),
+          appStateData.getDiagnostics(), appStateData.getFinishTime());
+    LOG.info("Updating final state " + appStateData.getState() + " for app: "
+        + appId);
+    ApplicationId applicationId = updatedAppState.getAppId();
+    if (state.appState.get(applicationId) != null) {
+      // add the earlier attempts back
+      updatedAppState.attempts
+        .putAll(state.appState.get(applicationId).attempts);
+    }
+    state.appState.put(applicationId, updatedAppState);
+  }
+
+  @Override
+  public synchronized void storeApplicationAttemptStateInternal(String attemptIdStr, 
                             ApplicationAttemptStateDataPBImpl attemptStateData)
                             throws Exception {
     ApplicationAttemptId attemptId = ConverterUtils
@@ -105,22 +122,48 @@ public class MemoryRMStateStore extends 
     }
     ApplicationAttemptState attemptState =
         new ApplicationAttemptState(attemptId,
-          attemptStateData.getMasterContainer(), credentials);
+          attemptStateData.getMasterContainer(), credentials,
+          attemptStateData.getStartTime());
 
     ApplicationState appState = state.getApplicationState().get(
         attemptState.getAttemptId().getApplicationId());
     if (appState == null) {
       throw new YarnRuntimeException("Application doesn't exist");
     }
+    appState.attempts.put(attemptState.getAttemptId(), attemptState);
+  }
 
-    if (appState.attempts.containsKey(attemptState.getAttemptId())) {
-      Exception e = new IOException("Attempt: " +
-          attemptState.getAttemptId() + " is already stored.");
-      LOG.info("Error storing info for attempt: " +
-          attemptState.getAttemptId(), e);
-      throw e;
+  @Override
+  public synchronized void updateApplicationAttemptStateInternal(
+      String attemptIdStr, ApplicationAttemptStateDataPBImpl attemptStateData)
+      throws Exception {
+    ApplicationAttemptId attemptId =
+        ConverterUtils.toApplicationAttemptId(attemptIdStr);
+    Credentials credentials = null;
+    if (attemptStateData.getAppAttemptTokens() != null) {
+      DataInputByteBuffer dibb = new DataInputByteBuffer();
+      credentials = new Credentials();
+      dibb.reset(attemptStateData.getAppAttemptTokens());
+      credentials.readTokenStorageStream(dibb);
     }
-    appState.attempts.put(attemptState.getAttemptId(), attemptState);
+    ApplicationAttemptState updatedAttemptState =
+        new ApplicationAttemptState(attemptId,
+          attemptStateData.getMasterContainer(), credentials,
+          attemptStateData.getStartTime(), attemptStateData.getState(),
+          attemptStateData.getFinalTrackingUrl(),
+          attemptStateData.getDiagnostics(),
+          attemptStateData.getFinalApplicationStatus());
+
+    ApplicationState appState =
+        state.getApplicationState().get(
+          updatedAttemptState.getAttemptId().getApplicationId());
+    if (appState == null) {
+      throw new YarnRuntimeException("Application doesn't exist");
+    }
+    LOG.info("Updating final state " + updatedAttemptState.getState()
+        + " for attempt: " + updatedAttemptState.getAttemptId());
+    appState.attempts.put(updatedAttemptState.getAttemptId(),
+      updatedAttemptState);
   }
 
   @Override

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java Sun Nov 10 20:09:09 2013
@@ -50,13 +50,13 @@ public class NullRMStateStore extends RM
   }
 
   @Override
-  protected void storeApplicationState(String appId,
+  protected void storeApplicationStateInternal(String appId,
       ApplicationStateDataPBImpl appStateData) throws Exception {
     // Do nothing
   }
 
   @Override
-  protected void storeApplicationAttemptState(String attemptId,
+  protected void storeApplicationAttemptStateInternal(String attemptId,
       ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception {
     // Do nothing
   }
@@ -89,4 +89,16 @@ public class NullRMStateStore extends RM
   public void removeRMDTMasterKeyState(DelegationKey delegationKey) throws Exception {
     // Do nothing
   }
+
+  @Override
+  protected void updateApplicationStateInternal(String appId,
+      ApplicationStateDataPBImpl appStateData) throws Exception {
+    // Do nothing 
+  }
+
+  @Override
+  protected void updateApplicationAttemptStateInternal(String attemptId,
+      ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception {
+    // Do nothing
+  }
 }



Mime
View raw message