Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BF698101F8 for ; Fri, 7 Feb 2014 02:44:07 +0000 (UTC) Received: (qmail 72303 invoked by uid 500); 7 Feb 2014 02:44:07 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 72134 invoked by uid 500); 7 Feb 2014 02:44:05 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 72125 invoked by uid 99); 7 Feb 2014 02:44:05 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Feb 2014 02:44:05 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Feb 2014 02:43:52 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 8987D2388A33; Fri, 7 Feb 2014 02:43:12 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1565519 [1/3] - in /hadoop/common/branches/HDFS-5535/hadoop-yarn-project: ./ hadoop-yarn/bin/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop... Date: Fri, 07 Feb 2014 02:43:10 -0000 To: yarn-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140207024312.8987D2388A33@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Fri Feb 7 02:43:04 2014 New Revision: 1565519 URL: http://svn.apache.org/r1565519 Log: Merge r1555021 through r1565516 from trunk. Added: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ - copied from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ - copied from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml - copied unchanged from r1565516, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml Removed: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt Fri Feb 7 02:43:04 2014 @@ -14,6 +14,9 @@ Trunk - Unreleased YARN-1504. RM changes for moving apps between queues (Sandy Ryza) + YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy + Ryza) + IMPROVEMENTS OPTIMIZATIONS @@ -82,6 +85,24 @@ Release 2.4.0 - UNRELEASED YARN-1633. Defined user-facing entity, entity-info and event objects related to Application Timeline feature. (Zhijie Shen via vinodkv) + YARN-1611. Introduced the concept of a configuration provider which can be + used by ResourceManager to read configuration locally or from remote systems + so as to help RM failover. (Xuan Gong via vinodkv) + + YARN-1659. Defined the ApplicationTimelineStore store as an abstraction for + implementing different storage impls for storing timeline information. + (Billie Rinaldi via vinodkv) + + YARN-1634. Added a testable in-memory implementation of + ApplicationTimelineStore. (Zhijie Shen via vinodkv) + + YARN-1461. Added tags for YARN applications and changed RM to handle them. + (Karthik Kambatla via zjshen) + + YARN-1636. Augmented Application-history server's web-services to also expose + new APIs for retrieving and storing timeline information. (Zhijie Shen via + vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via @@ -99,6 +120,25 @@ Release 2.4.0 - UNRELEASED YARN-1617. Remove ancient comment and surround LOG.debug in AppSchedulingInfo.allocate (Sandy Ryza) + YARN-1639. Modified RM HA configuration handling to have a way of not + requiring separate configuration files for each RM. (Xuan Gong via vinodkv) + + YARN-1668. Modified RM HA handling of admin-acls to be available across RM + failover by making using of a remote configuration-provider. (Xuan Gong via + vinodkv) + + YARN-1667. Modified RM HA handling of super users (with proxying ability) to + be available across RM failover by making using of a remote + configuration-provider. (Xuan Gong via vinodkv) + + YARN-1285. Changed the default value of yarn.acl.enable in yarn-default.xml + to be consistent with what exists (false) in the code and documentation. + (Kenji Kikushima via vinodkv) + + YARN-1669. Modified RM HA handling of protocol level service-ACLS to + be available across RM failover by making using of a remote + configuration-provider. (Xuan Gong via vinodkv) + OPTIMIZATIONS BUG FIXES @@ -148,6 +188,12 @@ Release 2.4.0 - UNRELEASED YARN-1632. TestApplicationMasterServices should be under org.apache.hadoop.yarn.server.resourcemanager package (Chen He via jeagles) + YARN-1673. Fix option parsing in YARN's application CLI after it is broken + by YARN-967. (Mayank Bansal via vinodkv) + + YARN-1684. Fixed history server heap size in yarn script. (Billie Rinaldi + via zjshen) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -581,6 +627,15 @@ Release 2.3.0 - UNRELEASED YARN-1629. IndexOutOfBoundsException in MaxRunningAppsEnforcer (Sandy Ryza) + YARN-1628. Fixed the test failure in TestContainerManagerSecurity. (Vinod + Kumar Vavilapalli via zjshen) + + YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod + Kumar Vavilapalli via zjshen) + + YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. (Vinod + Kumar Vavilapalli via zjshen) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn Fri Feb 7 02:43:04 2014 @@ -204,7 +204,7 @@ elif [ "$COMMAND" = "historyserver" ] ; CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/ahs-config/log4j.properties CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' YARN_OPTS="$YARN_OPTS $YARN_HISTORYSERVER_OPTS" - if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then + if [ "$YARN_HISTORYSERVER_HEAPSIZE" != "" ]; then JAVA_HEAP_MAX="-Xmx""$YARN_HISTORYSERVER_HEAPSIZE""m" fi elif [ "$COMMAND" = "nodemanager" ] ; then Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd Fri Feb 7 02:43:04 2014 @@ -207,7 +207,7 @@ goto :eof set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\ahs-config\log4j.properties set CLASS=org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer set YARN_OPTS=%YARN_OPTS% %HADOOP_HISTORYSERVER_OPTS% - if defined YARN_RESOURCEMANAGER_HEAPSIZE ( + if defined YARN_HISTORYSERVER_HEAPSIZE ( set JAVA_HEAP_MAX=-Xmx%YARN_HISTORYSERVER_HEAPSIZE%m ) goto :eof Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java Fri Feb 7 02:43:04 2014 @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.api.proto import java.util.EnumSet; import java.util.Set; -import org.apache.commons.collections.buffer.UnboundedFifoBuffer; import org.apache.commons.lang.math.LongRange; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -56,6 +55,86 @@ public abstract class GetApplicationsReq * ResourceManager. *

* + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + * + *

Setting any of the parameters to null, would just disable that + * filter

+ * + * @param scope {@link ApplicationsRequestScope} to filter by + * @param users list of users to filter by + * @param queues list of scheduler queues to filter by + * @param applicationTypes types of applications + * @param applicationTags application tags to filter by + * @param applicationStates application states to filter by + * @param startRange range of application start times to filter by + * @param finishRange range of application finish times to filter by + * @param limit number of applications to limit to + * @return {@link GetApplicationsRequest} to be used with + * {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)} + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + ApplicationsRequestScope scope, + Set users, + Set queues, + Set applicationTypes, + Set applicationTags, + EnumSet applicationStates, + LongRange startRange, + LongRange finishRange, + Long limit) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + if (scope != null) { + request.setScope(scope); + } + request.setUsers(users); + request.setQueues(queues); + request.setApplicationTypes(applicationTypes); + request.setApplicationTags(applicationTags); + request.setApplicationStates(applicationStates); + if (startRange != null) { + request.setStartRange( + startRange.getMinimumLong(), startRange.getMaximumLong()); + } + if (finishRange != null) { + request.setFinishRange( + finishRange.getMinimumLong(), finishRange.getMaximumLong()); + } + if (limit != null) { + request.setLimit(limit); + } + return request; + } + + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application types in the cluster from the + * ResourceManager. + *

+ * + * @param scope {@link ApplicationsRequestScope} to filter by + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + ApplicationsRequestScope scope) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + request.setScope(scope); + return request; + } + + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application types in the cluster from the + * ResourceManager. + *

+ * * * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) */ @@ -257,4 +336,40 @@ public abstract class GetApplicationsReq @Private @Unstable public abstract void setFinishRange(long begin, long end); + + /** + * Get the tags to filter applications on + * + * @return list of tags to filter on + */ + @Private + @Unstable + public abstract Set getApplicationTags(); + + /** + * Set the list of tags to filter applications on + * + * @param tags list of tags to filter on + */ + @Private + @Unstable + public abstract void setApplicationTags(Set tags); + + /** + * Get the {@link ApplicationsRequestScope} of applications to be filtered. + * + * @return {@link ApplicationsRequestScope} of applications to return. + */ + @Private + @Unstable + public abstract ApplicationsRequestScope getScope(); + + /** + * Set the {@link ApplicationsRequestScope} of applications to filter. + * + * @param scope scope to use for filtering applications + */ + @Private + @Unstable + public abstract void setScope(ApplicationsRequestScope scope); } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java Fri Feb 7 02:43:04 2014 @@ -25,6 +25,8 @@ import org.apache.hadoop.classification. import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; +import java.util.Set; + /** *

ApplicationReport is a report of an application.

* @@ -321,6 +323,18 @@ public abstract class ApplicationReport @Unstable public abstract void setApplicationType(String applicationType); + /** + * Get all tags corresponding to the application + * @return Application's tags + */ + @Public + @Stable + public abstract Set getApplicationTags(); + + @Private + @Unstable + public abstract void setApplicationTags(Set tags); + @Private @Stable public abstract void setAMRMToken(Token amRmToken); Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java Fri Feb 7 02:43:04 2014 @@ -25,8 +25,11 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.Records; +import java.util.Set; + /** *

ApplicationSubmissionContext represents all of the * information needed by the ResourceManager to launch @@ -284,7 +287,6 @@ public abstract class ApplicationSubmiss @Stable public abstract void setApplicationType(String applicationType); - /** * Get the flag which indicates whether to keep containers across application * attempts or not. @@ -314,4 +316,26 @@ public abstract class ApplicationSubmiss @Stable public abstract void setKeepContainersAcrossApplicationAttempts( boolean keepContainers); + + /** + * Get tags for the application + * + * @return the application tags + */ + @Public + @Stable + public abstract Set getApplicationTags(); + + /** + * Set tags for the application. A maximum of + * {@link YarnConfiguration#APPLICATION_MAX_TAGS} are allowed + * per application. Each tag can be at most + * {@link YarnConfiguration#APPLICATION_MAX_TAG_LENGTH} + * characters, and can contain only ASCII characters. + * + * @param tags tags to set + */ + @Public + @Stable + public abstract void setApplicationTags(Set tags); } \ No newline at end of file Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java Fri Feb 7 02:43:04 2014 @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -49,14 +50,14 @@ import org.apache.hadoop.classification. @XmlAccessorType(XmlAccessType.NONE) @Public @Unstable -public class ATSEntity { +public class ATSEntity implements Comparable { private String entityType; private String entityId; - private long startTime; + private Long startTime; private List events = new ArrayList(); - private Map> relatedEntities = - new HashMap>(); + private Map> relatedEntities = + new HashMap>(); private Map primaryFilters = new HashMap(); private Map otherInfo = @@ -112,7 +113,7 @@ public class ATSEntity { * @return the start time of the entity */ @XmlElement(name = "starttime") - public long getStartTime() { + public Long getStartTime() { return startTime; } @@ -122,7 +123,7 @@ public class ATSEntity { * @param startTime * the start time of the entity */ - public void setStartTime(long startTime) { + public void setStartTime(Long startTime) { this.startTime = startTime; } @@ -172,26 +173,25 @@ public class ATSEntity { * @return the related entities */ @XmlElement(name = "relatedentities") - public Map> getRelatedEntities() { + public Map> getRelatedEntities() { return relatedEntities; } /** - * Add a list of entity of the same type to the existing related entity map + * Add an entity to the existing related entity map * * @param entityType * the entity type - * @param entityIds - * a list of entity Ids + * @param entityId + * the entity Id */ - public void addRelatedEntity(String entityType, List entityIds) { - List thisRelatedEntity = relatedEntities.get(entityType); - relatedEntities.put(entityType, entityIds); + public void addRelatedEntity(String entityType, String entityId) { + List thisRelatedEntity = relatedEntities.get(entityType); if (thisRelatedEntity == null) { - relatedEntities.put(entityType, entityIds); - } else { - thisRelatedEntity.addAll(entityIds); + thisRelatedEntity = new ArrayList(); + relatedEntities.put(entityType, thisRelatedEntity); } + thisRelatedEntity.add(entityId); } /** @@ -200,11 +200,10 @@ public class ATSEntity { * @param relatedEntities * a map of related entities */ - public void addRelatedEntities( - Map> relatedEntities) { - for (Map.Entry> relatedEntity : relatedEntities - .entrySet()) { - List thisRelatedEntity = + public void addRelatedEntities(Map> relatedEntities) { + for (Entry> relatedEntity : + relatedEntities.entrySet()) { + List thisRelatedEntity = this.relatedEntities.get(relatedEntity.getKey()); if (thisRelatedEntity == null) { this.relatedEntities.put( @@ -222,7 +221,7 @@ public class ATSEntity { * a map of related entities */ public void setRelatedEntities( - Map> relatedEntities) { + Map> relatedEntities) { this.relatedEntities = relatedEntities; } @@ -311,4 +310,92 @@ public class ATSEntity { this.otherInfo = otherInfo; } + @Override + public int hashCode() { + // generated by eclipse + final int prime = 31; + int result = 1; + result = prime * result + ((entityId == null) ? 0 : entityId.hashCode()); + result = + prime * result + ((entityType == null) ? 0 : entityType.hashCode()); + result = prime * result + ((events == null) ? 0 : events.hashCode()); + result = prime * result + ((otherInfo == null) ? 0 : otherInfo.hashCode()); + result = + prime * result + + ((primaryFilters == null) ? 0 : primaryFilters.hashCode()); + result = + prime * result + + ((relatedEntities == null) ? 0 : relatedEntities.hashCode()); + result = prime * result + ((startTime == null) ? 0 : startTime.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + // generated by eclipse + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ATSEntity other = (ATSEntity) obj; + if (entityId == null) { + if (other.entityId != null) + return false; + } else if (!entityId.equals(other.entityId)) + return false; + if (entityType == null) { + if (other.entityType != null) + return false; + } else if (!entityType.equals(other.entityType)) + return false; + if (events == null) { + if (other.events != null) + return false; + } else if (!events.equals(other.events)) + return false; + if (otherInfo == null) { + if (other.otherInfo != null) + return false; + } else if (!otherInfo.equals(other.otherInfo)) + return false; + if (primaryFilters == null) { + if (other.primaryFilters != null) + return false; + } else if (!primaryFilters.equals(other.primaryFilters)) + return false; + if (relatedEntities == null) { + if (other.relatedEntities != null) + return false; + } else if (!relatedEntities.equals(other.relatedEntities)) + return false; + if (startTime == null) { + if (other.startTime != null) + return false; + } else if (!startTime.equals(other.startTime)) + return false; + return true; + } + + @Override + public int compareTo(ATSEntity other) { + int comparison = entityType.compareTo(other.entityType); + if (comparison == 0) { + long thisStartTime = + startTime == null ? Long.MIN_VALUE : startTime; + long otherStartTime = + other.startTime == null ? Long.MIN_VALUE : other.startTime; + if (thisStartTime > otherStartTime) { + return -1; + } else if (thisStartTime < otherStartTime) { + return 1; + } else { + return entityId.compareTo(other.entityId); + } + } else { + return comparison; + } + } + } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java Fri Feb 7 02:43:04 2014 @@ -39,7 +39,7 @@ import org.apache.hadoop.classification. @XmlAccessorType(XmlAccessType.NONE) @Public @Unstable -public class ATSEvent { +public class ATSEvent implements Comparable { private long timestamp; private String eventType; @@ -131,4 +131,42 @@ public class ATSEvent { this.eventInfo = eventInfo; } + @Override + public int compareTo(ATSEvent other) { + if (timestamp > other.timestamp) { + return -1; + } else if (timestamp < other.timestamp) { + return 1; + } else { + return eventType.compareTo(other.eventType); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + ATSEvent atsEvent = (ATSEvent) o; + + if (timestamp != atsEvent.timestamp) + return false; + if (!eventType.equals(atsEvent.eventType)) + return false; + if (eventInfo != null ? !eventInfo.equals(atsEvent.eventInfo) : + atsEvent.eventInfo != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = (int) (timestamp ^ (timestamp >>> 32)); + result = 31 * result + eventType.hashCode(); + result = 31 * result + (eventInfo != null ? eventInfo.hashCode() : 0); + return result; + } } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java Fri Feb 7 02:43:04 2014 @@ -160,7 +160,7 @@ public class ATSEvents { * @param event * a single event */ - public void addEntity(ATSEvent event) { + public void addEvent(ATSEvent event) { events.add(event); } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java Fri Feb 7 02:43:04 2014 @@ -21,10 +21,13 @@ package org.apache.hadoop.yarn.conf; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import java.net.InetSocketAddress; import java.util.Collection; @InterfaceAudience.Private @@ -108,8 +111,7 @@ public class HAUtil { String errmsg = iae.getMessage(); if (confKey == null) { // Error at addSuffix - errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, - getRMHAId(conf)); + errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, id); } throwBadConfigurationException(errmsg); } @@ -122,10 +124,18 @@ public class HAUtil { } private static void verifyAndSetCurrentRMHAId(Configuration conf) { - String rmId = conf.getTrimmed(YarnConfiguration.RM_HA_ID); + String rmId = getRMHAId(conf); if (rmId == null) { - throwBadConfigurationException( - getNeedToSetValueMessage(YarnConfiguration.RM_HA_ID)); + StringBuilder msg = new StringBuilder(); + msg.append("Can not find valid RM_HA_ID. None of "); + for (String id : conf + .getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS)) { + msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id) + " "); + } + msg.append(" are matching" + + " the local address OR " + YarnConfiguration.RM_HA_ID + " is not" + + " specified in HA Configuration"); + throwBadConfigurationException(msg.toString()); } else { Collection ids = getRMHAIds(conf); if (!ids.contains(rmId)) { @@ -179,7 +189,34 @@ public class HAUtil { * @return RM Id on success */ public static String getRMHAId(Configuration conf) { - return conf.get(YarnConfiguration.RM_HA_ID); + int found = 0; + String currentRMId = conf.getTrimmed(YarnConfiguration.RM_HA_ID); + if(currentRMId == null) { + for(String rmId : getRMHAIds(conf)) { + String key = addSuffix(YarnConfiguration.RM_ADDRESS, rmId); + String addr = conf.get(key); + if (addr == null) { + continue; + } + InetSocketAddress s; + try { + s = NetUtils.createSocketAddr(addr); + } catch (Exception e) { + LOG.warn("Exception in creating socket address " + addr, e); + continue; + } + if (!s.isUnresolved() && NetUtils.isLocalAddress(s.getAddress())) { + currentRMId = rmId.trim(); + found++; + } + } + } + if (found > 1) { // Only one address must match the local address + String msg = "The HA Configuration has multiple addresses that match " + + "local node's address."; + throw new HadoopIllegalArgumentException(msg); + } + return currentRMId; } @VisibleForTesting Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Feb 7 02:43:04 2014 @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -37,8 +38,26 @@ import org.apache.hadoop.yarn.api.Applic @Evolving public class YarnConfiguration extends Configuration { + @Private + public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + + @Private + public static final String HADOOP_POLICY_CONFIGURATION_FILE = + "hadoop-policy.xml"; + + @Private + public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; + + @Private + public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; + + @Evolving + public static final int APPLICATION_MAX_TAGS = 10; + + @Evolving + public static final int APPLICATION_MAX_TAG_LENGTH = 100; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; - private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; static { Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE); @@ -329,6 +348,16 @@ public class YarnConfiguration extends C public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids"; public static final String RM_HA_ID = RM_HA_PREFIX + "id"; + /** Store the related configuration files in File System */ + public static final String FS_BASED_RM_CONF_STORE = RM_PREFIX + + "configuration.file-system-based-store"; + public static final String DEFAULT_FS_BASED_RM_CONF_STORE = "/yarn/conf"; + + public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX + + "configuration.provider-class"; + public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = + "org.apache.hadoop.yarn.LocalConfigurationProvider"; + @Private public static final List RM_SERVICES_ADDRESS_CONF_KEYS = Collections.unmodifiableList(Arrays.asList( @@ -1000,6 +1029,15 @@ public class YarnConfiguration extends C AHS_PREFIX + "webapp.spnego-keytab-file"; //////////////////////////////// + // ATS Configs + //////////////////////////////// + + public static final String ATS_PREFIX = YARN_PREFIX + "ats."; + + /** ATS store class */ + public static final String ATS_STORE = ATS_PREFIX + "store.class"; + + //////////////////////////////// // Other Configs //////////////////////////////// Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Fri Feb 7 02:43:04 2014 @@ -190,6 +190,7 @@ message ApplicationReportProto { optional float progress = 17; optional string applicationType = 18; optional hadoop.common.TokenProto am_rm_token = 19; + repeated string applicationTags = 20; } message ApplicationAttemptReportProto { @@ -287,6 +288,7 @@ message ApplicationSubmissionContextProt optional ResourceProto resource = 9; optional string applicationType = 10 [default = "YARN"]; optional bool keep_containers_across_application_attempts = 11 [default = false]; + repeated string applicationTags = 12; } enum ApplicationAccessTypeProto { Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Fri Feb 7 02:43:04 2014 @@ -136,6 +136,12 @@ message MoveApplicationAcrossQueuesReque message MoveApplicationAcrossQueuesResponseProto { } +enum ApplicationsRequestScopeProto { + ALL = 0; + VIEWABLE = 1; + OWN = 2; +} + message GetApplicationsRequestProto { repeated string application_types = 1; repeated YarnApplicationStateProto application_states = 2; @@ -146,6 +152,8 @@ message GetApplicationsRequestProto { optional int64 start_end = 7; optional int64 finish_begin = 8; optional int64 finish_end = 9; + repeated string applicationTags = 10; + optional ApplicationsRequestScopeProto scope = 11 [default = ALL]; } message GetApplicationsResponseProto { Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Fri Feb 7 02:43:04 2014 @@ -232,7 +232,6 @@ public class ApplicationMaster { private static final String shellArgsPath = "shellArgs"; private volatile boolean done; - private volatile boolean success; private ByteBuffer allTokens; @@ -254,8 +253,8 @@ public class ApplicationMaster { if (!doRun) { System.exit(0); } - result = appMaster.run(); - appMaster.finish(); + appMaster.run(); + result = appMaster.finish(); } catch (Throwable t) { LOG.fatal("Error running ApplicationMaster", t); System.exit(1); @@ -480,7 +479,7 @@ public class ApplicationMaster { * @throws IOException */ @SuppressWarnings({ "unchecked" }) - public boolean run() throws YarnException, IOException { + public void run() throws YarnException, IOException { LOG.info("Starting ApplicationMaster"); Credentials credentials = @@ -561,7 +560,6 @@ public class ApplicationMaster { amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainersToRequest); - return success; } @VisibleForTesting @@ -569,7 +567,8 @@ public class ApplicationMaster { return new NMCallbackHandler(this); } - protected void finish() { + @VisibleForTesting + protected boolean finish() { // wait for completion. while (!done && (numCompletedContainers.get() != numTotalContainers)) { @@ -600,7 +599,7 @@ public class ApplicationMaster { FinalApplicationStatus appStatus; String appMessage = null; - success = true; + boolean success = true; if (numFailedContainers.get() == 0 && numCompletedContainers.get() == numTotalContainers) { appStatus = FinalApplicationStatus.SUCCEEDED; @@ -621,6 +620,8 @@ public class ApplicationMaster { } amRMClient.stop(); + + return success; } private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler { Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java Fri Feb 7 02:43:04 2014 @@ -18,13 +18,13 @@ package org.apache.hadoop.yarn.applications.distributedshell; +import java.nio.ByteBuffer; +import java.util.Map; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ContainerId; -import java.nio.ByteBuffer; -import java.util.Map; - public class ContainerLaunchFailAppMaster extends ApplicationMaster { private static final Log LOG = @@ -66,8 +66,8 @@ public class ContainerLaunchFailAppMaste if (!doRun) { System.exit(0); } - result = appMaster.run(); - appMaster.finish(); + appMaster.run(); + result = appMaster.finish(); } catch (Throwable t) { LOG.fatal("Error running ApplicationMaster", t); System.exit(1); Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java Fri Feb 7 02:43:04 2014 @@ -29,8 +29,8 @@ public class TestDSFailedAppMaster exten private static final Log LOG = LogFactory.getLog(TestDSFailedAppMaster.class); @Override - public boolean run() throws YarnException, IOException { - boolean res = super.run(); + public void run() throws YarnException, IOException { + super.run(); // for the 2nd attempt. if (appAttemptID.getAttemptId() == 2) { @@ -39,11 +39,12 @@ public class TestDSFailedAppMaster exten // numRequestedContainers should be set to 0. if (numAllocatedContainers.get() != 1 || numRequestedContainers.get() != 0) { - LOG.info("Application Master failed. exiting"); + LOG.info("NumAllocatedContainers is " + numAllocatedContainers.get() + + " and NumRequestedContainers is " + numAllocatedContainers.get() + + ".Application Master failed. exiting"); System.exit(200); } } - return res; } public static void main(String[] args) { @@ -54,7 +55,7 @@ public class TestDSFailedAppMaster exten if (!doRun) { System.exit(0); } - result = appMaster.run(); + appMaster.run(); if (appMaster.appAttemptID.getAttemptId() == 1) { try { // sleep some time, wait for the AM to launch a container. @@ -63,7 +64,7 @@ public class TestDSFailedAppMaster exten // fail the first am. System.exit(100); } - appMaster.finish(); + result = appMaster.finish(); } catch (Throwable t) { System.exit(1); } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Fri Feb 7 02:43:04 2014 @@ -197,7 +197,7 @@ public class ApplicationCLI extends Yarn listApplications(appTypes, appStates); } } else if (cliParser.hasOption(KILL_CMD)) { - if (args.length != 2) { + if (args.length != 3) { printUsage(opts); return exitCode; } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Fri Feb 7 02:43:04 2014 @@ -681,7 +681,7 @@ public class TestYarnCLI { sysOutStream.reset(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); result = - cli.run(new String[] { "-kill", applicationId.toString(), "args" }); + cli.run(new String[] {"application", "-kill", applicationId.toString(), "args" }); verify(spyCli).printUsage(any(Options.class)); Assert.assertEquals(createApplicationCLIHelpMessage(), sysOutStream.toString()); @@ -717,7 +717,7 @@ public class TestYarnCLI { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); - int result = cli.run(new String[] { "-kill", applicationId.toString() }); + int result = cli.run(new String[] { "application","-kill", applicationId.toString() }); assertEquals(0, result); verify(client, times(0)).killApplication(any(ApplicationId.class)); verify(sysOut).println( @@ -730,7 +730,7 @@ public class TestYarnCLI { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); - result = cli.run(new String[] { "-kill", applicationId.toString() }); + result = cli.run(new String[] { "application","-kill", applicationId.toString() }); assertEquals(0, result); verify(client).killApplication(any(ApplicationId.class)); verify(sysOut).println("Killing application application_1234_0005"); @@ -740,7 +740,7 @@ public class TestYarnCLI { .getApplicationReport(applicationId); cli = createAndGetAppCLI(); try { - cli.run(new String[] { "-kill", applicationId.toString() }); + cli.run(new String[] { "application","-kill", applicationId.toString() }); Assert.fail(); } catch (Exception ex) { Assert.assertTrue(ex instanceof ApplicationNotFoundException); Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java Fri Feb 7 02:43:04 2014 @@ -27,6 +27,7 @@ import java.util.Set; import org.apache.commons.lang.math.LongRange; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; @@ -49,6 +50,8 @@ public class GetApplicationsRequestPBImp Set queues = null; long limit = Long.MAX_VALUE; LongRange start = null, finish = null; + private Set applicationTags; + private ApplicationsRequestScope scope; public GetApplicationsRequestPBImpl() { builder = GetApplicationsRequestProto.newBuilder(); @@ -112,6 +115,12 @@ public class GetApplicationsRequestPBImp }; builder.addAllApplicationStates(iterable); } + if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.addAllApplicationTags(this.applicationTags); + } + if (this.scope != null) { + builder.setScope(ProtoUtils.convertToProtoFormat(scope)); + } } private void addLocalApplicationTypesToProto() { @@ -187,12 +196,64 @@ public class GetApplicationsRequestPBImp this.applicationTypes = applicationTypes; } + private void initApplicationTags() { + if (this.applicationTags != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + this.applicationTags = new HashSet(); + this.applicationTags.addAll(p.getApplicationTagsList()); + } + + @Override + public Set getApplicationTags() { + initApplicationTags(); + return this.applicationTags; + } + + @Override + public void setApplicationTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearApplicationTags(); + this.applicationTags = null; + return; + } + // Convert applicationTags to lower case and add + this.applicationTags = new HashSet(); + for (String tag : tags) { + this.applicationTags.add(tag.toLowerCase()); + } + } + @Override public EnumSet getApplicationStates() { initApplicationStates(); return this.applicationStates; } + private void initScope() { + if (this.scope != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + this.scope = ProtoUtils.convertFromProtoFormat(p.getScope()); + } + + @Override + public ApplicationsRequestScope getScope() { + initScope(); + return this.scope; + } + + public void setScope(ApplicationsRequestScope scope) { + maybeInitBuilder(); + if (scope == null) { + builder.clearScope(); + } + this.scope = scope; + } + @Override public void setApplicationStates(EnumSet applicationStates) { maybeInitBuilder(); @@ -223,7 +284,6 @@ public class GetApplicationsRequestPBImp return this.users; } - @Override public void setUsers(Set users) { maybeInitBuilder(); if (users == null) { Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Fri Feb 7 02:43:04 2014 @@ -38,6 +38,9 @@ import org.apache.hadoop.yarn.proto.Yarn import com.google.protobuf.TextFormat; +import java.util.HashSet; +import java.util.Set; + @Private @Unstable public class ApplicationReportPBImpl extends ApplicationReport { @@ -49,6 +52,7 @@ public class ApplicationReportPBImpl ext private ApplicationAttemptId currentApplicationAttemptId; private Token clientToAMToken = null; private Token amRmToken = null; + private Set applicationTags = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -245,6 +249,21 @@ public class ApplicationReportPBImpl ext return amRmToken; } + private void initApplicationTags() { + if (this.applicationTags != null) { + return; + } + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + this.applicationTags = new HashSet(); + this.applicationTags.addAll(p.getApplicationTagsList()); + } + + @Override + public Set getApplicationTags() { + initApplicationTags(); + return this.applicationTags; + } + @Override public void setApplicationId(ApplicationId applicationId) { maybeInitBuilder(); @@ -356,6 +375,15 @@ public class ApplicationReportPBImpl ext } @Override + public void setApplicationTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearApplicationTags(); + } + this.applicationTags = tags; + } + + @Override public void setDiagnostics(String diagnostics) { maybeInitBuilder(); if (diagnostics == null) { @@ -450,6 +478,9 @@ public class ApplicationReportPBImpl ext builder.getAmRmToken())) { builder.setAmRmToken(convertToProtoFormat(this.amRmToken)); } + if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.addAllApplicationTags(this.applicationTags); + } } private void mergeLocalToProto() { Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java Fri Feb 7 02:43:04 2014 @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import com.google.common.base.CharMatcher; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -25,6 +26,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder; @@ -34,6 +36,9 @@ import org.apache.hadoop.yarn.proto.Yarn import com.google.protobuf.TextFormat; +import java.util.HashSet; +import java.util.Set; + @Private @Unstable public class ApplicationSubmissionContextPBImpl @@ -47,6 +52,7 @@ extends ApplicationSubmissionContext { private Priority priority = null; private ContainerLaunchContext amContainer = null; private Resource resource = null; + private Set applicationTags = null; public ApplicationSubmissionContextPBImpl() { builder = ApplicationSubmissionContextProto.newBuilder(); @@ -100,6 +106,9 @@ extends ApplicationSubmissionContext { builder.getResource())) { builder.setResource(convertToProtoFormat(this.resource)); } + if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.addAllApplicationTags(this.applicationTags); + } } private void mergeLocalToProto() { @@ -196,7 +205,22 @@ extends ApplicationSubmissionContext { } return (p.getApplicationType()); } - + + private void initApplicationTags() { + if (this.applicationTags != null) { + return; + } + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + this.applicationTags = new HashSet(); + this.applicationTags.addAll(p.getApplicationTagsList()); + } + + @Override + public Set getApplicationTags() { + initApplicationTags(); + return this.applicationTags; + } + @Override public void setQueue(String queue) { maybeInitBuilder(); @@ -217,6 +241,40 @@ extends ApplicationSubmissionContext { builder.setApplicationType((applicationType)); } + private void checkTags(Set tags) { + if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) { + throw new IllegalArgumentException("Too many applicationTags, a maximum of only " + + YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!"); + } + for (String tag : tags) { + if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) { + throw new IllegalArgumentException("Tag " + tag + " is too long, " + + "maximum allowed length of a tag is " + + YarnConfiguration.APPLICATION_MAX_TAG_LENGTH); + } + if (!CharMatcher.ASCII.matchesAllOf(tag)) { + throw new IllegalArgumentException("A tag can only have ASCII " + + "characters! Invalid tag - " + tag); + } + } + } + + @Override + public void setApplicationTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearApplicationTags(); + this.applicationTags = null; + return; + } + checkTags(tags); + // Convert applicationTags to lower case and add + this.applicationTags = new HashSet(); + for (String tag : tags) { + this.applicationTags.add(tag.toLowerCase()); + } + } + @Override public ContainerLaunchContext getAMContainerSpec() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java Fri Feb 7 02:43:04 2014 @@ -22,6 +22,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -50,6 +51,7 @@ import org.apache.hadoop.yarn.proto.Yarn import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import com.google.protobuf.ByteString; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; @Private @Unstable @@ -114,6 +116,18 @@ public class ProtoUtils { } /* + * ApplicationsRequestScope + */ + public static YarnServiceProtos.ApplicationsRequestScopeProto + convertToProtoFormat(ApplicationsRequestScope e) { + return YarnServiceProtos.ApplicationsRequestScopeProto.valueOf(e.name()); + } + public static ApplicationsRequestScope convertFromProtoFormat + (YarnServiceProtos.ApplicationsRequestScopeProto e) { + return ApplicationsRequestScope.valueOf(e.name()); + } + + /* * ApplicationResourceUsageReport */ public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) { Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Feb 7 02:43:04 2014 @@ -130,7 +130,7 @@ Are acls enabled. yarn.acl.enable - true + false @@ -395,7 +395,9 @@ the Active mode when prompted to. (2) The nodes in the RM ensemble are listed in yarn.resourcemanager.ha.rm-ids - (3) The id of each RM comes from yarn.resourcemanager.ha.id + (3) The id of each RM either comes from yarn.resourcemanager.ha.id + if yarn.resourcemanager.ha.id is explicitly specified or can be + figured out by matching yarn.resourcemanager.address.{id} with local address (4) The actual physical addresses come from the configs of the pattern - {rpc-config}.{id} yarn.resourcemanager.ha.enabled @@ -442,7 +444,10 @@ The id (string) of the current RM. When HA is enabled, this - is a required config. See description of yarn.resourcemanager.ha.enabled + is an optional config. The id of current RM can be set by explicitly + specifying yarn.resourcemanager.ha.id or figured out by matching + yarn.resourcemanager.address.{id} with local address + See description of yarn.resourcemanager.ha.enabled for full details on how this is used. yarn.resourcemanager.ha.id @@ -588,6 +593,18 @@ org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore + + The class to use as the configuration provider. + If org.apache.hadoop.yarn.LocalConfigurationProvider is used, + the local configuration will be loaded. + If org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider is used, + the configuration which will be loaded should be uploaded to remote File system first. + > + yarn.resourcemanager.configuration.provider-class + org.apache.hadoop.yarn.LocalConfigurationProvider + + + The hostname of the NM. @@ -1120,6 +1137,14 @@ org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore + + + + Store class name for application timeline store + yarn.ats.store.class + org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore + + The interval that the yarn client library uses to poll the Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java Fri Feb 7 02:43:04 2014 @@ -18,10 +18,13 @@ package org.apache.hadoop.yarn.api.records.apptimeline; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import junit.framework.Assert; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; import org.junit.Test; public class TestApplicationTimelineRecords { @@ -42,10 +45,8 @@ public class TestApplicationTimelineReco event.addEventInfo("key2", "val2"); entity.addEvent(event); } - entity.addRelatedEntity( - "test ref type 1", Arrays.asList((Object) "test ref id 1")); - entity.addRelatedEntity( - "test ref type 2", Arrays.asList((Object) "test ref id 2")); + entity.addRelatedEntity("test ref type 1", "test ref id 1"); + entity.addRelatedEntity("test ref type 2", "test ref id 2"); entity.addPrimaryFilter("pkey1", "pval1"); entity.addPrimaryFilter("pkey2", "pval2"); entity.addOtherInfo("okey1", "oval1"); @@ -83,7 +84,7 @@ public class TestApplicationTimelineReco event.setEventType("event type " + i); event.addEventInfo("key1", "val1"); event.addEventInfo("key2", "val2"); - partEvents.addEntity(event); + partEvents.addEvent(event); } events.addEvent(partEvents); } @@ -110,4 +111,36 @@ public class TestApplicationTimelineReco Assert.assertEquals(2, event22.getEventInfo().size()); } + @Test + public void testATSPutErrors() { + ATSPutErrors atsPutErrors = new ATSPutErrors(); + ATSPutError error1 = new ATSPutError(); + error1.setEntityId("entity id 1"); + error1.setEntityId("entity type 1"); + error1.setErrorCode(1); + atsPutErrors.addError(error1); + List errors = new ArrayList(); + errors.add(error1); + ATSPutError error2 = new ATSPutError(); + error2.setEntityId("entity id 2"); + error2.setEntityId("entity type 2"); + error2.setErrorCode(2); + errors.add(error2); + atsPutErrors.addErrors(errors); + + Assert.assertEquals(3, atsPutErrors.getErrors().size()); + ATSPutError e = atsPutErrors.getErrors().get(0); + Assert.assertEquals(error1.getEntityId(), e.getEntityId()); + Assert.assertEquals(error1.getEntityType(), e.getEntityType()); + Assert.assertEquals(error1.getErrorCode(), e.getErrorCode()); + e = atsPutErrors.getErrors().get(1); + Assert.assertEquals(error1.getEntityId(), e.getEntityId()); + Assert.assertEquals(error1.getEntityType(), e.getEntityType()); + Assert.assertEquals(error1.getErrorCode(), e.getErrorCode()); + e = atsPutErrors.getErrors().get(2); + Assert.assertEquals(error2.getEntityId(), e.getEntityId()); + Assert.assertEquals(error2.getEntityType(), e.getEntityType()); + Assert.assertEquals(error2.getErrorCode(), e.getErrorCode()); + } + } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java Fri Feb 7 02:43:04 2014 @@ -27,11 +27,14 @@ import org.apache.hadoop.metrics2.source import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -51,6 +54,7 @@ public class ApplicationHistoryServer ex ApplicationHistoryClientService ahsClientService; ApplicationHistoryManager historyManager; + ApplicationTimelineStore timelineStore; private WebApp webApp; public ApplicationHistoryServer() { @@ -63,6 +67,8 @@ public class ApplicationHistoryServer ex ahsClientService = createApplicationHistoryClientService(historyManager); addService(ahsClientService); addService((Service) historyManager); + timelineStore = createApplicationTimelineStore(conf); + addIfService(timelineStore); super.serviceInit(conf); } @@ -135,6 +141,15 @@ public class ApplicationHistoryServer ex return new ApplicationHistoryManagerImpl(); } + protected ApplicationTimelineStore createApplicationTimelineStore( + Configuration conf) { + // TODO: need to replace the MemoryApplicationTimelineStore.class with the + // LevelDB implementation + return ReflectionUtils.newInstance(conf.getClass( + YarnConfiguration.ATS_STORE, MemoryApplicationTimelineStore.class, + ApplicationTimelineStore.class), conf); + } + protected void startWebApp() { String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig()); LOG.info("Instantiating AHSWebApp at " + bindAddress); @@ -148,7 +163,8 @@ public class ApplicationHistoryServer ex YarnConfiguration.AHS_WEBAPP_SPNEGO_USER_NAME_KEY) .withHttpSpnegoKeytabKey( YarnConfiguration.AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) - .at(bindAddress).start(new AHSWebApp(historyManager)); + .at(bindAddress) + .start(new AHSWebApp(historyManager, timelineStore)); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e);