hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1517887 [1/2] - in /hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoo...
Date Tue, 27 Aug 2013 17:05:15 GMT
Author: arp
Date: Tue Aug 27 17:05:13 2013
New Revision: 1517887

URL: http://svn.apache.org/r1517887
Log:
Merge all changes from trunk to branch HDFS-2832.

Modified:
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Tue Aug 27 17:05:13 2013
@@ -23,6 +23,8 @@ Release 2.3.0 - UNRELEASED
 
   IMPROVEMENTS
 
+    YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -42,7 +44,16 @@ Release 2.1.1-beta - UNRELEASED
   IMPROVEMENTS
 
     YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza).
+
+    YARN-1074. Cleaned up YARN CLI application list to only display running
+    applications by default. (Xuan Gong via vinodkv)
     
+    YARN-1093. Corrections to Fair Scheduler documentation (Wing Yew Poon via
+    Sandy Ryza)
+
+    YARN-942. In Fair Scheduler documentation, inconsistency on which
+    properties have prefix (Akira Ajisaka via Sandy Ryza)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -85,6 +96,15 @@ Release 2.1.1-beta - UNRELEASED
     YARN-1082. Create base directories on HDFS after RM login to ensure RM
     recovery doesn't fail in secure mode. (vinodkv via acmurthy)
 
+    YARN-1085. Modified YARN and MR2 web-apps to do HTTP authentication in
+    secure setup with kerberos. (Omkar Vinit Joshi via vinodkv)
+
+    YARN-1094. Fixed a blocker with RM restart code because of which RM crashes
+    when try to recover an existing app. (vinodkv)
+
+    YARN-1008. MiniYARNCluster with multiple nodemanagers, all nodes have same 
+    key for allocations. (tucu)
+
 Release 2.1.0-beta - 2013-08-22
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java Tue Aug 27 17:05:13 2013
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords;
 
+import java.util.EnumSet;
 import java.util.Set;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -25,6 +26,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.util.Records;
 
 /**
@@ -45,13 +47,65 @@ public abstract class GetApplicationsReq
     return request;
   }
 
+  /**
+   * <p>
+   * The request from clients to get a report of Applications matching the
+   * giving application types in the cluster from the
+   * <code>ResourceManager</code>.
+   * </p>
+   *
+   *
+   * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+   */
+  @Public
+  @Stable
+  public static GetApplicationsRequest
+      newInstance(Set<String> applicationTypes) {
+    GetApplicationsRequest request =
+        Records.newRecord(GetApplicationsRequest.class);
+    request.setApplicationTypes(applicationTypes);
+    return request;
+  }
+
+  /**
+   * <p>
+   * The request from clients to get a report of Applications matching the
+   * giving application states in the cluster from the
+   * <code>ResourceManager</code>.
+   * </p>
+   *
+   *
+   * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+   */
   @Public
   @Stable
   public static GetApplicationsRequest newInstance(
-      Set<String> applicationTypes) {
+      EnumSet<YarnApplicationState> applicationStates) {
+    GetApplicationsRequest request =
+        Records.newRecord(GetApplicationsRequest.class);
+    request.setApplicationStates(applicationStates);
+    return request;
+  }
+
+  /**
+   * <p>
+   * The request from clients to get a report of Applications matching the
+   * giving and application types and application types in the cluster from the
+   * <code>ResourceManager</code>.
+   * </p>
+   *
+   *
+   * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+   */
+  @Public
+  @Stable
+  public static GetApplicationsRequest newInstance(
+      Set<String> applicationTypes,
+      EnumSet<YarnApplicationState> applicationStates) {
     GetApplicationsRequest request =
         Records.newRecord(GetApplicationsRequest.class);
     request.setApplicationTypes(applicationTypes);
+    request.setApplicationStates(applicationStates);
     return request;
   }
 
@@ -75,4 +129,25 @@ public abstract class GetApplicationsReq
   @Unstable
   public abstract void
       setApplicationTypes(Set<String> applicationTypes);
+
+  /**
+   * Get the application states to filter applications on
+   *
+   * @return Set of Application states to filter on
+   */
+  @Public
+  @Stable
+  public abstract EnumSet<YarnApplicationState> getApplicationStates();
+
+  /**
+   * Set the application states to filter applications on
+   *
+   * @param applicationStates
+   * A Set of Application states to filter on.
+   * If not defined, match all running applications
+   */
+  @Private
+  @Unstable
+  public abstract void
+      setApplicationStates(EnumSet<YarnApplicationState> applicationStates);
 }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java Tue Aug 27 17:05:13 2013
@@ -33,7 +33,7 @@ public abstract class GetDelegationToken
 
   @Public
   @Stable
-  public GetDelegationTokenRequest newInstance(String renewer) {
+  public static GetDelegationTokenRequest newInstance(String renewer) {
     GetDelegationTokenRequest request =
         Records.newRecord(GetDelegationTokenRequest.class);
     request.setRenewer(renewer);

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Tue Aug 27 17:05:13 2013
@@ -132,6 +132,15 @@ public class YarnConfiguration extends C
     RM_PREFIX + "scheduler.client.thread-count";
   public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50;
 
+  /** If the port should be included or not in the node name. The node name
+   * is used by the scheduler for resource requests allocation location 
+   * matching. Typically this is just the hostname, using the port is needed
+   * when using minicluster and specific NM are required.*/
+  public static final String RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME =
+      YARN_PREFIX + "scheduler.include-port-in-node-name";
+  public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME = 
+      false;
+
   /**
    * Enable periodic monitor threads.
    * @see #RM_SCHEDULER_MONITOR_POLICIES
@@ -205,6 +214,14 @@ public class YarnConfiguration extends C
   public static final String RM_KEYTAB = 
     RM_PREFIX + "keytab";
 
+  /**The kerberos principal to be used for spnego filter for RM.*/
+  public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY =
+      RM_PREFIX + "webapp.spnego-principal";
+  
+  /**The kerberos keytab to be used for spnego filter for RM.*/
+  public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
+      RM_PREFIX + "webapp.spnego-keytab-file";
+
   /** How long to wait until a container is considered dead.*/
   public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 
     RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
@@ -599,7 +616,15 @@ public class YarnConfiguration extends C
 
   public static final String NM_USER_HOME_DIR =
       NM_PREFIX + "user-home-dir";
-
+  
+  /**The kerberos principal to be used for spnego filter for NM.*/
+  public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY =
+      NM_PREFIX + "webapp.spnego-principal";
+  
+  /**The kerberos keytab to be used for spnego filter for NM.*/
+  public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
+      NM_PREFIX + "webapp.spnego-keytab-file";
+  
   public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
 
   ////////////////////////////////

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Tue Aug 27 17:05:13 2013
@@ -122,6 +122,7 @@ message GetClusterMetricsResponseProto {
 
 message GetApplicationsRequestProto {
   repeated string application_types = 1;
+  repeated YarnApplicationStateProto application_states = 2;
 }
 
 message GetApplicationsResponseProto {

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java Tue Aug 27 17:05:13 2013
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.client.api;
 
 import java.io.IOException;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Set;
 
@@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
 import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -171,13 +173,13 @@ public abstract class YarnClient extends
    * <p>
    * Get a report (ApplicationReport) of all Applications in the cluster.
    * </p>
-   * 
+   *
    * <p>
    * If the user does not have <code>VIEW_APP</code> access for an application
    * then the corresponding report will be filtered as described in
    * {@link #getApplicationReport(ApplicationId)}.
    * </p>
-   * 
+   *
    * @return a list of reports of all running applications
    * @throws YarnException
    * @throws IOException
@@ -207,6 +209,50 @@ public abstract class YarnClient extends
 
   /**
    * <p>
+   * Get a report (ApplicationReport) of Applications matching the given
+   * application states in the cluster.
+   * </p>
+   *
+   * <p>
+   * If the user does not have <code>VIEW_APP</code> access for an application
+   * then the corresponding report will be filtered as described in
+   * {@link #getApplicationReport(ApplicationId)}.
+   * </p>
+   *
+   * @param applicationStates
+   * @return a list of reports of applications
+   * @throws YarnException
+   * @throws IOException
+   */
+  public abstract List<ApplicationReport>
+      getApplications(EnumSet<YarnApplicationState> applicationStates)
+          throws YarnException, IOException;
+
+  /**
+   * <p>
+   * Get a report (ApplicationReport) of Applications matching the given
+   * application types and application states in the cluster.
+   * </p>
+   *
+   * <p>
+   * If the user does not have <code>VIEW_APP</code> access for an application
+   * then the corresponding report will be filtered as described in
+   * {@link #getApplicationReport(ApplicationId)}.
+   * </p>
+   *
+   * @param applicationTypes
+   * @param applicationStates
+   * @return a list of reports of applications
+   * @throws YarnException
+   * @throws IOException
+   */
+  public abstract List<ApplicationReport> getApplications(
+      Set<String> applicationTypes,
+      EnumSet<YarnApplicationState> applicationStates) throws YarnException,
+      IOException;
+
+  /**
+   * <p>
    * Get metrics ({@link YarnClusterMetrics}) about the cluster.
    * </p>
    * 

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Tue Aug 27 17:05:13 2013
@@ -211,15 +211,29 @@ public class YarnClientImpl extends Yarn
   @Override
   public List<ApplicationReport> getApplications() throws YarnException,
       IOException {
-    return getApplications(null);
+    return getApplications(null, null);
+  }
+
+  @Override
+  public List<ApplicationReport> getApplications(Set<String> applicationTypes)
+      throws YarnException,
+      IOException {
+    return getApplications(applicationTypes, null);
   }
 
   @Override
   public List<ApplicationReport> getApplications(
-      Set<String> applicationTypes) throws YarnException, IOException {
+      EnumSet<YarnApplicationState> applicationStates)
+      throws YarnException, IOException {
+    return getApplications(null, applicationStates);
+  }
+
+  @Override
+  public List<ApplicationReport> getApplications(Set<String> applicationTypes,
+      EnumSet<YarnApplicationState> applicationStates) throws YarnException,
+      IOException {
     GetApplicationsRequest request =
-        applicationTypes == null ? GetApplicationsRequest.newInstance()
-            : GetApplicationsRequest.newInstance(applicationTypes);
+        GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
     GetApplicationsResponse response = rmClient.getApplications(request);
     return response.getApplicationList();
   }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Tue Aug 27 17:05:13 2013
@@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.text.DecimalFormat;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -49,6 +50,10 @@ public class ApplicationCLI extends Yarn
     System.getProperty("line.separator");
 
   private static final String APP_TYPE_CMD = "appTypes";
+  private static final String APP_STATE_CMD ="appStates";
+  private static final String ALLSTATES_OPTION = "ALL";
+
+  private boolean allAppStates;
 
   public static void main(String[] args) throws Exception {
     ApplicationCLI cli = new ApplicationCLI();
@@ -66,7 +71,8 @@ public class ApplicationCLI extends Yarn
     opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
     opts.addOption(LIST_CMD, false, "List applications from the RM. " +
         "Supports optional use of --appTypes to filter applications " +
-        "based on application type.");
+        "based on application type, " +
+        "and --appStates to filter applications based on application state");
     opts.addOption(KILL_CMD, true, "Kills the application.");
     opts.addOption(HELP_CMD, false, "Displays help for all commands.");
     Option appTypeOpt = new Option(APP_TYPE_CMD, true,
@@ -75,6 +81,16 @@ public class ApplicationCLI extends Yarn
     appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
     appTypeOpt.setArgName("Comma-separated list of application types");
     opts.addOption(appTypeOpt);
+    Option appStateOpt =
+        new Option(
+            APP_STATE_CMD,
+            true,
+            "Works with --list to filter applications based on their state. "
+                + getAllValidApplicationStates());
+    appStateOpt.setValueSeparator(',');
+    appStateOpt.setArgs(Option.UNLIMITED_VALUES);
+    appStateOpt.setArgName("Comma-separated list of application states");
+    opts.addOption(appStateOpt);
     opts.getOption(KILL_CMD).setArgName("Application ID");
     opts.getOption(STATUS_CMD).setArgName("Application ID");
     CommandLine cliParser = new GnuParser().parse(opts, args);
@@ -87,18 +103,44 @@ public class ApplicationCLI extends Yarn
       }
       printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
     } else if (cliParser.hasOption(LIST_CMD)) {
+      allAppStates = false;
       Set<String> appTypes = new HashSet<String>();
       if(cliParser.hasOption(APP_TYPE_CMD)) {
         String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
         if (types != null) {
           for (String type : types) {
             if (!type.trim().isEmpty()) {
-              appTypes.add(type.trim());
+              appTypes.add(type.toUpperCase().trim());
             }
           }
         }
       }
-      listApplications(appTypes);
+
+      EnumSet<YarnApplicationState> appStates =
+          EnumSet.noneOf(YarnApplicationState.class);
+      if (cliParser.hasOption(APP_STATE_CMD)) {
+        String[] states = cliParser.getOptionValues(APP_STATE_CMD);
+        if (states != null) {
+          for (String state : states) {
+            if (!state.trim().isEmpty()) {
+              if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) {
+                allAppStates = true;
+                break;
+              }
+              try {
+                appStates.add(YarnApplicationState.valueOf(state.toUpperCase()
+                    .trim()));
+              } catch (IllegalArgumentException ex) {
+                sysout.println("The application state " + state
+                    + " is invalid.");
+                sysout.println(getAllValidApplicationStates());
+                return exitCode;
+              }
+            }
+          }
+        }
+      }
+      listApplications(appTypes, appStates);
     } else if (cliParser.hasOption(KILL_CMD)) {
       if (args.length != 2) {
         printUsage(opts);
@@ -127,19 +169,33 @@ public class ApplicationCLI extends Yarn
 
   /**
    * Lists the applications matching the given application Types
-   * present in the Resource Manager
+   * And application States present in the Resource Manager
    * 
    * @param appTypes
+   * @param appStates
    * @throws YarnException
    * @throws IOException
    */
-  private void listApplications(Set<String> appTypes)
-      throws YarnException, IOException {
+  private void listApplications(Set<String> appTypes,
+      EnumSet<YarnApplicationState> appStates) throws YarnException,
+      IOException {
     PrintWriter writer = new PrintWriter(sysout);
+    if (allAppStates) {
+      for(YarnApplicationState appState : YarnApplicationState.values()) {
+        appStates.add(appState);
+      }
+    } else {
+      if (appStates.isEmpty()) {
+        appStates.add(YarnApplicationState.RUNNING);
+      }
+    }
+
     List<ApplicationReport> appsReport =
-        client.getApplications(appTypes);
+        client.getApplications(appTypes, appStates);
 
-    writer.println("Total Applications:" + appsReport.size());
+    writer
+        .println("Total number of applications (application-types: " + appTypes
+            + " and states: " + appStates + ")" + ":" + appsReport.size());
     writer.printf(APPLICATIONS_PATTERN, "Application-Id",
         "Application-Name","Application-Type", "User", "Queue", 
         "State", "Final-State","Progress", "Tracking-URL");
@@ -229,4 +285,16 @@ public class ApplicationCLI extends Yarn
     sysout.println(baos.toString("UTF-8"));
   }
 
+  private String getAllValidApplicationStates() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("The valid application state can be"
+        + " one of the following: ");
+    sb.append(ALLSTATES_OPTION + ",");
+    for (YarnApplicationState appState : YarnApplicationState
+        .values()) {
+      sb.append(appState+",");
+    }
+    String output = sb.toString();
+    return output.substring(0, output.length()-1);
+  }
 }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java Tue Aug 27 17:05:13 2013
@@ -21,11 +21,14 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang.time.DateFormatUtils;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -40,9 +43,12 @@ import org.apache.hadoop.yarn.util.Conve
 @Private
 @Unstable
 public class NodeCLI extends YarnCLI {
-  private static final String NODES_PATTERN = "%16s\t%10s\t%17s\t%18s" +
+  private static final String NODES_PATTERN = "%16s\t%15s\t%17s\t%18s" +
     System.getProperty("line.separator");
 
+  private static final String NODE_STATE_CMD = "states";
+  private static final String NODE_ALL = "all";
+
   public static void main(String[] args) throws Exception {
     NodeCLI cli = new NodeCLI();
     cli.setSysOutPrintStream(System.out);
@@ -57,7 +63,18 @@ public class NodeCLI extends YarnCLI {
 
     Options opts = new Options();
     opts.addOption(STATUS_CMD, true, "Prints the status report of the node.");
-    opts.addOption(LIST_CMD, false, "Lists all the nodes in the RUNNING state.");
+    opts.addOption(LIST_CMD, false, "List all running nodes. " +
+        "Supports optional use of --states to filter nodes " +
+        "based on node state, all --all to list all nodes.");
+    Option nodeStateOpt = new Option(NODE_STATE_CMD, true,
+        "Works with -list to filter nodes based on their states.");
+    nodeStateOpt.setValueSeparator(',');
+    nodeStateOpt.setArgs(Option.UNLIMITED_VALUES);
+    nodeStateOpt.setArgName("Comma-separated list of node states");
+    opts.addOption(nodeStateOpt);
+    Option allOpt = new Option(NODE_ALL, false,
+        "Works with -list to list all nodes.");
+    opts.addOption(allOpt);
     CommandLine cliParser = new GnuParser().parse(opts, args);
 
     int exitCode = -1;
@@ -68,7 +85,24 @@ public class NodeCLI extends YarnCLI {
       }
       printNodeStatus(cliParser.getOptionValue("status"));
     } else if (cliParser.hasOption("list")) {
-      listClusterNodes();
+      Set<NodeState> nodeStates = new HashSet<NodeState>();
+      if (cliParser.hasOption(NODE_ALL)) {
+        for (NodeState state : NodeState.values()) {
+          nodeStates.add(state);
+        }
+      } else if (cliParser.hasOption(NODE_STATE_CMD)) {
+        String[] types = cliParser.getOptionValues(NODE_STATE_CMD);
+        if (types != null) {
+          for (String type : types) {
+            if (!type.trim().isEmpty()) {
+              nodeStates.add(NodeState.valueOf(type.trim().toUpperCase()));
+            }
+          }
+        }
+      } else {
+        nodeStates.add(NodeState.RUNNING);
+      }
+      listClusterNodes(nodeStates);
     } else {
       syserr.println("Invalid Command Usage : ");
       printUsage(opts);
@@ -86,14 +120,17 @@ public class NodeCLI extends YarnCLI {
   }
 
   /**
-   * Lists all the nodes present in the cluster
+   * Lists the nodes matching the given node states
    * 
+   * @param nodeStates
    * @throws YarnException
    * @throws IOException
    */
-  private void listClusterNodes() throws YarnException, IOException {
+  private void listClusterNodes(Set<NodeState> nodeStates) 
+            throws YarnException, IOException {
     PrintWriter writer = new PrintWriter(sysout);
-    List<NodeReport> nodesReport = client.getNodeReports(NodeState.RUNNING);
+    List<NodeReport> nodesReport = client.getNodeReports(
+                                       nodeStates.toArray(new NodeState[0]));
     writer.println("Total Nodes:" + nodesReport.size());
     writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
         "Running-Containers");

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Tue Aug 27 17:05:13 2013
@@ -27,6 +27,7 @@ import static org.mockito.Mockito.when;
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -55,10 +56,12 @@ import org.apache.hadoop.yarn.client.api
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
@@ -163,11 +166,15 @@ public class TestYarnClient {
 
     List<ApplicationReport> expectedReports = ((MockYarnClient)client).getReports();
 
+    List<ApplicationReport>  reports = client.getApplications();
+    Assert.assertEquals(reports, expectedReports);
+
     Set<String> appTypes = new HashSet<String>();
     appTypes.add("YARN");
     appTypes.add("NON-YARN");
 
-    List<ApplicationReport> reports = client.getApplications(appTypes);
+    reports =
+        client.getApplications(appTypes, null);
     Assert.assertEquals(reports.size(), 2);
     Assert
         .assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports
@@ -178,8 +185,28 @@ public class TestYarnClient {
       Assert.assertTrue(expectedReports.contains(report));
     }
 
-    reports = client.getApplications();
-    Assert.assertEquals(reports, expectedReports);
+    EnumSet<YarnApplicationState> appStates =
+        EnumSet.noneOf(YarnApplicationState.class);
+    appStates.add(YarnApplicationState.FINISHED);
+    appStates.add(YarnApplicationState.FAILED);
+    reports = client.getApplications(null, appStates);
+    Assert.assertEquals(reports.size(), 2);
+    Assert
+    .assertTrue((reports.get(0).getApplicationType().equals("NON-YARN") && reports
+        .get(1).getApplicationType().equals("NON-MAPREDUCE"))
+        || (reports.get(1).getApplicationType().equals("NON-YARN") && reports
+            .get(0).getApplicationType().equals("NON-MAPREDUCE")));
+    for (ApplicationReport report : reports) {
+      Assert.assertTrue(expectedReports.contains(report));
+    }
+
+    reports = client.getApplications(appTypes, appStates);
+    Assert.assertEquals(reports.size(), 1);
+    Assert
+    .assertTrue((reports.get(0).getApplicationType().equals("NON-YARN")));
+    for (ApplicationReport report : reports) {
+      Assert.assertTrue(expectedReports.contains(report));
+    }
 
     client.stop();
   }
@@ -187,6 +214,8 @@ public class TestYarnClient {
   private static class MockYarnClient extends YarnClientImpl {
     private ApplicationReport mockReport;
     private List<ApplicationReport> reports;
+    GetApplicationsResponse mockAppResponse =
+        mock(GetApplicationsResponse.class);
 
     public MockYarnClient() {
       super();
@@ -202,6 +231,8 @@ public class TestYarnClient {
       try{
         when(rmClient.getApplicationReport(any(
             GetApplicationReportRequest.class))).thenReturn(mockResponse);
+        when(rmClient.getApplications(any(GetApplicationsRequest.class)))
+            .thenReturn(mockAppResponse);
       } catch (YarnException e) {
         Assert.fail("Exception is not expected.");
       } catch (IOException e) {
@@ -212,16 +243,11 @@ public class TestYarnClient {
 
     @Override
     public List<ApplicationReport> getApplications(
-        Set<String> applicationTypes) throws YarnException, IOException {
-      GetApplicationsRequest request =
-          applicationTypes == null ? GetApplicationsRequest.newInstance()
-              : GetApplicationsRequest.newInstance(applicationTypes);
-      when(rmClient.getApplications(request))
-          .thenReturn(
-              getApplicationReports(reports,
-                  request));
-      GetApplicationsResponse response = rmClient.getApplications(request);
-      return response.getApplicationList();
+        Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates)
+        throws YarnException, IOException {
+      when(mockAppResponse.getApplicationList()).thenReturn(
+          getApplicationReports(reports, applicationTypes, applicationStates));
+      return super.getApplications(applicationTypes, applicationStates);
     }
 
     @Override
@@ -243,7 +269,7 @@ public class TestYarnClient {
       ApplicationReport newApplicationReport = ApplicationReport.newInstance(
           applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
           "user", "queue", "appname", "host", 124, null,
-          YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+          YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
           FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
       List<ApplicationReport> applicationReports =
           new ArrayList<ApplicationReport>();
@@ -262,31 +288,44 @@ public class TestYarnClient {
       ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
           applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
           "user3", "queue3", "appname3", "host3", 126, null,
-          YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3,
-          FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", 
+          YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
+          FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
         null);
       applicationReports.add(newApplicationReport3);
+
+      ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
+      ApplicationReport newApplicationReport4 =
+          ApplicationReport.newInstance(
+              applicationId4,
+              ApplicationAttemptId.newInstance(applicationId4, 4),
+              "user4", "queue4", "appname4", "host4", 127, null,
+              YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
+              FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
+              "NON-MAPREDUCE", null);
+      applicationReports.add(newApplicationReport4);
       return applicationReports;
     }
 
-    private GetApplicationsResponse getApplicationReports(
+    private List<ApplicationReport> getApplicationReports(
         List<ApplicationReport> applicationReports,
-        GetApplicationsRequest request) {
+        Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates) {
 
       List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
-      Set<String> appTypes = request.getApplicationTypes();
-      boolean bypassFilter = appTypes.isEmpty();
-
       for (ApplicationReport appReport : applicationReports) {
-        if (!(bypassFilter || appTypes.contains(
-            appReport.getApplicationType()))) {
-          continue;
+        if (applicationTypes != null && !applicationTypes.isEmpty()) {
+          if (!applicationTypes.contains(appReport.getApplicationType())) {
+            continue;
+          }
+        }
+
+        if (applicationStates != null && !applicationStates.isEmpty()) {
+          if (!applicationStates.contains(appReport.getYarnApplicationState())) {
+            continue;
+          }
         }
         appReports.add(appReport);
       }
-      GetApplicationsResponse response =
-          GetApplicationsResponse.newInstance(appReports);
-      return response;
+      return appReports;
     }
   }
 

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Tue Aug 27 17:05:13 2013
@@ -33,6 +33,7 @@ import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -134,7 +135,7 @@ public class TestYarnCLI {
     ApplicationReport newApplicationReport = ApplicationReport.newInstance(
         applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
         "user", "queue", "appname", "host", 124, null,
-        YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+        YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
         FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
     List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
     applicationReports.add(newApplicationReport);
@@ -152,23 +153,39 @@ public class TestYarnCLI {
     ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
         applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
         "user3", "queue3", "appname3", "host3", 126, null,
-        YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3,
+        YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
         FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", 
         null);
     applicationReports.add(newApplicationReport3);
 
-    Set<String> appType1 = new HashSet<String>();
-    appType1.add("YARN");
+    ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
+    ApplicationReport newApplicationReport4 = ApplicationReport.newInstance(
+        applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4),
+        "user4", "queue4", "appname4", "host4", 127, null,
+        YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
+        null);
+    applicationReports.add(newApplicationReport4);
 
-    when(client.getApplications(appType1)).thenReturn(
-        getApplicationReports(applicationReports, appType1));
-    int result = cli.run(new String[] { "-list", "-appTypes", "YARN" });
+    // Test command yarn application -list
+    // if the set appStates is empty, RUNNING state will be automatically added
+    // to the appStates list
+    // the output of yarn application -list should be the same as
+    // equals to yarn application -list --appStates RUNNING
+    Set<String> appType1 = new HashSet<String>();
+    EnumSet<YarnApplicationState> appState1 =
+        EnumSet.noneOf(YarnApplicationState.class);
+    appState1.add(YarnApplicationState.RUNNING);
+    when(client.getApplications(appType1, appState1)).thenReturn(
+        getApplicationReports(applicationReports, appType1, appState1, false));
+    int result = cli.run(new String[] { "-list" });
     assertEquals(0, result);
-    verify(client).getApplications(appType1);
+    verify(client).getApplications(appType1, appState1);
 
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintWriter pw = new PrintWriter(baos);
-    pw.println("Total Applications:1");
+    pw.println("Total number of applications (application-types: " + appType1
+        + " and states: " + appState1 + ")" + ":" + 2);
     pw.print("                Application-Id\t    Application-Name");
     pw.print("\t    Application-Type");
     pw.print("\t      User\t     Queue\t             State\t       ");
@@ -176,27 +193,41 @@ public class TestYarnCLI {
     pw.println("\t                       Tracking-URL");
     pw.print("         application_1234_0005\t             ");
     pw.print("appname\t                YARN\t      user\t     ");
-    pw.print("queue\t          FINISHED\t         ");
+    pw.print("queue\t           RUNNING\t         ");
     pw.print("SUCCEEDED\t         53.79%");
     pw.println("\t                                N/A");
+    pw.print("         application_1234_0007\t            ");
+    pw.print("appname3\t           MAPREDUCE\t     user3\t    ");
+    pw.print("queue3\t           RUNNING\t         ");
+    pw.print("SUCCEEDED\t         73.79%");
+    pw.println("\t                                N/A");
     pw.close();
     String appsReportStr = baos.toString("UTF-8");
     Assert.assertEquals(appsReportStr, sysOutStream.toString());
     verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt());
 
+    //Test command yarn application -list --appTypes apptype1,apptype2
+    //the output should be the same as
+    //yarn application -list --appTypes apptyp1, apptype2 --appStates RUNNING
     sysOutStream.reset();
     Set<String> appType2 = new HashSet<String>();
     appType2.add("YARN");
-    appType2.add("FOO-YARN");
-    when(client.getApplications(appType2)).thenReturn(
-        getApplicationReports(applicationReports, appType2));
-    cli.run(new String[] { "-list", "-appTypes", "YARN , ,, ,FOO-YARN",
-        ",,,,, YARN,," });
+    appType2.add("NON-YARN");
+
+    EnumSet<YarnApplicationState> appState2 =
+        EnumSet.noneOf(YarnApplicationState.class);
+    appState2.add(YarnApplicationState.RUNNING);
+    when(client.getApplications(appType2, appState2)).thenReturn(
+        getApplicationReports(applicationReports, appType2, appState2, false));
+    result =
+        cli.run(new String[] { "-list", "-appTypes", "YARN, ,,  NON-YARN",
+            "   ,, ,," });
     assertEquals(0, result);
-    verify(client).getApplications(appType2);
+    verify(client).getApplications(appType2, appState2);
     baos = new ByteArrayOutputStream();
     pw = new PrintWriter(baos);
-    pw.println("Total Applications:1");
+    pw.println("Total number of applications (application-types: " + appType2
+        + " and states: " + appState2 + ")" + ":" + 1);
     pw.print("                Application-Id\t    Application-Name");
     pw.print("\t    Application-Type");
     pw.print("\t      User\t     Queue\t             State\t       ");
@@ -204,7 +235,7 @@ public class TestYarnCLI {
     pw.println("\t                       Tracking-URL");
     pw.print("         application_1234_0005\t             ");
     pw.print("appname\t                YARN\t      user\t     ");
-    pw.print("queue\t          FINISHED\t         ");
+    pw.print("queue\t           RUNNING\t         ");
     pw.print("SUCCEEDED\t         53.79%");
     pw.println("\t                                N/A");
     pw.close();
@@ -212,50 +243,122 @@ public class TestYarnCLI {
     Assert.assertEquals(appsReportStr, sysOutStream.toString());
     verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt());
 
+    //Test command yarn application -list --appStates appState1,appState2
     sysOutStream.reset();
     Set<String> appType3 = new HashSet<String>();
-    appType3.add("YARN");
-    appType3.add("NON-YARN");
-    when(client.getApplications(appType3)).thenReturn(
-        getApplicationReports(applicationReports, appType3));
 
-    result = cli.run(new String[] { "-list", "-appTypes", "YARN,NON-YARN" });
+    EnumSet<YarnApplicationState> appState3 =
+        EnumSet.noneOf(YarnApplicationState.class);
+    appState3.add(YarnApplicationState.FINISHED);
+    appState3.add(YarnApplicationState.FAILED);
+
+    when(client.getApplications(appType3, appState3)).thenReturn(
+        getApplicationReports(applicationReports, appType3, appState3, false));
+    result =
+        cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , FAILED",
+            ",,FINISHED" });
     assertEquals(0, result);
-    verify(client).getApplications(appType3);
+    verify(client).getApplications(appType3, appState3);
     baos = new ByteArrayOutputStream();
     pw = new PrintWriter(baos);
-    pw.println("Total Applications:2");
+    pw.println("Total number of applications (application-types: " + appType3
+        + " and states: " + appState3 + ")" + ":" + 2);
     pw.print("                Application-Id\t    Application-Name");
     pw.print("\t    Application-Type");
     pw.print("\t      User\t     Queue\t             State\t       ");
     pw.print("Final-State\t       Progress");
     pw.println("\t                       Tracking-URL");
-    pw.print("         application_1234_0005\t             ");
-    pw.print("appname\t                YARN\t      user\t     ");
-    pw.print("queue\t          FINISHED\t         ");
-    pw.print("SUCCEEDED\t         53.79%");
-    pw.println("\t                                N/A");
     pw.print("         application_1234_0006\t            ");
     pw.print("appname2\t            NON-YARN\t     user2\t    ");
     pw.print("queue2\t          FINISHED\t         ");
     pw.print("SUCCEEDED\t         63.79%");
     pw.println("\t                                N/A");
+    pw.print("         application_1234_0008\t            ");
+    pw.print("appname4\t       NON-MAPREDUCE\t     user4\t    ");
+    pw.print("queue4\t            FAILED\t         ");
+    pw.print("SUCCEEDED\t         83.79%");
+    pw.println("\t                                N/A");
     pw.close();
     appsReportStr = baos.toString("UTF-8");
     Assert.assertEquals(appsReportStr, sysOutStream.toString());
     verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
 
+    // Test command yarn application -list --appTypes apptype1,apptype2
+    // --appStates appstate1,appstate2
     sysOutStream.reset();
     Set<String> appType4 = new HashSet<String>();
-    when(client.getApplications(appType4)).thenReturn(
-        getApplicationReports(applicationReports, appType4));
-    result = cli.run(new String[] { "-list" });
+    appType4.add("YARN");
+    appType4.add("NON-YARN");
+
+    EnumSet<YarnApplicationState> appState4 =
+        EnumSet.noneOf(YarnApplicationState.class);
+    appState4.add(YarnApplicationState.FINISHED);
+    appState4.add(YarnApplicationState.FAILED);
+
+    when(client.getApplications(appType4, appState4)).thenReturn(
+        getApplicationReports(applicationReports, appType4, appState4, false));
+    result =
+        cli.run(new String[] { "-list", "--appTypes", "YARN,NON-YARN",
+        "--appStates", "FINISHED ,, , FAILED" });
     assertEquals(0, result);
-    verify(client).getApplications(appType4);
+    verify(client).getApplications(appType2, appState2);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total number of applications (application-types: " + appType4
+        + " and states: " + appState4 + ")" + ":" + 1);
+    pw.print("                Application-Id\t    Application-Name");
+    pw.print("\t    Application-Type");
+    pw.print("\t      User\t     Queue\t             State\t       ");
+    pw.print("Final-State\t       Progress");
+    pw.println("\t                       Tracking-URL");
+    pw.print("         application_1234_0006\t            ");
+    pw.print("appname2\t            NON-YARN\t     user2\t    ");
+    pw.print("queue2\t          FINISHED\t         ");
+    pw.print("SUCCEEDED\t         63.79%");
+    pw.println("\t                                N/A");
+    pw.close();
+    appsReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(appsReportStr, sysOutStream.toString());
+    verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
 
+    //Test command yarn application -list --appStates with invalid appStates
+    sysOutStream.reset();
+    result =
+        cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , INVALID" });
+    assertEquals(-1, result);
     baos = new ByteArrayOutputStream();
     pw = new PrintWriter(baos);
-    pw.println("Total Applications:3");
+    pw.println("The application state  INVALID is invalid.");
+    pw.print("The valid application state can be one of the following: ");
+    StringBuilder sb = new StringBuilder();
+    sb.append("ALL,");
+    for(YarnApplicationState state : YarnApplicationState.values()) {
+      sb.append(state+",");
+    }
+    String output = sb.toString();
+    pw.println(output.substring(0, output.length()-1));
+    pw.close();
+    appsReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(appsReportStr, sysOutStream.toString());
+    verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+
+    //Test command yarn application -list --appStates all
+    sysOutStream.reset();
+    Set<String> appType5 = new HashSet<String>();
+
+    EnumSet<YarnApplicationState> appState5 =
+        EnumSet.noneOf(YarnApplicationState.class);
+    appState5.add(YarnApplicationState.FINISHED);
+    when(client.getApplications(appType5, appState5)).thenReturn(
+        getApplicationReports(applicationReports, appType5, appState5, true));
+    result =
+        cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , ALL" });
+    assertEquals(0, result);
+    verify(client).getApplications(appType5, appState5);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total number of applications (application-types: " + appType5
+        + " and states: " + appState5 + ")" + ":" + 4);
     pw.print("                Application-Id\t    Application-Name");
     pw.print("\t    Application-Type");
     pw.print("\t      User\t     Queue\t             State\t       ");
@@ -263,7 +366,7 @@ public class TestYarnCLI {
     pw.println("\t                       Tracking-URL");
     pw.print("         application_1234_0005\t             ");
     pw.print("appname\t                YARN\t      user\t     ");
-    pw.print("queue\t          FINISHED\t         ");
+    pw.print("queue\t           RUNNING\t         ");
     pw.print("SUCCEEDED\t         53.79%");
     pw.println("\t                                N/A");
     pw.print("         application_1234_0006\t            ");
@@ -273,27 +376,80 @@ public class TestYarnCLI {
     pw.println("\t                                N/A");
     pw.print("         application_1234_0007\t            ");
     pw.print("appname3\t           MAPREDUCE\t     user3\t    ");
-    pw.print("queue3\t          FINISHED\t         ");
+    pw.print("queue3\t           RUNNING\t         ");
     pw.print("SUCCEEDED\t         73.79%");
     pw.println("\t                                N/A");
+    pw.print("         application_1234_0008\t            ");
+    pw.print("appname4\t       NON-MAPREDUCE\t     user4\t    ");
+    pw.print("queue4\t            FAILED\t         ");
+    pw.print("SUCCEEDED\t         83.79%");
+    pw.println("\t                                N/A");
     pw.close();
     appsReportStr = baos.toString("UTF-8");
     Assert.assertEquals(appsReportStr, sysOutStream.toString());
-    verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+    verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
+
+    // Test command yarn application user case insensitive
+    sysOutStream.reset();
+    Set<String> appType6 = new HashSet<String>();
+    appType6.add("YARN");
+    appType6.add("NON-YARN");
+
+    EnumSet<YarnApplicationState> appState6 =
+        EnumSet.noneOf(YarnApplicationState.class);
+    appState6.add(YarnApplicationState.FINISHED);
+    when(client.getApplications(appType6, appState6)).thenReturn(
+        getApplicationReports(applicationReports, appType6, appState6, false));
+    result =
+        cli.run(new String[] { "-list", "-appTypes", "YARN, ,,  NON-YARN",
+            "--appStates", "finished" });
+    assertEquals(0, result);
+    verify(client).getApplications(appType6, appState6);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total number of applications (application-types: " + appType6
+        + " and states: " + appState6 + ")" + ":" + 1);
+    pw.print("                Application-Id\t    Application-Name");
+    pw.print("\t    Application-Type");
+    pw.print("\t      User\t     Queue\t             State\t       ");
+    pw.print("Final-State\t       Progress");
+    pw.println("\t                       Tracking-URL");
+    pw.print("         application_1234_0006\t            ");
+    pw.print("appname2\t            NON-YARN\t     user2\t    ");
+    pw.print("queue2\t          FINISHED\t         ");
+    pw.print("SUCCEEDED\t         63.79%");
+    pw.println("\t                                N/A");
+    pw.close();
+    appsReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(appsReportStr, sysOutStream.toString());
+    verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
   }
 
   private List<ApplicationReport> getApplicationReports(
       List<ApplicationReport> applicationReports,
-      Set<String> appTypes) {
+      Set<String> appTypes, EnumSet<YarnApplicationState> appStates,
+      boolean allStates) {
 
     List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
-    boolean bypassFilter = appTypes.isEmpty();
 
+    if (allStates) {
+      for(YarnApplicationState state : YarnApplicationState.values()) {
+        appStates.add(state);
+      }
+    }
     for (ApplicationReport appReport : applicationReports) {
-      if (!(bypassFilter || appTypes.contains(
-          appReport.getApplicationType()))) {
-        continue;
+      if (appTypes != null && !appTypes.isEmpty()) {
+        if (!appTypes.contains(appReport.getApplicationType())) {
+          continue;
+        }
       }
+
+      if (appStates != null && !appStates.isEmpty()) {
+        if (!appStates.contains(appReport.getYarnApplicationState())) {
+          continue;
+        }
+      }
+
       appReports.add(appReport);
     }
     return appReports;
@@ -363,36 +519,239 @@ public class TestYarnCLI {
 
   @Test
   public void testListClusterNodes() throws Exception {
+    List<NodeReport> nodeReports = new ArrayList<NodeReport>();
+    nodeReports.addAll(getNodeReports(1, NodeState.NEW));
+    nodeReports.addAll(getNodeReports(2, NodeState.RUNNING));
+    nodeReports.addAll(getNodeReports(1, NodeState.UNHEALTHY));
+    nodeReports.addAll(getNodeReports(1, NodeState.DECOMMISSIONED));
+    nodeReports.addAll(getNodeReports(1, NodeState.REBOOTED));
+    nodeReports.addAll(getNodeReports(1, NodeState.LOST));
+
     NodeCLI cli = new NodeCLI();
-    when(client.getNodeReports(NodeState.RUNNING)).thenReturn(
-        getNodeReports(3));
     cli.setClient(client);
     cli.setSysOutPrintStream(sysOut);
-    int result = cli.run(new String[] { "-list" });
+
+    Set<NodeState> nodeStates = new HashSet<NodeState>();
+    nodeStates.add(NodeState.NEW);
+    NodeState[] states = nodeStates.toArray(new NodeState[0]);
+    when(client.getNodeReports(states))
+        .thenReturn(getNodeReports(nodeReports, nodeStates));
+    int result = cli.run(new String[] { "-list", "--states", "NEW" });
     assertEquals(0, result);
-    verify(client).getNodeReports(NodeState.RUNNING);
+    verify(client).getNodeReports(states);
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintWriter pw = new PrintWriter(baos);
-    pw.println("Total Nodes:3");
-    pw.print("         Node-Id\tNode-State\tNode-Http-Address\t");
+    pw.println("Total Nodes:1");
+    pw.print("         Node-Id\t     Node-State\tNode-Http-Address\t");
     pw.println("Running-Containers");
-    pw.print("         host0:0\t   RUNNING\t       host1:8888");
-    pw.println("\t                 0");
-    pw.print("         host1:0\t   RUNNING\t       host1:8888");
-    pw.println("\t                 0");
-    pw.print("         host2:0\t   RUNNING\t       host1:8888");
+    pw.print("         host0:0\t            NEW\t       host1:8888");
     pw.println("\t                 0");
     pw.close();
     String nodesReportStr = baos.toString("UTF-8");
     Assert.assertEquals(nodesReportStr, sysOutStream.toString());
     verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt());
+
+    sysOutStream.reset();
+    nodeStates.clear();
+    nodeStates.add(NodeState.RUNNING);
+    states = nodeStates.toArray(new NodeState[0]);
+    when(client.getNodeReports(states))
+        .thenReturn(getNodeReports(nodeReports, nodeStates));
+    result = cli.run(new String[] { "-list", "--states", "RUNNING" });
+    assertEquals(0, result);
+    verify(client).getNodeReports(states);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total Nodes:2");
+    pw.print("         Node-Id\t     Node-State\tNode-Http-Address\t");
+    pw.println("Running-Containers");
+    pw.print("         host0:0\t        RUNNING\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host1:0\t        RUNNING\t       host1:8888");
+    pw.println("\t                 0");
+    pw.close();
+    nodesReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+    verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt());
+
+    sysOutStream.reset();
+    result = cli.run(new String[] { "-list" });
+    assertEquals(0, result);
+    Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+    verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
+
+    sysOutStream.reset();
+    nodeStates.clear();
+    nodeStates.add(NodeState.UNHEALTHY);
+    states = nodeStates.toArray(new NodeState[0]);
+    when(client.getNodeReports(states))
+        .thenReturn(getNodeReports(nodeReports, nodeStates));
+    result = cli.run(new String[] { "-list", "--states", "UNHEALTHY" });
+    assertEquals(0, result);
+    verify(client).getNodeReports(states);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total Nodes:1");
+    pw.print("         Node-Id\t     Node-State\tNode-Http-Address\t");
+    pw.println("Running-Containers");
+    pw.print("         host0:0\t      UNHEALTHY\t       host1:8888");
+    pw.println("\t                 0");
+    pw.close();
+    nodesReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+    verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+
+    sysOutStream.reset();
+    nodeStates.clear();
+    nodeStates.add(NodeState.DECOMMISSIONED);
+    states = nodeStates.toArray(new NodeState[0]);
+    when(client.getNodeReports(states))
+        .thenReturn(getNodeReports(nodeReports, nodeStates));
+    result = cli.run(new String[] { "-list", "--states", "DECOMMISSIONED" });
+    assertEquals(0, result);
+    verify(client).getNodeReports(states);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total Nodes:1");
+    pw.print("         Node-Id\t     Node-State\tNode-Http-Address\t");
+    pw.println("Running-Containers");
+    pw.print("         host0:0\t DECOMMISSIONED\t       host1:8888");
+    pw.println("\t                 0");
+    pw.close();
+    nodesReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+    verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
+
+    sysOutStream.reset();
+    nodeStates.clear();
+    nodeStates.add(NodeState.REBOOTED);
+    states = nodeStates.toArray(new NodeState[0]);
+    when(client.getNodeReports(states))
+        .thenReturn(getNodeReports(nodeReports, nodeStates));
+    result = cli.run(new String[] { "-list", "--states", "REBOOTED" });
+    assertEquals(0, result);
+    verify(client).getNodeReports(states);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total Nodes:1");
+    pw.print("         Node-Id\t     Node-State\tNode-Http-Address\t");
+    pw.println("Running-Containers");
+    pw.print("         host0:0\t       REBOOTED\t       host1:8888");
+    pw.println("\t                 0");
+    pw.close();
+    nodesReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+    verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
+
+    sysOutStream.reset();
+    nodeStates.clear();
+    nodeStates.add(NodeState.LOST);
+    states = nodeStates.toArray(new NodeState[0]);
+    when(client.getNodeReports(states))
+        .thenReturn(getNodeReports(nodeReports, nodeStates));
+    result = cli.run(new String[] { "-list", "--states", "LOST" });
+    assertEquals(0, result);
+    verify(client).getNodeReports(states);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total Nodes:1");
+    pw.print("         Node-Id\t     Node-State\tNode-Http-Address\t");
+    pw.println("Running-Containers");
+    pw.print("         host0:0\t           LOST\t       host1:8888");
+    pw.println("\t                 0");
+    pw.close();
+    nodesReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+    verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
+
+    sysOutStream.reset();
+    nodeStates.clear();
+    nodeStates.add(NodeState.NEW);
+    nodeStates.add(NodeState.RUNNING);
+    nodeStates.add(NodeState.LOST);
+    nodeStates.add(NodeState.REBOOTED);
+    states = nodeStates.toArray(new NodeState[0]);
+    when(client.getNodeReports(states))
+        .thenReturn(getNodeReports(nodeReports, nodeStates));
+    result = cli.run(new String[] { "-list", "--states", 
+                                        "NEW,RUNNING,LOST,REBOOTED" });
+    assertEquals(0, result);
+    verify(client).getNodeReports(states);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total Nodes:5");
+    pw.print("         Node-Id\t     Node-State\tNode-Http-Address\t");
+    pw.println("Running-Containers");
+    pw.print("         host0:0\t            NEW\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host0:0\t        RUNNING\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host1:0\t        RUNNING\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host0:0\t       REBOOTED\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host0:0\t           LOST\t       host1:8888");
+    pw.println("\t                 0");
+    pw.close();
+    nodesReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+    verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
+
+    sysOutStream.reset();
+    nodeStates.clear();
+    for (NodeState s : NodeState.values()) {
+      nodeStates.add(s);
+    }
+    states = nodeStates.toArray(new NodeState[0]);
+    when(client.getNodeReports(states))
+        .thenReturn(getNodeReports(nodeReports, nodeStates));
+    result = cli.run(new String[] { "-list", "--all" });
+    assertEquals(0, result);
+    verify(client).getNodeReports(states);
+    baos = new ByteArrayOutputStream();
+    pw = new PrintWriter(baos);
+    pw.println("Total Nodes:7");
+    pw.print("         Node-Id\t     Node-State\tNode-Http-Address\t");
+    pw.println("Running-Containers");
+    pw.print("         host0:0\t            NEW\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host0:0\t        RUNNING\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host1:0\t        RUNNING\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host0:0\t      UNHEALTHY\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host0:0\t DECOMMISSIONED\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host0:0\t       REBOOTED\t       host1:8888");
+    pw.println("\t                 0");
+    pw.print("         host0:0\t           LOST\t       host1:8888");
+    pw.println("\t                 0");
+    pw.close();
+    nodesReportStr = baos.toString("UTF-8");
+    Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+    verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
+  }
+
+  private List<NodeReport> getNodeReports(
+      List<NodeReport> nodeReports,
+      Set<NodeState> nodeStates) {
+    List<NodeReport> reports = new ArrayList<NodeReport>();
+
+    for (NodeReport nodeReport : nodeReports) {
+      if (nodeStates.contains(nodeReport.getNodeState())) {
+        reports.add(nodeReport);
+      }
+    }
+    return reports;
   }
 
   @Test
   public void testNodeStatus() throws Exception {
     NodeId nodeId = NodeId.newInstance("host0", 0);
     NodeCLI cli = new NodeCLI();
-    when(client.getNodeReports()).thenReturn(getNodeReports(3));
+    when(client.getNodeReports()).thenReturn(
+                    getNodeReports(3, NodeState.RUNNING));
     cli.setClient(client);
     cli.setSysOutPrintStream(sysOut);
     cli.setSysErrPrintStream(sysErr);
@@ -424,7 +783,8 @@ public class TestYarnCLI {
   public void testAbsentNodeStatus() throws Exception {
     NodeId nodeId = NodeId.newInstance("Absenthost0", 0);
     NodeCLI cli = new NodeCLI();
-    when(client.getNodeReports()).thenReturn(getNodeReports(0));
+    when(client.getNodeReports()).thenReturn(
+                getNodeReports(0, NodeState.RUNNING));
     cli.setClient(client);
     cli.setSysOutPrintStream(sysOut);
     cli.setSysErrPrintStream(sysErr);
@@ -452,12 +812,12 @@ public class TestYarnCLI {
     verify(sysErr).println("Invalid Command Usage : ");
   }
 
-  private List<NodeReport> getNodeReports(int noOfNodes) {
+  private List<NodeReport> getNodeReports(int noOfNodes, NodeState state) {
     List<NodeReport> nodeReports = new ArrayList<NodeReport>();
 
     for (int i = 0; i < noOfNodes; i++) {
       NodeReport nodeReport = NodeReport.newInstance(NodeId
-        .newInstance("host" + i, 0), NodeState.RUNNING, "host" + 1 + ":8888",
+        .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
           "rack1", Records.newRecord(Resource.class), Records
               .newRecord(Resource.class), 0, "", 0);
       nodeReports.add(nodeReport);

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java Tue Aug 27 17:05:13 2013
@@ -18,13 +18,18 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
+import java.util.EnumSet;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder;
 
@@ -38,6 +43,7 @@ public class GetApplicationsRequestPBImp
   boolean viaProto = false;
 
   Set<String> applicationTypes = null;
+  EnumSet<YarnApplicationState> applicationStates = null;
 
   public GetApplicationsRequestPBImpl() {
     builder = GetApplicationsRequestProto.newBuilder();
@@ -67,6 +73,40 @@ public class GetApplicationsRequestPBImp
     if (this.applicationTypes != null) {
       addLocalApplicationTypesToProto();
     }
+    if (this.applicationStates != null) {
+      maybeInitBuilder();
+      builder.clearApplicationStates();
+      Iterable<YarnApplicationStateProto> iterable =
+          new Iterable<YarnApplicationStateProto>() {
+
+            @Override
+            public Iterator<YarnApplicationStateProto> iterator() {
+              return new Iterator<YarnApplicationStateProto>() {
+
+                Iterator<YarnApplicationState> iter = applicationStates
+                    .iterator();
+
+                @Override
+                public boolean hasNext() {
+                  return iter.hasNext();
+                }
+
+                @Override
+                public YarnApplicationStateProto next() {
+                  return ProtoUtils.convertToProtoFormat(iter.next());
+                }
+
+                @Override
+                public void remove() {
+                  throw new UnsupportedOperationException();
+
+                }
+              };
+
+            }
+          };
+      builder.addAllApplicationStates(iterable);
+    }
   }
 
   private void addLocalApplicationTypesToProto() {
@@ -94,6 +134,20 @@ public class GetApplicationsRequestPBImp
     this.applicationTypes.addAll(appTypeList);
   }
 
+  private void initApplicationStates() {
+    if (this.applicationStates != null) {
+      return;
+    }
+    GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder;
+    List<YarnApplicationStateProto> appStatesList =
+        p.getApplicationStatesList();
+    this.applicationStates = EnumSet.noneOf(YarnApplicationState.class);
+
+    for (YarnApplicationStateProto c : appStatesList) {
+      this.applicationStates.add(ProtoUtils.convertFromProtoFormat(c));
+    }
+  }
+
   @Override
   public Set<String> getApplicationTypes() {
     initApplicationTypes();
@@ -109,6 +163,21 @@ public class GetApplicationsRequestPBImp
   }
 
   @Override
+  public EnumSet<YarnApplicationState> getApplicationStates() {
+    initApplicationStates();
+    return this.applicationStates;
+  }
+
+  @Override
+  public void setApplicationStates(EnumSet<YarnApplicationState> applicationStates) {
+    maybeInitBuilder();
+    if (applicationStates == null) {
+      builder.clearApplicationStates();
+    }
+    this.applicationStates = applicationStates;
+  }
+
+  @Override
   public int hashCode() {
     return getProto().hashCode();
   }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Tue Aug 27 17:05:13 2013
@@ -33,6 +33,8 @@ import org.apache.commons.lang.StringUti
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.AdminACLsManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -65,7 +67,6 @@ import com.google.inject.servlet.GuiceFi
 @InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
 public class WebApps {
   static final Logger LOG = LoggerFactory.getLogger(WebApps.class);
-
   public static class Builder<T> {
     static class ServletStruct {
       public Class<? extends HttpServlet> clazz;
@@ -82,6 +83,8 @@ public class WebApps {
     boolean findPort = false;
     Configuration conf;
     boolean devMode = false;
+    private String spnegoPrincipalKey;
+    private String spnegoKeytabKey;
     private final HashSet<ServletStruct> servlets = new HashSet<ServletStruct>();
     private final HashMap<String, Object> attributes = new HashMap<String, Object>();
 
@@ -135,6 +138,16 @@ public class WebApps {
       this.conf = conf;
       return this;
     }
+    
+    public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
+      this.spnegoPrincipalKey = spnegoPrincipalKey;
+      return this;
+    }
+    
+    public Builder<T> withHttpSpnegoKeytabKey(String spnegoKeytabKey) {
+      this.spnegoKeytabKey = spnegoKeytabKey;
+      return this;
+    }
 
     public Builder<T> inDevMode() {
       devMode = true;
@@ -197,8 +210,32 @@ public class WebApps {
           }
         }
         HttpServer server =
-            new HttpServer(name, bindAddress, port, findPort, conf, 
-            new AdminACLsManager(conf).getAdminAcl(), null, webapp.getServePathSpecs());
+            new HttpServer(name, bindAddress, port, findPort, conf,
+                new AdminACLsManager(conf).getAdminAcl(), null,
+                webapp.getServePathSpecs()) {
+
+              {
+                if (UserGroupInformation.isSecurityEnabled()) {
+                  boolean initSpnego = true;
+                  if (spnegoPrincipalKey == null
+                      || conf.get(spnegoPrincipalKey, "").isEmpty()) {
+                    LOG.warn("Principal for spnego filter is not set");
+                    initSpnego = false;
+                  }
+                  if (spnegoKeytabKey == null
+                      || conf.get(spnegoKeytabKey, "").isEmpty()) {
+                    LOG.warn("Keytab for spnego filter is not set");
+                    initSpnego = false;
+                  }
+                  if (initSpnego) {
+                    LOG.info("Initializing spnego filter with principal key : "
+                        + spnegoPrincipalKey + " keytab key : "
+                        + spnegoKeytabKey);
+                    initSpnego(conf, spnegoPrincipalKey, spnegoKeytabKey);
+                  }
+                }
+              }
+            };
         for(ServletStruct struct: servlets) {
           server.addServlet(struct.name, struct.spec, struct.clazz);
         }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java Tue Aug 27 17:05:13 2013
@@ -59,8 +59,15 @@ public class WebServer extends AbstractS
     LOG.info("Instantiating NMWebApp at " + bindAddress);
     try {
       this.webApp =
-          WebApps.$for("node", Context.class, this.nmContext, "ws")
-              .at(bindAddress).with(getConfig()).start(this.nmWebApp);
+          WebApps
+            .$for("node", Context.class, this.nmContext, "ws")
+            .at(bindAddress)
+            .with(getConfig())
+            .withHttpSpnegoPrincipalKey(
+              YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
+            .withHttpSpnegoKeytabKey(
+              YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+            .start(this.nmWebApp);
       this.port = this.webApp.httpServer().getPort();
     } catch (Exception e) {
       String msg = "NMWebapps failed to start.";



Mime
View raw message