hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1567874 - in /hadoop/common/branches/HDFS-5535/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-...
Date Thu, 13 Feb 2014 09:53:54 GMT
Author: szetszwo
Date: Thu Feb 13 09:53:52 2014
New Revision: 1567874

URL: http://svn.apache.org/r1567874
Log:
Merge r1555021 through r1566358 from trunk.

Added:
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityIdentifier.java
      - copied unchanged from r1566358, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityIdentifier.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/GenericObjectMapper.java
      - copied unchanged from r1566358, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/GenericObjectMapper.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/LeveldbApplicationTimelineStore.java
      - copied unchanged from r1566358, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/LeveldbApplicationTimelineStore.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestGenericObjectMapper.java
      - copied unchanged from r1566358, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestGenericObjectMapper.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestLeveldbApplicationTimelineStore.java
      - copied unchanged from r1566358, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestLeveldbApplicationTimelineStore.java
Removed:
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java
Modified:
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
    hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt Thu Feb 13 09:53:52 2014
@@ -17,6 +17,9 @@ Trunk - Unreleased 
     YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy
     Ryza)
 
+    YARN-1497. Command line additions for moving apps between queues (Sandy
+    Ryza)
+
   IMPROVEMENTS
 
   OPTIMIZATIONS
@@ -103,6 +106,19 @@ Release 2.4.0 - UNRELEASED
     new APIs for retrieving and storing timeline information. (Zhijie Shen via
     vinodkv)
 
+    YARN-1490. Introduced the ability to make ResourceManager optionally not kill
+    all containers when an ApplicationMaster exits. (Jian He via vinodkv)
+
+    YARN-1041. Added the ApplicationMasterProtocol API for applications to use the
+    ability in ResourceManager to optionally not kill containers when the
+    ApplicationMaster exits. (Jian He via vinodkv)
+
+    YARN-1566. Changed Distributed Shell to retain containers across application
+    attempts. (Jian He via vinodkv)
+
+    YARN-1635. Implemented a Leveldb based ApplicationTimelineStore. (Billie
+    Rinaldi via zjshen)
+
   IMPROVEMENTS
 
     YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via
@@ -139,6 +155,17 @@ Release 2.4.0 - UNRELEASED
     be available across RM failover by making using of a remote
     configuration-provider. (Xuan Gong via vinodkv)
 
+    YARN-1665. Simplify the configuration of RM HA by having better default
+    values. (Xuan Gong via vinodkv)
+
+    YARN-1660. Simplified the RM HA configuration to accept and be able to simply
+    depend just on configuration properties of the form
+    yarn.resourcemanager.hostname.RMID and use the default ports for all service
+    addresses. (Xuan Gong via vinodkv)
+
+    YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize
+    app-attempts separately from apps. (Jian He via vinodkv)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -194,6 +221,18 @@ Release 2.4.0 - UNRELEASED
     YARN-1684. Fixed history server heap size in yarn script. (Billie Rinaldi
     via zjshen)
 
+    YARN-1166. Fixed app-specific and attempt-specific QueueMetrics to be
+    triggered by accordingly app event and attempt event. 
+
+    YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. (Vinod
+    Kumar Vavilapalli via zjshen)
+
+    YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod
+    Kumar Vavilapalli via zjshen)
+
+    YARN-1672. YarnConfiguration is missing a default for 
+    yarn.nodemanager.log.retain-seconds (Naren Koneru via kasha)
+
 Release 2.3.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -237,18 +276,8 @@ Release 2.3.0 - UNRELEASED
     YARN-1029. Added embedded leader election in the ResourceManager. (Karthik
     Kambatla via vinodkv)
 
-    YARN-1490. Introduced the ability to make ResourceManager optionally not kill
-    all containers when an ApplicationMaster exits. (Jian He via vinodkv)
-
     YARN-1033. Expose RM active/standby state to Web UI and REST API (kasha)
 
-    YARN-1041. Added the ApplicationMasterProtocol API for applications to use the
-    ability in ResourceManager to optionally not kill containers when the
-    ApplicationMaster exits. (Jian He via vinodkv)
-
-    YARN-1566. Changed Distributed Shell to retain containers across application
-    attempts. (Jian He via vinodkv)
-
   IMPROVEMENTS
 
     YARN-305. Fair scheduler logs too many "Node offered to app" messages.
@@ -412,9 +441,6 @@ Release 2.3.0 - UNRELEASED
     YARN-1541. Changed ResourceManager to invalidate ApplicationMaster host/port
     information once an AM crashes. (Jian He via vinodkv)
 
-    YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize
-    app-attempts separately from apps. (Jian He via vinodkv)
-
     YARN-1482. Modified WebApplicationProxy to make it work across ResourceManager
     fail-over. (Xuan Gong via vinodkv)
 
@@ -600,9 +626,6 @@ Release 2.3.0 - UNRELEASED
     YARN-1574. RMDispatcher should be reset on transition to standby. (Xuan Gong
     via kasha)
 
-    YARN-1166. Fixed app-specific and attempt-specific QueueMetrics to be
-    triggered by accordingly app event and attempt event. 
-
     YARN-1598. HA-related rmadmin commands don't work on a secure cluster (kasha)
 
     YARN-1603. Remove two *.orig files which were unexpectedly committed. 
@@ -630,12 +653,6 @@ Release 2.3.0 - UNRELEASED
     YARN-1628. Fixed the test failure in TestContainerManagerSecurity. (Vinod
     Kumar Vavilapalli via zjshen)
 
-    YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod
-    Kumar Vavilapalli via zjshen)
-
-    YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. (Vinod
-    Kumar Vavilapalli via zjshen)
-
 Release 2.2.0 - 2013-10-13
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java Thu Feb 13 09:53:52 2014
@@ -94,9 +94,21 @@ public class ATSPutErrors {
   @Public
   @Unstable
   public static class ATSPutError {
+    /**
+     * Error code returned when no start time can be found when putting an
+     * entity. This occurs when the entity does not already exist in the
+     * store and it is put with no start time or events specified.
+     */
+    public static final int NO_START_TIME = 1;
+    /**
+     * Error code returned if an IOException is encountered when putting an
+     * entity.
+     */
+    public static final int IO_EXCEPTION = 2;
+
     private String entityId;
     private String entityType;
-    private Integer errorCode;
+    private int errorCode;
 
     /**
      * Get the entity Id
@@ -144,7 +156,7 @@ public class ATSPutErrors {
      * @return an error code
      */
     @XmlElement(name = "errorcode")
-    public Integer getErrorCode() {
+    public int getErrorCode() {
       return errorCode;
     }
 
@@ -154,7 +166,7 @@ public class ATSPutErrors {
      * @param errorCode
      *          an error code
      */
-    public void setErrorCode(Integer errorCode) {
+    public void setErrorCode(int errorCode) {
       this.errorCode = errorCode;
     }
 

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java Thu Feb 13 09:53:52 2014
@@ -101,20 +101,7 @@ public class HAUtil {
     for (String id: ids) {
       // verify the RM service addresses configurations for every RMIds
       for (String prefix : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
-        String confKey = null;
-        try {
-          confKey = addSuffix(prefix, id);
-          if (conf.getTrimmed(confKey) == null) {
-            throwBadConfigurationException(getNeedToSetValueMessage(confKey));
-          }
-        } catch (IllegalArgumentException iae) {
-          String errmsg = iae.getMessage();
-          if (confKey == null) {
-            // Error at addSuffix
-            errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, id);
-          }
-          throwBadConfigurationException(errmsg);
-        }
+        checkAndSetRMRPCAddress(prefix, id, conf);
       }
       setValue.append(id);
       setValue.append(",");
@@ -249,9 +236,13 @@ public class HAUtil {
   @InterfaceAudience.Private
   @VisibleForTesting
   static String getConfKeyForRMInstance(String prefix, Configuration conf) {
-    return YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix)
-        ? addSuffix(prefix, getRMHAId(conf))
-        : prefix;
+    if (!YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix)) {
+      return prefix;
+    } else {
+      String RMId = getRMHAId(conf);
+      checkAndSetRMRPCAddress(prefix, RMId, conf);
+      return addSuffix(prefix, RMId);
+    }
   }
 
   public static String getConfValueForRMInstance(String prefix,
@@ -284,4 +275,30 @@ public class HAUtil {
     }
     return key + "." + suffix;
   }
+
+  private static void checkAndSetRMRPCAddress(String prefix, String RMId,
+      Configuration conf) {
+    String rpcAddressConfKey = null;
+    try {
+      rpcAddressConfKey = addSuffix(prefix, RMId);
+      if (conf.getTrimmed(rpcAddressConfKey) == null) {
+        String hostNameConfKey = addSuffix(YarnConfiguration.RM_HOSTNAME, RMId);
+        String confVal = conf.getTrimmed(hostNameConfKey);
+        if (confVal == null) {
+          throwBadConfigurationException(getNeedToSetValueMessage(
+              hostNameConfKey + " or " + addSuffix(prefix, RMId)));
+        } else {
+          conf.set(addSuffix(prefix, RMId), confVal + ":"
+              + YarnConfiguration.getRMDefaultPortNumber(prefix));
+        }
+      }
+    } catch (IllegalArgumentException iae) {
+      String errmsg = iae.getMessage();
+      if (rpcAddressConfKey == null) {
+        // Error at addSuffix
+        errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, RMId);
+      }
+      throwBadConfigurationException(errmsg);
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Thu Feb 13 09:53:52 2014
@@ -23,6 +23,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -108,6 +109,8 @@ public class YarnConfiguration extends C
 
   public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id";
 
+  public static final String RM_HOSTNAME = RM_PREFIX + "hostname";
+
   /** The address of the applications manager interface in the RM.*/
   public static final String RM_ADDRESS = 
     RM_PREFIX + "address";
@@ -373,11 +376,11 @@ public class YarnConfiguration extends C
 
   public static final String AUTO_FAILOVER_ENABLED =
       AUTO_FAILOVER_PREFIX + "enabled";
-  public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = false;
+  public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true;
 
   public static final String AUTO_FAILOVER_EMBEDDED =
       AUTO_FAILOVER_PREFIX + "embedded";
-  public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = false;
+  public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true;
 
   public static final String AUTO_FAILOVER_ZK_BASE_PATH =
       AUTO_FAILOVER_PREFIX + "zk-base-path";
@@ -626,6 +629,7 @@ public class YarnConfiguration extends C
    */
   public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX
       + "log.retain-seconds";
+  public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60;
 
   /**
    * Number of threads used in log cleanup. Only applicable if Log aggregation
@@ -1037,6 +1041,10 @@ public class YarnConfiguration extends C
   /** ATS store class */
   public static final String ATS_STORE = ATS_PREFIX + "store.class";
 
+  /** ATS leveldb path */
+  public static final String ATS_LEVELDB_PATH_PROPERTY =
+      ATS_PREFIX + "leveldb-apptimeline-store.path";
+
   ////////////////////////////////
   // Other Configs
   ////////////////////////////////
@@ -1139,4 +1147,27 @@ public class YarnConfiguration extends C
     }
     return super.updateConnectAddr(prefix, addr);
   }
+
+  @Private
+  public static int getRMDefaultPortNumber(String addressPrefix) {
+    if (addressPrefix.equals(YarnConfiguration.RM_ADDRESS)) {
+      return YarnConfiguration.DEFAULT_RM_PORT;
+    } else if (addressPrefix.equals(YarnConfiguration.RM_SCHEDULER_ADDRESS)) {
+      return YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT;
+    } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_ADDRESS)) {
+      return YarnConfiguration.DEFAULT_RM_WEBAPP_PORT;
+    } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS)) {
+      return YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT;
+    } else if (addressPrefix
+        .equals(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS)) {
+      return YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT;
+    } else if (addressPrefix.equals(YarnConfiguration.RM_ADMIN_ADDRESS)) {
+      return YarnConfiguration.DEFAULT_RM_ADMIN_PORT;
+    } else {
+      throw new HadoopIllegalArgumentException(
+          "Invalid RM RPC address Prefix: " + addressPrefix
+              + ". The valid value should be one of "
+              + YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS);
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java Thu Feb 13 09:53:52 2014
@@ -436,4 +436,19 @@ public abstract class YarnClient extends
   public abstract List<ContainerReport> getContainers(
       ApplicationAttemptId applicationAttemptId) throws YarnException,
       IOException;
+  
+  /**
+   * <p>
+   * Attempts to move the given application to the given queue.
+   * </p>
+   * 
+   * @param appId
+   *    Application to move.
+   * @param queue
+   *    Queue to place it in to.
+   * @throws YarnException
+   * @throws IOException
+   */
+  public abstract void moveApplicationAcrossQueues(ApplicationId appId,
+      String queue) throws YarnException, IOException;
 }

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Thu Feb 13 09:53:52 2014
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.protoc
 import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
@@ -478,4 +479,12 @@ public class YarnClientImpl extends Yarn
     }
     throw new YarnException("History service is not enabled.");
   }
+  
+  @Override
+  public void moveApplicationAcrossQueues(ApplicationId appId,
+      String queue) throws YarnException, IOException {
+    MoveApplicationAcrossQueuesRequest request =
+        MoveApplicationAcrossQueuesRequest.newInstance(appId, queue);
+    rmClient.moveApplicationAcrossQueues(request);
+  }
 }

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Thu Feb 13 09:53:52 2014
@@ -61,6 +61,7 @@ public class ApplicationCLI extends Yarn
   private static final String APP_TYPE_CMD = "appTypes";
   private static final String APP_STATE_CMD = "appStates";
   private static final String ALLSTATES_OPTION = "ALL";
+  private static final String QUEUE_CMD = "queue";
   public static final String APPLICATION = "application";
   public static final String APPLICATION_ATTEMPT = "applicationattempt";
   public static final String CONTAINER = "container";
@@ -96,6 +97,10 @@ public class ApplicationCLI extends Yarn
           + "and -appStates to filter applications based on application state");
     }
     opts.addOption(KILL_CMD, true, "Kills the application.");
+    opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a "
+        + "different queue.");
+    opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to"
+        + " specify which queue to move an application to.");
     opts.addOption(HELP_CMD, false, "Displays help for all commands.");
     Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to "
         + "filter applications based on "
@@ -112,6 +117,8 @@ public class ApplicationCLI extends Yarn
     appStateOpt.setArgName("States");
     opts.addOption(appStateOpt);
     opts.getOption(KILL_CMD).setArgName("Application ID");
+    opts.getOption(MOVE_TO_QUEUE_CMD).setArgName("Application ID");
+    opts.getOption(QUEUE_CMD).setArgName("Queue Name");
     opts.getOption(STATUS_CMD).setArgName("Application ID");
 
     int exitCode = -1;
@@ -202,6 +209,13 @@ public class ApplicationCLI extends Yarn
         return exitCode;
       }
       killApplication(cliParser.getOptionValue(KILL_CMD));
+    } else if (cliParser.hasOption(MOVE_TO_QUEUE_CMD)) {
+      if (!cliParser.hasOption(QUEUE_CMD)) {
+        printUsage(opts);
+        return exitCode;
+      }
+      moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD),
+          cliParser.getOptionValue(QUEUE_CMD));
     } else if (cliParser.hasOption(HELP_CMD)) {
       printUsage(opts);
       return 0;
@@ -366,6 +380,28 @@ public class ApplicationCLI extends Yarn
       client.killApplication(appId);
     }
   }
+  
+  /**
+   * Kills the application with the application id as appId
+   * 
+   * @param applicationId
+   * @throws YarnException
+   * @throws IOException
+   */
+  private void moveApplicationAcrossQueues(String applicationId, String queue)
+      throws YarnException, IOException {
+    ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
+    ApplicationReport appReport = client.getApplicationReport(appId);
+    if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
+        || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
+        || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
+      sysout.println("Application " + applicationId + " has already finished ");
+    } else {
+      sysout.println("Moving application " + applicationId + " to queue " + queue);
+      client.moveApplicationAcrossQueues(appId, queue);
+      sysout.println("Successfully completed move.");
+    }
+  }
 
   /**
    * Prints the application report for an application id.

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java Thu Feb 13 09:53:52 2014
@@ -33,6 +33,7 @@ public abstract class YarnCLI extends Co
   public static final String STATUS_CMD = "status";
   public static final String LIST_CMD = "list";
   public static final String KILL_CMD = "kill";
+  public static final String MOVE_TO_QUEUE_CMD = "movetoqueue";
   public static final String HELP_CMD = "help";
   protected PrintStream sysout;
   protected PrintStream syserr;

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java Thu Feb 13 09:53:52 2014
@@ -172,8 +172,6 @@ public class TestRMFailover extends Clie
   @Test
   public void testAutomaticFailover()
       throws YarnException, InterruptedException, IOException {
-    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
-    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
     conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
     conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
     conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 2000);
@@ -193,6 +191,7 @@ public class TestRMFailover extends Clie
   @Test
   public void testWebAppProxyInStandAloneMode() throws YarnException,
       InterruptedException, IOException {
+    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     WebAppProxyServer webAppProxyServer = new WebAppProxyServer();
     try {
       conf.set(YarnConfiguration.PROXY_ADDRESS, "0.0.0.0:9099");
@@ -227,6 +226,7 @@ public class TestRMFailover extends Clie
   @Test
   public void testEmbeddedWebAppProxy() throws YarnException,
       InterruptedException, IOException {
+    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     cluster.init(conf);
     cluster.start();
     getAdminService(0).transitionToActive(req);

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Thu Feb 13 09:53:52 2014
@@ -675,6 +675,7 @@ public class TestYarnCLI {
     int result = spyCli.run(new String[] { "-help" });
     Assert.assertTrue(result == 0);
     verify(spyCli).printUsage(any(Options.class));
+    System.err.println(sysOutStream.toString()); //todo sandyt remove this hejfkdsl
     Assert.assertEquals(createApplicationCLIHelpMessage(),
         sysOutStream.toString());
 
@@ -748,6 +749,56 @@ public class TestYarnCLI {
           "' doesn't exist in RM.", ex.getMessage());
     }
   }
+  
+  @Test
+  public void testMoveApplicationAcrossQueues() throws Exception {
+    ApplicationCLI cli = createAndGetAppCLI();
+    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+
+    ApplicationReport newApplicationReport2 = ApplicationReport.newInstance(
+        applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
+        "user", "queue", "appname", "host", 124, null,
+        YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+    when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
+        newApplicationReport2);
+    int result = cli.run(new String[] { "-movetoqueue", applicationId.toString(),
+        "-queue", "targetqueue"});
+    assertEquals(0, result);
+    verify(client, times(0)).moveApplicationAcrossQueues(
+        any(ApplicationId.class), any(String.class));
+    verify(sysOut).println(
+        "Application " + applicationId + " has already finished ");
+
+    ApplicationReport newApplicationReport = ApplicationReport.newInstance(
+        applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
+        "user", "queue", "appname", "host", 124, null,
+        YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+    when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
+        newApplicationReport);
+    result = cli.run(new String[] { "-movetoqueue", applicationId.toString(),
+        "-queue", "targetqueue"});
+    assertEquals(0, result);
+    verify(client).moveApplicationAcrossQueues(any(ApplicationId.class),
+        any(String.class));
+    verify(sysOut).println("Moving application application_1234_0005 to queue targetqueue");
+    verify(sysOut).println("Successfully completed move.");
+
+    doThrow(new ApplicationNotFoundException("Application with id '"
+        + applicationId + "' doesn't exist in RM.")).when(client)
+        .moveApplicationAcrossQueues(applicationId, "targetqueue");
+    cli = createAndGetAppCLI();
+    try {
+      result = cli.run(new String[] { "-movetoqueue", applicationId.toString(),
+          "-queue", "targetqueue"});
+      Assert.fail();
+    } catch (Exception ex) {
+      Assert.assertTrue(ex instanceof ApplicationNotFoundException);
+      Assert.assertEquals("Application with id '" + applicationId +
+          "' doesn't exist in RM.", ex.getMessage());
+    }
+  }
 
   @Test
   public void testListClusterNodes() throws Exception {
@@ -1087,23 +1138,28 @@ public class TestYarnCLI {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintWriter pw = new PrintWriter(baos);
     pw.println("usage: application");
-    pw.println(" -appStates <States>        Works with -list to filter applications based");
-    pw.println("                            on input comma-separated list of application");
-    pw.println("                            states. The valid application state can be one");
-    pw.println("                            of the following:");
-    pw.println("                            ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUNNING,");
-    pw.println("                            FINISHED,FAILED,KILLED");
-    pw.println(" -appTypes <Types>          Works with -list to filter applications based");
-    pw.println("                            on input comma-separated list of application");
-    pw.println("                            types.");
-    pw.println(" -help                      Displays help for all commands.");
-    pw.println(" -kill <Application ID>     Kills the application.");
-    pw.println(" -list                      List applications from the RM. Supports");
-    pw.println("                            optional use of -appTypes to filter");
-    pw.println("                            applications based on application type, and");
-    pw.println("                            -appStates to filter applications based on");
-    pw.println("                            application state");
-    pw.println(" -status <Application ID>   Prints the status of the application.");
+    pw.println(" -appStates <States>             Works with -list to filter applications");
+    pw.println("                                 based on input comma-separated list of");
+    pw.println("                                 application states. The valid application");
+    pw.println("                                 state can be one of the following:");
+    pw.println("                                 ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN");
+    pw.println("                                 NING,FINISHED,FAILED,KILLED");
+    pw.println(" -appTypes <Types>               Works with -list to filter applications");
+    pw.println("                                 based on input comma-separated list of");
+    pw.println("                                 application types.");
+    pw.println(" -help                           Displays help for all commands.");
+    pw.println(" -kill <Application ID>          Kills the application.");
+    pw.println(" -list                           List applications from the RM. Supports");
+    pw.println("                                 optional use of -appTypes to filter");
+    pw.println("                                 applications based on application type,");
+    pw.println("                                 and -appStates to filter applications");
+    pw.println("                                 based on application state");
+    pw.println(" -movetoqueue <Application ID>   Moves the application to a different");
+    pw.println("                                 queue.");
+    pw.println(" -queue <Queue Name>             Works with the movetoqueue command to");
+    pw.println("                                 specify which queue to move an");
+    pw.println("                                 application to.");
+    pw.println(" -status <Application ID>        Prints the status of the application.");
     pw.close();
     String appsHelpStr = baos.toString("UTF-8");
     return appsHelpStr;

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Thu Feb 13 09:53:52 2014
@@ -405,17 +405,20 @@
   </property>
 
   <property>
-    <description>Enable automatic failover.</description>
+    <description>Enable automatic failover.
+      By default, it is enabled only when HA is enabled</description>
     <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
-    <value>false</value>
+    <value>true</value>
   </property>
 
   <property>
-    <description>Enable embedded automatic failover. The embedded elector
-      relies on the RM state store to handle fencing, and is primarily intended
-      to be used in conjunction with ZKRMStateStore.</description>
+    <description>Enable embedded automatic failover.
+      By default, it is enabled only when HA is enabled.
+      The embedded elector relies on the RM state store to handle fencing,
+      and is primarily intended to be used in conjunction with ZKRMStateStore.
+    </description>
     <name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
-    <value>false</value>
+    <value>true</value>
   </property>
 
   <property>
@@ -1142,7 +1145,13 @@
   <property>
     <description>Store class name for application timeline store</description>
     <name>yarn.ats.store.class</name>
-    <value>org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore</value>
+    <value>org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.LeveldbApplicationTimelineStore</value>
+  </property>
+
+  <property>
+    <description>Store file name for leveldb application timeline store</description>
+    <name>yarn.ats.leveldb-apptimeline-store.path</name>
+    <value>${yarn.log.dir}/ats</value>
   </property>
 
   <!-- Other configuration -->

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java Thu Feb 13 09:53:52 2014
@@ -117,14 +117,14 @@ public class TestApplicationTimelineReco
     ATSPutError error1 = new ATSPutError();
     error1.setEntityId("entity id 1");
     error1.setEntityId("entity type 1");
-    error1.setErrorCode(1);
+    error1.setErrorCode(ATSPutError.NO_START_TIME);
     atsPutErrors.addError(error1);
     List<ATSPutError> errors = new ArrayList<ATSPutError>();
     errors.add(error1);
     ATSPutError error2 = new ATSPutError();
     error2.setEntityId("entity id 2");
     error2.setEntityId("entity type 2");
-    error2.setErrorCode(2);
+    error2.setErrorCode(ATSPutError.IO_EXCEPTION);
     errors.add(error2);
     atsPutErrors.addErrors(errors);
 

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java Thu Feb 13 09:53:52 2014
@@ -159,8 +159,9 @@ public class TestHAUtil {
       String confKey =
         HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID);
       assertEquals("YarnRuntimeException by Configuration#set()",
-        HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(confKey),
-        e.getMessage());
+        HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(
+            HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, RM1_NODE_ID)
+            + " or " + confKey), e.getMessage());
     }
 
     // simulate the case YarnConfiguration.RM_HA_IDS doesn't contain

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml Thu Feb 13 09:53:52 2014
@@ -167,6 +167,25 @@
       <artifactId>jersey-test-framework-grizzly2</artifactId>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.codehaus.jackson</groupId>
+      <artifactId>jackson-core-asl</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.codehaus.jackson</groupId>
+      <artifactId>jackson-mapper-asl</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-collections</groupId>
+      <artifactId>commons-collections</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.fusesource.leveldbjni</groupId>
+      <artifactId>leveldbjni-all</artifactId>
+    </dependency>
  
   </dependencies>
 </project>

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java Thu Feb 13 09:53:52 2014
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.Set;
@@ -78,13 +79,15 @@ public interface ApplicationTimelineRead
    *                         retrieve (see {@link Field}). If the set of fields
    *                         contains {@link Field#LAST_EVENT_ONLY} and not
    *                         {@link Field#EVENTS}, the most recent event for
-   *                         each entity is retrieved.
+   *                         each entity is retrieved. If null, retrieves all
+   *                         fields.
    * @return An {@link ATSEntities} object.
+   * @throws IOException
    */
   ATSEntities getEntities(String entityType,
       Long limit, Long windowStart, Long windowEnd,
       NameValuePair primaryFilter, Collection<NameValuePair> secondaryFilters,
-      EnumSet<Field> fieldsToRetrieve);
+      EnumSet<Field> fieldsToRetrieve) throws IOException;
 
   /**
    * This method retrieves the entity information for a given entity.
@@ -95,11 +98,13 @@ public interface ApplicationTimelineRead
    *                         retrieve (see {@link Field}). If the set of
    *                         fields contains {@link Field#LAST_EVENT_ONLY} and
    *                         not {@link Field#EVENTS}, the most recent event
-   *                         for each entity is retrieved.
+   *                         for each entity is retrieved. If null, retrieves
+   *                         all fields.
    * @return An {@link ATSEntity} object.
+   * @throws IOException
    */
   ATSEntity getEntity(String entity, String entityType, EnumSet<Field>
-      fieldsToRetrieve);
+      fieldsToRetrieve) throws IOException;
 
   /**
    * This method retrieves the events for a list of entities all of the same
@@ -118,8 +123,9 @@ public interface ApplicationTimelineRead
    * @param eventTypes Restricts the events returned to the given types. If
    *                   null, events of all types will be returned.
    * @return An {@link ATSEvents} object.
+   * @throws IOException
    */
   ATSEvents getEntityTimelines(String entityType,
       SortedSet<String> entityIds, Long limit, Long windowStart,
-      Long windowEnd, Set<String> eventTypes);
+      Long windowEnd, Set<String> eventTypes) throws IOException;
 }

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java Thu Feb 13 09:53:52 2014
@@ -23,6 +23,8 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities;
 import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors;
 
+import java.io.IOException;
+
 /**
  * This interface is for storing application timeline information.
  */
@@ -37,7 +39,8 @@ public interface ApplicationTimelineWrit
    *
    * @param data An {@link ATSEntities} object.
    * @return An {@link ATSPutErrors} object.
+   * @throws IOException
    */
-  ATSPutErrors put(ATSEntities data);
+  ATSPutErrors put(ATSEntities data) throws IOException;
 
 }

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java Thu Feb 13 09:53:52 2014
@@ -53,8 +53,8 @@ import org.apache.hadoop.yarn.api.record
 public class MemoryApplicationTimelineStore
     extends AbstractService implements ApplicationTimelineStore {
 
-  private Map<EntityId, ATSEntity> entities =
-      new HashMap<EntityId, ATSEntity>();
+  private Map<EntityIdentifier, ATSEntity> entities =
+      new HashMap<EntityIdentifier, ATSEntity>();
 
   public MemoryApplicationTimelineStore() {
     super(MemoryApplicationTimelineStore.class.getName());
@@ -125,7 +125,7 @@ public class MemoryApplicationTimelineSt
     if (fieldsToRetrieve == null) {
       fieldsToRetrieve = EnumSet.allOf(Field.class);
     }
-    ATSEntity entity = entities.get(new EntityId(entityId, entityType));
+    ATSEntity entity = entities.get(new EntityIdentifier(entityId, entityType));
     if (entity == null) {
       return null;
     } else {
@@ -152,7 +152,7 @@ public class MemoryApplicationTimelineSt
       windowEnd = Long.MAX_VALUE;
     }
     for (String entityId : entityIds) {
-      EntityId entityID = new EntityId(entityId, entityType);
+      EntityIdentifier entityID = new EntityIdentifier(entityId, entityType);
       ATSEntity entity = entities.get(entityID);
       if (entity == null) {
         continue;
@@ -184,8 +184,8 @@ public class MemoryApplicationTimelineSt
   public ATSPutErrors put(ATSEntities data) {
     ATSPutErrors errors = new ATSPutErrors();
     for (ATSEntity entity : data.getEntities()) {
-      EntityId entityId =
-          new EntityId(entity.getEntityId(), entity.getEntityType());
+      EntityIdentifier entityId =
+          new EntityIdentifier(entity.getEntityId(), entity.getEntityType());
       // store entity info in memory
       ATSEntity existingEntity = entities.get(entityId);
       if (existingEntity == null) {
@@ -210,7 +210,7 @@ public class MemoryApplicationTimelineSt
           ATSPutError error = new ATSPutError();
           error.setEntityId(entityId.getId());
           error.setEntityType(entityId.getType());
-          error.setErrorCode(1);
+          error.setErrorCode(ATSPutError.NO_START_TIME);
           errors.addError(error);
           entities.remove(entityId);
           continue;
@@ -242,12 +242,20 @@ public class MemoryApplicationTimelineSt
           continue;
         }
         for (String idStr : partRelatedEntities.getValue()) {
-          EntityId relatedEntityId =
-              new EntityId(idStr, partRelatedEntities.getKey());
+          EntityIdentifier relatedEntityId =
+              new EntityIdentifier(idStr, partRelatedEntities.getKey());
           ATSEntity relatedEntity = entities.get(relatedEntityId);
           if (relatedEntity != null) {
             relatedEntity.addRelatedEntity(
                 existingEntity.getEntityType(), existingEntity.getEntityId());
+          } else {
+            relatedEntity = new ATSEntity();
+            relatedEntity.setEntityId(relatedEntityId.getId());
+            relatedEntity.setEntityType(relatedEntityId.getType());
+            relatedEntity.setStartTime(existingEntity.getStartTime());
+            relatedEntity.addRelatedEntity(existingEntity.getEntityType(),
+                existingEntity.getEntityId());
+            entities.put(relatedEntityId, relatedEntity);
           }
         }
       }

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java Thu Feb 13 09:53:52 2014
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.EnumSet;
@@ -45,6 +46,8 @@ import javax.xml.bind.annotation.XmlAcce
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities;
@@ -64,6 +67,8 @@ import com.google.inject.Singleton;
 //TODO: support XML serialization/deserialization
 public class ATSWebServices {
 
+  private static final Log LOG = LogFactory.getLog(ATSWebServices.class);
+
   private ApplicationTimelineStore store;
 
   @Inject
@@ -143,6 +148,10 @@ public class ATSWebServices {
           "windowStart, windowEnd or limit is not a numeric value.");
     } catch (IllegalArgumentException e) {
       throw new BadRequestException("requested invalid field.");
+    } catch (IOException e) {
+      LOG.error("Error getting entities", e);
+      throw new WebApplicationException(e,
+          Response.Status.INTERNAL_SERVER_ERROR);
     }
     if (entities == null) {
       return new ATSEntities();
@@ -171,6 +180,10 @@ public class ATSWebServices {
     } catch (IllegalArgumentException e) {
       throw new BadRequestException(
           "requested invalid field.");
+    } catch (IOException e) {
+      LOG.error("Error getting entity", e);
+      throw new WebApplicationException(e,
+          Response.Status.INTERNAL_SERVER_ERROR);
     }
     if (entity == null) {
       throw new WebApplicationException(Response.Status.NOT_FOUND);
@@ -206,6 +219,10 @@ public class ATSWebServices {
     } catch (NumberFormatException e) {
       throw new BadRequestException(
           "windowStart, windowEnd or limit is not a numeric value.");
+    } catch (IOException e) {
+      LOG.error("Error getting entity timelines", e);
+      throw new WebApplicationException(e,
+          Response.Status.INTERNAL_SERVER_ERROR);
     }
     if (events == null) {
       return new ATSEvents();
@@ -228,7 +245,13 @@ public class ATSWebServices {
     if (entities == null) {
       return new ATSPutErrors();
     }
-    return store.put(entities);
+    try {
+      return store.put(entities);
+    } catch (IOException e) {
+      LOG.error("Error putting entities", e);
+      throw new WebApplicationException(e,
+          Response.Status.INTERNAL_SERVER_ERROR);
+    }
   }
 
   private void init(HttpServletResponse response) {
@@ -275,7 +298,17 @@ public class ATSWebServices {
     String[] strs = str.split(delimiter);
     List<Field> fieldList = new ArrayList<Field>();
     for (String s : strs) {
-      fieldList.add(Field.valueOf(s.toUpperCase()));
+      s = s.trim().toUpperCase();
+      if (s.equals("EVENTS"))
+        fieldList.add(Field.EVENTS);
+      else if (s.equals("LASTEVENTONLY"))
+        fieldList.add(Field.LAST_EVENT_ONLY);
+      else if (s.equals("RELATEDENTITIES"))
+        fieldList.add(Field.RELATED_ENTITIES);
+      else if (s.equals("PRIMARYFILTERS"))
+        fieldList.add(Field.PRIMARY_FILTERS);
+      else if (s.equals("OTHERINFO"))
+        fieldList.add(Field.OTHER_INFO);
     }
     if (fieldList.size() == 0)
       return null;

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java Thu Feb 13 09:53:52 2014
@@ -21,6 +21,8 @@ import static org.junit.Assert.assertEqu
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
+import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -71,7 +73,7 @@ public class ApplicationTimelineStoreTes
   /**
    * Load test data into the given store
    */
-  protected void loadTestData() {
+  protected void loadTestData() throws IOException {
     ATSEntities atsEntities = new ATSEntities();
     Map<String, Object> primaryFilters = new HashMap<String, Object>();
     primaryFilters.put("user", "username");
@@ -126,7 +128,7 @@ public class ApplicationTimelineStoreTes
     response = store.put(atsEntities);
     assertEquals(0, response.getErrors().size());
     atsEntities.setEntities(Collections.singletonList(createEntity(entity1b,
-        entityType1, 123l, Collections.singletonList(ev2), null,
+        entityType1, 789l, Collections.singletonList(ev2), null,
         primaryFilters, otherInfo2)));
     response = store.put(atsEntities);
     assertEquals(0, response.getErrors().size());
@@ -138,11 +140,11 @@ public class ApplicationTimelineStoreTes
     ATSPutError error = response.getErrors().get(0);
     assertEquals("badentityid", error.getEntityId());
     assertEquals("badentity", error.getEntityType());
-    assertEquals((Integer) 1, error.getErrorCode());
+    assertEquals(ATSPutError.NO_START_TIME, error.getErrorCode());
   }
 
   /**
-   * Load veification data
+   * Load verification data
    */
   protected void loadVerificationData() throws Exception {
     userFilter = new NameValuePair("user",
@@ -197,7 +199,7 @@ public class ApplicationTimelineStoreTes
     events2.add(ev4);
   }
 
-  public void testGetSingleEntity() {
+  public void testGetSingleEntity() throws IOException {
     // test getting entity info
     verifyEntityInfo(null, null, null, null, null, null,
         store.getEntity("id_1", "type_2", EnumSet.allOf(Field.class)));
@@ -222,6 +224,10 @@ public class ApplicationTimelineStoreTes
         null, null, null, store.getEntity(entity1, entityType1,
         EnumSet.of(Field.LAST_EVENT_ONLY)));
 
+    verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES,
+        primaryFilters, otherInfo, store.getEntity(entity1b, entityType1,
+        null));
+
     verifyEntityInfo(entity1, entityType1, null, null, primaryFilters, null,
         store.getEntity(entity1, entityType1,
             EnumSet.of(Field.PRIMARY_FILTERS)));
@@ -234,7 +240,7 @@ public class ApplicationTimelineStoreTes
             EnumSet.of(Field.RELATED_ENTITIES)));
   }
 
-  public void testGetEntities() {
+  public void testGetEntities() throws IOException {
     // test getting entities
     assertEquals("nonzero entities size for nonexistent type", 0,
         store.getEntities("type_0", null, null, null, null, null,
@@ -305,7 +311,7 @@ public class ApplicationTimelineStoreTes
         primaryFilters, otherInfo, entities.get(1));
   }
 
-  public void testGetEntitiesWithPrimaryFilters() {
+  public void testGetEntitiesWithPrimaryFilters() throws IOException {
     // test using primary filter
     assertEquals("nonzero entities size for primary filter", 0,
         store.getEntities("type_1", null, null, null,
@@ -361,7 +367,7 @@ public class ApplicationTimelineStoreTes
         primaryFilters, otherInfo, entities.get(1));
   }
 
-  public void testGetEntitiesWithSecondaryFilters() {
+  public void testGetEntitiesWithSecondaryFilters() throws IOException {
     // test using secondary filter
     List<ATSEntity> entities = store.getEntities("type_1", null, null, null,
         null, goodTestingFilters, EnumSet.allOf(Field.class)).getEntities();
@@ -388,7 +394,7 @@ public class ApplicationTimelineStoreTes
     assertEquals(0, entities.size());
   }
 
-  public void testGetEvents() {
+  public void testGetEvents() throws IOException {
     // test getting entity timelines
     SortedSet<String> sortedSet = new TreeSet<String>();
     sortedSet.add(entity1);

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java Thu Feb 13 09:53:52 2014
@@ -23,6 +23,7 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
 
 public class TestMemoryApplicationTimelineStore
     extends ApplicationTimelineStoreTestUtils {
@@ -46,27 +47,27 @@ public class TestMemoryApplicationTimeli
   }
 
   @Test
-  public void testGetSingleEntity() {
+  public void testGetSingleEntity() throws IOException {
     super.testGetSingleEntity();
   }
 
   @Test
-  public void testGetEntities() {
+  public void testGetEntities() throws IOException {
     super.testGetEntities();
   }
 
   @Test
-  public void testGetEntitiesWithPrimaryFilters() {
+  public void testGetEntitiesWithPrimaryFilters() throws IOException {
     super.testGetEntitiesWithPrimaryFilters();
   }
 
   @Test
-  public void testGetEntitiesWithSecondaryFilters() {
+  public void testGetEntitiesWithSecondaryFilters() throws IOException {
     super.testGetEntitiesWithSecondaryFilters();
   }
 
   @Test
-  public void testGetEvents() {
+  public void testGetEvents() throws IOException {
     super.testGetEvents();
   }
 

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java Thu Feb 13 09:53:52 2014
@@ -157,6 +157,43 @@ public class TestATSWebServices extends 
   }
 
   @Test
+  public void testGetEntityFields1() throws Exception {
+    WebResource r = resource();
+    ClientResponse response = r.path("ws").path("v1").path("apptimeline")
+        .path("type_1").path("id_1").queryParam("fields", "events,otherinfo")
+        .accept(MediaType.APPLICATION_JSON)
+        .get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    ATSEntity entity = response.getEntity(ATSEntity.class);
+    Assert.assertNotNull(entity);
+    Assert.assertEquals("id_1", entity.getEntityId());
+    Assert.assertEquals("type_1", entity.getEntityType());
+    Assert.assertEquals(123l, entity.getStartTime().longValue());
+    Assert.assertEquals(2, entity.getEvents().size());
+    Assert.assertEquals(0, entity.getPrimaryFilters().size());
+    Assert.assertEquals(4, entity.getOtherInfo().size());
+  }
+
+  @Test
+  public void testGetEntityFields2() throws Exception {
+    WebResource r = resource();
+    ClientResponse response = r.path("ws").path("v1").path("apptimeline")
+        .path("type_1").path("id_1").queryParam("fields", "lasteventonly," +
+            "primaryfilters,relatedentities")
+        .accept(MediaType.APPLICATION_JSON)
+        .get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    ATSEntity entity = response.getEntity(ATSEntity.class);
+    Assert.assertNotNull(entity);
+    Assert.assertEquals("id_1", entity.getEntityId());
+    Assert.assertEquals("type_1", entity.getEntityType());
+    Assert.assertEquals(123l, entity.getStartTime().longValue());
+    Assert.assertEquals(1, entity.getEvents().size());
+    Assert.assertEquals(2, entity.getPrimaryFilters().size());
+    Assert.assertEquals(0, entity.getOtherInfo().size());
+  }
+
+  @Test
   public void testGetEvents() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("apptimeline")

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java Thu Feb 13 09:53:52 2014
@@ -73,7 +73,8 @@ public class NonAggregatingLogHandler ex
   protected void serviceInit(Configuration conf) throws Exception {
     // Default 3 hours.
     this.deleteDelaySeconds =
-        conf.getLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 3 * 60 * 60);
+        conf.getLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS,
+                YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS);
     sched = createScheduledThreadPoolExecutor(conf);
     super.serviceInit(conf);
   }

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java Thu Feb 13 09:53:52 2014
@@ -145,7 +145,8 @@ public class TestNonAggregatingLogHandle
     conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
     conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
 
-    conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 10800l);
+    conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS,
+            YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS);
 
     DrainDispatcher dispatcher = createDispatcher(conf);
     EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java Thu Feb 13 09:53:52 2014
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnC
 import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -119,6 +120,7 @@ public class TestRMHA {
    */
   @Test (timeout = 30000)
   public void testStartAndTransitions() throws IOException {
+    configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     Configuration conf = new YarnConfiguration(configuration);
     rm = new MockRM(conf);
     rm.init(conf);
@@ -178,7 +180,6 @@ public class TestRMHA {
         "automatic failover is enabled";
 
     Configuration conf = new YarnConfiguration(configuration);
-    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
 
     rm = new MockRM(conf);
     rm.init(conf);
@@ -236,6 +237,7 @@ public class TestRMHA {
     String errorMessageForEventHandler =
         "Expect to get the same number of handlers";
     String errorMessageForService = "Expect to get the same number of services";
+    configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     Configuration conf = new YarnConfiguration(configuration);
     rm = new MockRM(conf) {
       @Override
@@ -313,6 +315,57 @@ public class TestRMHA {
     }
   }
 
+  @Test
+  public void testHAWithRMHostName() {
+    //test if both RM_HOSTBANE_{rm_id} and RM_RPCADDRESS_{rm_id} are set
+    //We should only read rpc addresses from RM_RPCADDRESS_{rm_id} configuration
+    configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+        RM1_NODE_ID), "1.1.1.1");
+    configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+        RM2_NODE_ID), "0.0.0.0");
+    configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+        RM3_NODE_ID), "2.2.2.2");
+    try {
+      Configuration conf = new YarnConfiguration(configuration);
+      rm = new MockRM(conf);
+      rm.init(conf);
+      for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
+        assertEquals("RPC address not set for " + confKey,
+            RM1_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM1_NODE_ID)));
+        assertEquals("RPC address not set for " + confKey,
+            RM2_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM2_NODE_ID)));
+        assertEquals("RPC address not set for " + confKey,
+            RM3_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM3_NODE_ID)));
+      }
+    } catch (YarnRuntimeException e) {
+      fail("Should not throw any exceptions.");
+    }
+
+    //test if only RM_HOSTBANE_{rm_id} is set
+    configuration.clear();
+    configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+    configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
+        + RM2_NODE_ID);
+    configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+        RM1_NODE_ID), "1.1.1.1");
+    configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+        RM2_NODE_ID), "0.0.0.0");
+    try {
+      Configuration conf = new YarnConfiguration(configuration);
+      rm = new MockRM(conf);
+      rm.init(conf);
+      assertEquals("RPC address not set for " + YarnConfiguration.RM_ADDRESS,
+          "1.1.1.1:8032",
+          conf.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID)));
+      assertEquals("RPC address not set for " + YarnConfiguration.RM_ADDRESS,
+          "0.0.0.0:8032",
+          conf.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM2_NODE_ID)));
+
+    } catch (YarnRuntimeException e) {
+      fail("Should not throw any exceptions.");
+    }
+  }
+
   @SuppressWarnings("rawtypes")
   class MyCountingDispatcher extends AbstractService implements Dispatcher {
 

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Thu Feb 13 09:53:52 2014
@@ -159,6 +159,7 @@ public class TestZKRMStateStore extends 
         HAServiceProtocol.RequestSource.REQUEST_BY_USER);
 
     Configuration conf1 = createHARMConf("rm1,rm2", "rm1", 1234);
+    conf1.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     ResourceManager rm1 = new ResourceManager();
     rm1.init(conf1);
     rm1.start();
@@ -170,6 +171,7 @@ public class TestZKRMStateStore extends 
         rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
 
     Configuration conf2 = createHARMConf("rm1,rm2", "rm2", 5678);
+    conf2.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     ResourceManager rm2 = new ResourceManager();
     rm2.init(conf2);
     rm2.start();

Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java?rev=1567874&r1=1567873&r2=1567874&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java Thu Feb 13 09:53:52 2014
@@ -42,7 +42,7 @@ public class TestMiniYARNClusterForHA {
   @Before
   public void setup() throws IOException, InterruptedException {
     Configuration conf = new YarnConfiguration();
-
+    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     cluster = new MiniYARNCluster(TestMiniYARNClusterForHA.class.getName(),
         2, 1, 1, 1);
     cluster.init(conf);



Mime
View raw message