hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1562182 - in /hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn: hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ hadoop-yarn-api/src/main/proto/ hadoop-yarn-api/src/main/proto/server/ hadoop-yarn-common/src/...
Date Tue, 28 Jan 2014 19:50:29 GMT
Author: vinodkv
Date: Tue Jan 28 19:50:28 2014
New Revision: 1562182

URL: http://svn.apache.org/r1562182
Log:
YARN-934. Updated patch for HistoryWriter to split the write operations into a startOp and a finishOp. Contributed by Zhijie Shen.
YARN-947. Updated patch to have separate data object for startOp and finishOp. Contributed by Zhijie Shen.
YARN-956. Updated patch for In-memory Storage after the above two changes.
svn merge --ignore-ancestry -c 1556726 ../YARN-321

Added:
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
      - copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
Removed:
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java
Modified:
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto Tue Jan 28 19:50:28 2014
@@ -35,6 +35,25 @@ message ApplicationHistoryDataProto {
   optional int64 finish_time = 8;
   optional string diagnostics_info = 9;
   optional FinalApplicationStatusProto final_application_status = 10;
+  optional YarnApplicationStateProto yarn_application_state = 11;
+}
+
+message ApplicationStartDataProto {
+  optional ApplicationIdProto application_id = 1;
+  optional string application_name = 2;
+  optional string application_type = 3;
+  optional string user = 4;
+  optional string queue = 5;
+  optional int64 submit_time = 6;
+  optional int64 start_time = 7;
+}
+
+message ApplicationFinishDataProto {
+  optional ApplicationIdProto application_id = 1;
+  optional int64 finish_time = 2;
+  optional string diagnostics_info = 3;
+  optional FinalApplicationStatusProto final_application_status = 4;
+  optional YarnApplicationStateProto yarn_application_state = 5;
 }
 
 message ApplicationAttemptHistoryDataProto {
@@ -45,6 +64,22 @@ message ApplicationAttemptHistoryDataPro
   optional string diagnostics_info = 5;
   optional FinalApplicationStatusProto final_application_status = 6;
   optional ContainerIdProto master_container_id = 7;
+  optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 8;
+}
+
+message ApplicationAttemptStartDataProto {
+  optional ApplicationAttemptIdProto application_attempt_id = 1;
+  optional string host = 2;
+  optional int32 rpc_port = 3;
+  optional ContainerIdProto master_container_id = 4;
+}
+
+message ApplicationAttemptFinishDataProto {
+  optional ApplicationAttemptIdProto application_attempt_id = 1;
+  optional string tracking_url = 2;
+  optional string diagnostics_info = 3;
+  optional FinalApplicationStatusProto final_application_status = 4;
+  optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 5;
 }
 
 message ContainerHistoryDataProto {
@@ -56,5 +91,23 @@ message ContainerHistoryDataProto {
   optional int64 finish_time = 6;
   optional string diagnostics_info = 7;
   optional string log_url = 8;
-  optional ContainerStateProto final_container_status = 9;
+  optional int32 container_exit_status = 9;
+  optional ContainerStateProto container_state = 10;
+}
+
+message ContainerStartDataProto {
+  optional ContainerIdProto container_id = 1;
+  optional ResourceProto allocated_resource = 2;
+  optional NodeIdProto assigned_node_id = 3;
+  optional PriorityProto priority = 4;
+  optional int64 start_time = 5;
+}
+
+message ContainerFinishDataProto {
+  optional ContainerIdProto container_id = 1;
+  optional int64 finish_time = 2;
+  optional string diagnostics_info = 3;
+  optional string log_url = 4;
+  optional int32 container_exit_status = 5;
+  optional ContainerStateProto container_state = 6;
 }
\ No newline at end of file

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Tue Jan 28 19:50:28 2014
@@ -98,6 +98,21 @@ enum YarnApplicationStateProto {
   KILLED = 8;
 }
 
+enum YarnApplicationAttemptStateProto {
+  APP_ATTEMPT_NEW = 1;
+  APP_ATTEMPT_SUBMITTED = 2;
+  APP_ATTEMPT_SCHEDULED = 3;
+  APP_ATTEMPT_ALLOCATED_SAVING = 4;
+  APP_ATTEMPT_ALLOCATED = 5;
+  APP_ATTEMPT_LAUNCHED = 6;
+  APP_ATTEMPT_FAILED = 7;
+  APP_ATTEMPT_RUNNING = 8;
+  APP_ATTEMPT_FINAL_SAVING = 9;
+  APP_ATTEMPT_FINISHING = 10;
+  APP_ATTEMPT_FINISHED = 11;
+  APP_ATTEMPT_KILLED = 12;
+}
+
 enum FinalApplicationStatusProto {
   APP_UNDEFINED = 0;
   APP_SUCCEEDED = 1;

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java Tue Jan 28 19:50:28 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto;
@@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.proto.Yarn
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
 
 import com.google.protobuf.ByteString;
@@ -97,6 +99,21 @@ public class ProtoUtils {
   }
 
   /*
+   * YarnApplicationAttemptState
+   */
+  private static String YARN_APPLICATION_ATTEMPT_STATE_PREFIX = "APP_ATTEMPT_";
+  public static YarnApplicationAttemptStateProto convertToProtoFormat(
+      YarnApplicationAttemptState e) {
+    return YarnApplicationAttemptStateProto
+        .valueOf(YARN_APPLICATION_ATTEMPT_STATE_PREFIX + e.name());
+  }
+  public static YarnApplicationAttemptState convertFromProtoFormat(
+      YarnApplicationAttemptStateProto e) {
+    return YarnApplicationAttemptState.valueOf(e.name().replace(
+        YARN_APPLICATION_ATTEMPT_STATE_PREFIX, ""));
+  }
+
+  /*
    * ApplicationResourceUsageReport
    */
   public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) {

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java Tue Jan 28 19:50:28 2014
@@ -20,9 +20,18 @@ package org.apache.hadoop.yarn.server.ap
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.Service;
 
+/**
+ * This class is the abstract of the storage of the application history data. It
+ * is a {@link Service}, such that the implementation of this class can make use
+ * of the service life cycle to initialize and cleanup the storage. Users can
+ * access the storage via {@link ApplicationHistoryReader} and
+ * {@link ApplicationHistoryWriter} interfaces.
+ * 
+ */
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
-public interface ApplicationHistoryStore extends ApplicationHistoryReader,
-    ApplicationHistoryWriter {
+public interface ApplicationHistoryStore extends Service,
+    ApplicationHistoryReader, ApplicationHistoryWriter {
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java Tue Jan 28 19:50:28 2014
@@ -18,11 +18,16 @@
 
 package org.apache.hadoop.yarn.server.applicationhistoryservice;
 
+import java.io.IOException;
+
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
 
 /**
  * It is the interface of writing the application history, exposing the methods
@@ -34,25 +39,72 @@ import org.apache.hadoop.yarn.server.app
 public interface ApplicationHistoryWriter {
 
   /**
-   * This method persists an {@link ApplicationHistoryData} object.
-   * @param app the {@link ApplicationHistoryData} object
-   * @throws Throwable
+   * This method writes the information of <code>RMApp</code> that is available
+   * when it starts.
+   * 
+   * @param appStart
+   *          the record of the information of <code>RMApp</code> that is
+   *          available when it starts
+   * @throws IOException
+   */
+  void applicationStarted(ApplicationStartData appStart) throws IOException;
+
+  /**
+   * This method writes the information of <code>RMApp</code> that is available
+   * when it finishes.
+   * 
+   * @param appFinish
+   *          the record of the information of <code>RMApp</code> that is
+   *          available when it finishes
+   * @throws IOException
+   */
+  void applicationFinished(ApplicationFinishData appFinish) throws IOException;
+
+  /**
+   * This method writes the information of <code>RMAppAttempt</code> that is
+   * available when it starts.
+   * 
+   * @param appAttemptStart
+   *          the record of the information of <code>RMAppAttempt</code> that is
+   *          available when it starts
+   * @throws IOException
+   */
+  void applicationAttemptStarted(
+      ApplicationAttemptStartData appAttemptStart) throws IOException;
+
+  /**
+   * This method writes the information of <code>RMAppAttempt</code> that is
+   * available when it finishes.
+   * 
+   * @param appAttemptFinish
+   *          the record of the information of <code>RMAppAttempt</code> that is
+   *          available when it finishes
+   * @throws IOException
    */
-  void writeApplication(ApplicationHistoryData app) throws Throwable;
+  void applicationAttemptFinished(
+      ApplicationAttemptFinishData appAttemptFinish) throws IOException;
 
   /**
-   * This method persists an {@link ApplicationAttemptHistoryData} object.
-   * @param appAttempt the {@link ApplicationAttemptHistoryData} object
-   * @throws Throwable
+   * This method writes the information of <code>RMContainer</code> that is
+   * available when it starts.
+   * 
+   * @param containerStart
+   *          the record of the information of <code>RMContainer</code> that is
+   *          available when it starts
+   * @throws IOException
    */
-  void writeApplicationAttempt(
-      ApplicationAttemptHistoryData appAttempt) throws Throwable;
+  void containerStarted(ContainerStartData containerStart) throws IOException;
 
   /**
-   * This method persists a {@link ContainerHistoryData} object.
-   * @param container the {@link ContainerHistoryData} object
-   * @throws Throwable
+   * This method writes the information of <code>RMContainer</code> that is
+   * available when it finishes.
+   * 
+   * @param containerFinish
+   *          the record of the information of <code>RMContainer</code> that is
+   *          available when it finishes
+   * @throws IOException
    */
-  void writeContainer(ContainerHistoryData container) throws Throwable;
+  void containerFinished(ContainerFinishData containerFinish)
+      throws IOException;
 
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java Tue Jan 28 19:50:28 2014
@@ -19,46 +19,55 @@
 package org.apache.hadoop.yarn.server.applicationhistoryservice;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
 
-public class MemoryApplicationHistoryStore implements ApplicationHistoryStore {
+/**
+ * In-memory implementation of {@link ApplicationHistoryStore}.
+ * This implementation is for test purpose only. If users improperly instantiate
+ * it, they may encounter reading and writing history data in different memory
+ * store.
+ * 
+ */
+@Private
+@Unstable
+public class MemoryApplicationHistoryStore extends AbstractService
+    implements ApplicationHistoryStore {
 
-  private static MemoryApplicationHistoryStore memStore = null;
-
-  private ConcurrentHashMap<ApplicationId, ApplicationHistoryData> applicationData =
+  private final ConcurrentMap<ApplicationId, ApplicationHistoryData> applicationData =
       new ConcurrentHashMap<ApplicationId, ApplicationHistoryData>();
-  private ConcurrentHashMap<ApplicationId, ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>> applicationAttemptData =
-      new ConcurrentHashMap<ApplicationId, ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>>();
-  private ConcurrentHashMap<ContainerId, ContainerHistoryData> containerData =
-      new ConcurrentHashMap<ContainerId, ContainerHistoryData>();
+  private final ConcurrentMap<ApplicationId, ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData>> applicationAttemptData =
+      new ConcurrentHashMap<ApplicationId, ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData>>();
+  private final ConcurrentMap<ApplicationAttemptId, ConcurrentMap<ContainerId, ContainerHistoryData>> containerData =
+      new ConcurrentHashMap<ApplicationAttemptId, ConcurrentMap<ContainerId, ContainerHistoryData>>();
 
-  private MemoryApplicationHistoryStore() {
-  }
-
-  public static MemoryApplicationHistoryStore getMemoryStore() {
-    if (memStore == null) {
-      memStore = new MemoryApplicationHistoryStore();
-    }
-    return memStore;
+  public MemoryApplicationHistoryStore() {
+    super(MemoryApplicationHistoryStore.class.getName());
   }
 
   @Override
   public Map<ApplicationId, ApplicationHistoryData> getAllApplications() {
-    Map<ApplicationId, ApplicationHistoryData> listApps =
-        new HashMap<ApplicationId, ApplicationHistoryData>();
-    for (ApplicationId appId : applicationData.keySet()) {
-      listApps.put(appId, applicationData.get(appId));
-    }
-    return listApps;
+    return new HashMap<ApplicationId, ApplicationHistoryData>(
+        applicationData);
   }
 
   @Override
@@ -67,111 +76,209 @@ public class MemoryApplicationHistorySto
   }
 
   @Override
-  public Map<ApplicationAttemptId, ApplicationAttemptHistoryData> getApplicationAttempts(
-      ApplicationId appId) {
-    Map<ApplicationAttemptId, ApplicationAttemptHistoryData> listAttempts =
-        null;
-    ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
+  public Map<ApplicationAttemptId, ApplicationAttemptHistoryData>
+      getApplicationAttempts(
+          ApplicationId appId) {
+    ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
         applicationAttemptData.get(appId);
-    if (appAttempts != null) {
-      listAttempts =
-          new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
-      for (ApplicationAttemptId attemptId : appAttempts.keySet()) {
-        listAttempts.put(attemptId, appAttempts.get(attemptId));
-      }
+    if (subMap == null) {
+      return Collections.<ApplicationAttemptId, ApplicationAttemptHistoryData>emptyMap();
+    } else {
+      return new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>(subMap);
     }
-    return listAttempts;
   }
 
   @Override
   public ApplicationAttemptHistoryData getApplicationAttempt(
       ApplicationAttemptId appAttemptId) {
-    ApplicationAttemptHistoryData appAttemptHistoryData = null;
-    ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
+    ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
         applicationAttemptData.get(appAttemptId.getApplicationId());
-    if (appAttempts != null) {
-      appAttemptHistoryData = appAttempts.get(appAttemptId);
+    if (subMap == null) {
+      return null;
+    } else {
+      return subMap.get(appAttemptId);
     }
-    return appAttemptHistoryData;
   }
 
   @Override
-  public ContainerHistoryData getAMContainer(ApplicationAttemptId appAttemptId) {
-    ContainerHistoryData Container = null;
-    ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
-        applicationAttemptData.get(appAttemptId.getApplicationId());
-    if (appAttempts != null) {
-      containerData.get(appAttempts.get(appAttemptId).getMasterContainerId());
+  public ContainerHistoryData getAMContainer(
+      ApplicationAttemptId appAttemptId) {
+    ApplicationAttemptHistoryData appAttempt =
+        getApplicationAttempt(appAttemptId);
+    if (appAttempt == null || appAttempt.getMasterContainerId() == null) {
+      return null;
+    } else {
+      return getContainer(appAttempt.getMasterContainerId());
     }
-    return Container;
   }
 
   @Override
   public ContainerHistoryData getContainer(ContainerId containerId) {
-    return containerData.get(containerId);
+    Map<ContainerId, ContainerHistoryData> subMap =
+        containerData.get(containerId.getApplicationAttemptId());
+    if (subMap == null) {
+      return null;
+    } else {
+      return subMap.get(containerId);
+    }
   }
 
   @Override
-  public void writeApplication(ApplicationHistoryData app) throws Throwable {
-    if (app != null) {
-      ApplicationHistoryData oldData =
-          applicationData.putIfAbsent(app.getApplicationId(), app);
-      if (oldData != null) {
-        throw new IOException("This application "
-            + app.getApplicationId().toString() + " is already present.");
-      }
+  public Map<ContainerId, ContainerHistoryData> getContainers(
+      ApplicationAttemptId appAttemptId) throws IOException {
+    ConcurrentMap<ContainerId, ContainerHistoryData> subMap =
+        containerData.get(appAttemptId);
+    if (subMap == null) {
+      return Collections.<ContainerId, ContainerHistoryData>emptyMap();
+    } else {
+      return new HashMap<ContainerId, ContainerHistoryData>(subMap);
     }
   }
 
   @Override
-  public void writeApplicationAttempt(ApplicationAttemptHistoryData appAttempt)
-      throws Throwable {
-    if (appAttempt != null) {
-      if (applicationAttemptData.containsKey(appAttempt
-        .getApplicationAttemptId().getApplicationId())) {
-        ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttemptmap =
-            applicationAttemptData.get(appAttempt.getApplicationAttemptId()
-              .getApplicationId());
-        ApplicationAttemptHistoryData oldAppAttempt =
-            appAttemptmap.putIfAbsent(appAttempt.getApplicationAttemptId(),
-              appAttempt);
-        if (oldAppAttempt != null) {
-          throw new IOException("This application attempt "
-              + appAttempt.getApplicationAttemptId().toString()
-              + " already present.");
-        }
-      } else {
-        ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttemptmap =
-            new ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
-        appAttemptmap.put(appAttempt.getApplicationAttemptId(), appAttempt);
-        applicationAttemptData.putIfAbsent(appAttempt.getApplicationAttemptId()
-          .getApplicationId(), appAttemptmap);
-      }
+  public void applicationStarted(ApplicationStartData appStart)
+      throws IOException {
+    ApplicationHistoryData oldData =
+        applicationData.putIfAbsent(appStart.getApplicationId(),
+            ApplicationHistoryData.newInstance(
+                appStart.getApplicationId(),
+                appStart.getApplicationName(),
+                appStart.getApplicationType(),
+                appStart.getQueue(),
+                appStart.getUser(),
+                appStart.getSubmitTime(),
+                appStart.getStartTime(),
+                Long.MAX_VALUE, null, null, null));
+    if (oldData != null) {
+      throw new IOException("The start information of application "
+          + appStart.getApplicationId() + " is already stored.");
     }
   }
 
   @Override
-  public void writeContainer(ContainerHistoryData container) throws Throwable {
-    if (container != null) {
-      ContainerHistoryData oldContainer =
-          containerData.putIfAbsent(container.getContainerId(), container);
-      if (oldContainer != null) {
-        throw new IOException("This container "
-            + container.getContainerId().toString() + " is already present.");
-      }
+  public void applicationFinished(ApplicationFinishData appFinish)
+      throws IOException {
+    ApplicationHistoryData data =
+        applicationData.get(appFinish.getApplicationId());
+    if (data == null) {
+      throw new IOException("The finish information of application "
+          + appFinish.getApplicationId() + " is stored before the start"
+          + " information.");
+    }
+    // Make the assumption that YarnApplicationState should not be null if
+    // the finish information is already recorded
+    if (data.getYarnApplicationState() != null) {
+      throw new IOException("The finish information of application "
+          + appFinish.getApplicationId() + " is already stored.");
     }
+    data.setFinishTime(appFinish.getFinishTime());
+    data.setDiagnosticsInfo(appFinish.getDiagnosticsInfo());
+    data.setFinalApplicationStatus(appFinish.getFinalApplicationStatus());
+    data.setYarnApplicationState(appFinish.getYarnApplicationState());
   }
 
   @Override
-  public Map<ContainerId, ContainerHistoryData> getContainers(
-      ApplicationAttemptId appAttemptId) throws IOException {
-    HashMap<ContainerId, ContainerHistoryData> containers =
-        new HashMap<ContainerId, ContainerHistoryData>();
-    for (ContainerId container : containerData.keySet()) {
-      if (container.getApplicationAttemptId().equals(appAttemptId)) {
-        containers.put(container, containerData.get(container));
-      }
+  public void applicationAttemptStarted(
+      ApplicationAttemptStartData appAttemptStart) throws IOException {
+    ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
+        getSubMap(appAttemptStart.getApplicationAttemptId().getApplicationId());
+    ApplicationAttemptHistoryData oldData = subMap.putIfAbsent(
+        appAttemptStart.getApplicationAttemptId(),
+        ApplicationAttemptHistoryData.newInstance(
+            appAttemptStart.getApplicationAttemptId(),
+            appAttemptStart.getHost(),
+            appAttemptStart.getRPCPort(),
+            appAttemptStart.getMasterContainerId(),
+            null, null, null, null));
+    if (oldData != null) {
+      throw new IOException("The start information of application attempt "
+          + appAttemptStart.getApplicationAttemptId()
+          + " is already stored.");
     }
-    return containers;
   }
+
+  @Override
+  public void applicationAttemptFinished(
+      ApplicationAttemptFinishData appAttemptFinish) throws IOException {
+    ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
+        getSubMap(appAttemptFinish.getApplicationAttemptId().getApplicationId());
+    ApplicationAttemptHistoryData data =
+        subMap.get(appAttemptFinish.getApplicationAttemptId());
+    if (data == null) {
+      throw new IOException("The finish information of application attempt "
+          + appAttemptFinish.getApplicationAttemptId() + " is stored before"
+          + " the start information.");
+    }
+    // Make the assumption that YarnApplicationAttemptState should not be null
+    // if the finish information is already recorded
+    if (data.getYarnApplicationAttemptState() != null) {
+      throw new IOException("The finish information of application attempt "
+          + appAttemptFinish.getApplicationAttemptId()
+          + " is already stored.");
+    }
+    data.setTrackingURL(appAttemptFinish.getTrackingURL());
+    data.setDiagnosticsInfo(appAttemptFinish.getDiagnosticsInfo());
+    data.setFinalApplicationStatus(appAttemptFinish.getFinalApplicationStatus());
+    data.setYarnApplicationAttemptState(appAttemptFinish.getYarnApplicationAttemptState());
+  }
+
+  private ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData>
+      getSubMap(ApplicationId appId) {
+    applicationAttemptData.putIfAbsent(appId,
+        new ConcurrentHashMap<ApplicationAttemptId,
+        ApplicationAttemptHistoryData>());
+    return applicationAttemptData.get(appId);
+  }
+
+  @Override
+  public void containerStarted(ContainerStartData containerStart)
+      throws IOException {
+    ConcurrentMap<ContainerId, ContainerHistoryData> subMap =
+        getSubMap(containerStart.getContainerId().getApplicationAttemptId());
+    ContainerHistoryData oldData = subMap.putIfAbsent(
+        containerStart.getContainerId(),
+        ContainerHistoryData.newInstance(
+            containerStart.getContainerId(),
+            containerStart.getAllocatedResource(),
+            containerStart.getAssignedNode(),
+            containerStart.getPriority(),
+            containerStart.getStartTime(),
+            Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null));
+    if (oldData != null) {
+      throw new IOException("The start information of container "
+          + containerStart.getContainerId() + " is already stored.");
+    }
+  }
+
+  @Override
+  public void containerFinished(ContainerFinishData containerFinish)
+      throws IOException {
+    ConcurrentMap<ContainerId, ContainerHistoryData> subMap =
+        getSubMap(containerFinish.getContainerId().getApplicationAttemptId());
+    ContainerHistoryData data = subMap.get(containerFinish.getContainerId());
+    if (data == null) {
+      throw new IOException("The finish information of container "
+          + containerFinish.getContainerId() + " is stored before"
+          + " the start information.");
+    }
+    // Make the assumption that ContainerState should not be null if
+    // the finish information is already recorded
+    if (data.getContainerState() != null) {
+      throw new IOException("The finish information of container "
+          + containerFinish.getContainerId() + " is already stored.");
+    }
+    data.setFinishTime(containerFinish.getFinishTime());
+    data.setDiagnosticsInfo(containerFinish.getDiagnosticsInfo());
+    data.setLogURL(containerFinish.getLogURL());
+    data.setContainerExitStatus(containerFinish.getContainerExitStatus());
+    data.setContainerState(containerFinish.getContainerState());
+  }
+
+  private ConcurrentMap<ContainerId, ContainerHistoryData> getSubMap(
+      ApplicationAttemptId appAttemptId) {
+    containerData.putIfAbsent(appAttemptId,
+        new ConcurrentHashMap<ContainerId, ContainerHistoryData>());
+    return containerData.get(appAttemptId);
+  }
+
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java Tue Jan 28 19:50:28 2014
@@ -23,69 +23,150 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 
 /**
- * The class contains all the fields that need to be stored persistently for
+ * The class contains all the fields that are stored persistently for
  * <code>RMAppAttempt</code>.
  */
 @Public
 @Unstable
-public interface ApplicationAttemptHistoryData {
+public class ApplicationAttemptHistoryData {
+
+  private ApplicationAttemptId applicationAttemptId;
+
+  private String host;
+
+  private int rpcPort;
+
+  private String trackingURL;
+
+  private String diagnosticsInfo;
+
+  private FinalApplicationStatus finalApplicationStatus;
+
+  private ContainerId masterContainerId;
+
+  private YarnApplicationAttemptState yarnApplicationAttemptState;
+
+  @Public
+  @Unstable
+  public static ApplicationAttemptHistoryData newInstance(
+      ApplicationAttemptId appAttemptId, String host, int rpcPort,
+      ContainerId masterContainerId, String diagnosticsInfo,
+      String trackingURL,
+      FinalApplicationStatus finalApplicationStatus,
+      YarnApplicationAttemptState yarnApplicationAttemptState) {
+    ApplicationAttemptHistoryData appAttemptHD =
+        new ApplicationAttemptHistoryData();
+    appAttemptHD.setApplicationAttemptId(appAttemptId);
+    appAttemptHD.setHost(host);
+    appAttemptHD.setRPCPort(rpcPort);
+    appAttemptHD.setMasterContainerId(masterContainerId);
+    appAttemptHD.setDiagnosticsInfo(diagnosticsInfo);
+    appAttemptHD.setTrackingURL(trackingURL);
+    appAttemptHD.setFinalApplicationStatus(finalApplicationStatus);
+    appAttemptHD.setYarnApplicationAttemptState(yarnApplicationAttemptState);
+    return appAttemptHD;
+  }
+
+  @Public
+  @Unstable
+  public ApplicationAttemptId getApplicationAttemptId() {
+    return applicationAttemptId;
+  }
+
+  @Public
+  @Unstable
+  public void setApplicationAttemptId(
+      ApplicationAttemptId applicationAttemptId) {
+    this.applicationAttemptId = applicationAttemptId;
+  }
 
   @Public
   @Unstable
-  ApplicationAttemptId getApplicationAttemptId();
+  public String getHost() {
+    return host;
+  }
 
   @Public
   @Unstable
-  void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId);
+  public void setHost(String host) {
+    this.host = host;
+  }
 
   @Public
   @Unstable
-  String getHost();
+  public int getRPCPort() {
+    return rpcPort;
+  }
 
   @Public
   @Unstable
-  void setHost(String host);
+  public void setRPCPort(int rpcPort) {
+    this.rpcPort = rpcPort;
+  }
 
   @Public
   @Unstable
-  int getRPCPort();
+  public String getTrackingURL() {
+    return trackingURL;
+  }
 
   @Public
   @Unstable
-  void setRPCPort(int rpcPort);
+  public void setTrackingURL(String trackingURL) {
+    this.trackingURL = trackingURL;
+  }
 
   @Public
   @Unstable
-  String getTrackingURL();
+  public String getDiagnosticsInfo() {
+    return diagnosticsInfo;
+  }
 
   @Public
   @Unstable
-  void setTrackingURL(String trackingURL);
+  public void setDiagnosticsInfo(String diagnosticsInfo) {
+    this.diagnosticsInfo = diagnosticsInfo;
+  }
 
   @Public
   @Unstable
-  String getDiagnosticsInfo();
+  public FinalApplicationStatus getFinalApplicationStatus() {
+    return finalApplicationStatus;
+  }
 
   @Public
   @Unstable
-  void setDiagnosticsInfo(String diagnosticsInfo);
+  public void setFinalApplicationStatus(
+      FinalApplicationStatus finalApplicationStatus) {
+    this.finalApplicationStatus = finalApplicationStatus;
+  }
 
   @Public
   @Unstable
-  FinalApplicationStatus getFinalApplicationStatus();
+  public ContainerId getMasterContainerId() {
+    return masterContainerId;
+  }
 
   @Public
   @Unstable
-  void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus);
+  public void setMasterContainerId(ContainerId masterContainerId) {
+    this.masterContainerId = masterContainerId;
+  }
 
   @Public
   @Unstable
-  ContainerId getMasterContainerId();
+  public YarnApplicationAttemptState getYarnApplicationAttemptState() {
+    return yarnApplicationAttemptState;
+  }
 
   @Public
   @Unstable
-  void setMasterContainerId(ContainerId masterContainerId);
+  public void setYarnApplicationAttemptState(
+      YarnApplicationAttemptState yarnApplicationAttemptState) {
+    this.yarnApplicationAttemptState = yarnApplicationAttemptState;
+  }
 
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java Tue Jan 28 19:50:28 2014
@@ -22,93 +22,192 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 
 /**
- * The class contains all the fields that need to be stored persistently for
+ * The class contains all the fields that are stored persistently for
  * <code>RMApp</code>.
  */
 @Public
 @Unstable
-public interface ApplicationHistoryData {
+public class ApplicationHistoryData {
+
+  private ApplicationId applicationId;
+
+  private String applicationName;
+
+  private String applicationType;
+
+  private String user;
+
+  private String queue;
+
+  private long submitTime;
+
+  private long startTime;
+
+  private long finishTime;
+
+  private String diagnosticsInfo;
+
+  private FinalApplicationStatus finalApplicationStatus;
+
+  private YarnApplicationState yarnApplicationState;
+
+  @Public
+  @Unstable
+  public static ApplicationHistoryData newInstance(ApplicationId applicationId,
+      String applicationName, String applicationType, String queue,
+      String user, long submitTime, long startTime, long finishTime,
+      String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus,
+      YarnApplicationState yarnApplicationState) {
+    ApplicationHistoryData appHD = new ApplicationHistoryData();
+    appHD.setApplicationId(applicationId);
+    appHD.setApplicationName(applicationName);
+    appHD.setApplicationType(applicationType);
+    appHD.setQueue(queue);
+    appHD.setUser(user);
+    appHD.setSubmitTime(submitTime);
+    appHD.setStartTime(startTime);
+    appHD.setFinishTime(finishTime);
+    appHD.setDiagnosticsInfo(diagnosticsInfo);
+    appHD.setFinalApplicationStatus(finalApplicationStatus);
+    appHD.setYarnApplicationState(yarnApplicationState);
+    return appHD;
+  }
+
+  @Public
+  @Unstable
+  public ApplicationId getApplicationId() {
+    return applicationId;
+  }
+
+  @Public
+  @Unstable
+  public void setApplicationId(ApplicationId applicationId) {
+    this.applicationId = applicationId;
+  }
 
   @Public
   @Unstable
-  ApplicationId getApplicationId();
+  public String getApplicationName() {
+    return applicationName;
+  }
 
   @Public
   @Unstable
-  void setApplicationId(ApplicationId applicationId);
+  public void setApplicationName(String applicationName) {
+    this.applicationName = applicationName;
+  }
 
   @Public
   @Unstable
-  String getApplicationName();
+  public String getApplicationType() {
+    return applicationType;
+  }
 
   @Public
   @Unstable
-  void setApplicationName(String applicationName);
+  public void setApplicationType(String applicationType) {
+    this.applicationType = applicationType;
+  }
 
   @Public
   @Unstable
-  String getApplicationType();
+  public String getUser() {
+    return user;
+  }
 
   @Public
   @Unstable
-  void setApplicationType(String applicationType);
+  public void setUser(String user) {
+    this.user = user;
+  }
 
   @Public
   @Unstable
-  String getUser();
+  public String getQueue() {
+    return queue;
+  }
 
   @Public
   @Unstable
-  void setUser(String user);
+  public void setQueue(String queue) {
+    this.queue = queue;
+  }
 
   @Public
   @Unstable
-  String getQueue();
+  public long getSubmitTime() {
+    return submitTime;
+  }
 
   @Public
   @Unstable
-  void setQueue(String queue);
+  public void setSubmitTime(long submitTime) {
+    this.submitTime = submitTime;
+  }
 
   @Public
   @Unstable
-  long getSubmitTime();
+  public long getStartTime() {
+    return startTime;
+  }
 
   @Public
   @Unstable
-  void setSubmitTime(long submitTime);
+  public void setStartTime(long startTime) {
+    this.startTime = startTime;
+  }
 
   @Public
   @Unstable
-  long getStartTime();
+  public long getFinishTime() {
+    return finishTime;
+  }
 
   @Public
   @Unstable
-  void setStartTime(long startTime);
+  public void setFinishTime(long finishTime) {
+    this.finishTime = finishTime;
+  }
 
   @Public
   @Unstable
-  long getFinishTime();
+  public String getDiagnosticsInfo() {
+    return diagnosticsInfo;
+  }
 
   @Public
   @Unstable
-  void setFinishTime(long finishTime);
+  public void setDiagnosticsInfo(String diagnosticsInfo) {
+    this.diagnosticsInfo = diagnosticsInfo;
+  }
 
   @Public
   @Unstable
-  String getDiagnosticsInfo();
+  public FinalApplicationStatus getFinalApplicationStatus() {
+    return finalApplicationStatus;
+  }
 
   @Public
   @Unstable
-  void setDiagnosticsInfo(String diagnosticInfo);
+  public void setFinalApplicationStatus(
+      FinalApplicationStatus finalApplicationStatus) {
+    this.finalApplicationStatus = finalApplicationStatus;
+  }
 
   @Public
   @Unstable
-  FinalApplicationStatus getFinalApplicationStatus();
+  public YarnApplicationState getYarnApplicationState() {
+    return this.yarnApplicationState;
+  }
 
   @Public
   @Unstable
-  void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus);
+  public void setYarnApplicationState(
+      YarnApplicationState yarnApplicationState) {
+    this.yarnApplicationState = yarnApplicationState;
+  }
 
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java Tue Jan 28 19:50:28 2014
@@ -27,83 +27,171 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Resource;
 
 /**
- * The class contains all the fields that need to be stored persistently for
+ * The class contains all the fields that are stored persistently for
  * <code>RMContainer</code>.
  */
 @Public
 @Unstable
-public interface ContainerHistoryData {
+public class ContainerHistoryData {
+
+  private ContainerId containerId;
+
+  private Resource allocatedResource;
+
+  private NodeId assignedNode;
+
+  private Priority priority;
+
+  private long startTime;
+
+  private long finishTime;
+
+  private String diagnosticsInfo;
+
+  private String logURL;
+
+  private int containerExitStatus;
+
+  private ContainerState containerState;
+
+  @Public
+  @Unstable
+  public static ContainerHistoryData newInstance(ContainerId containerId,
+      Resource allocatedResource, NodeId assignedNode, Priority priority,
+      long startTime, long finishTime, String diagnosticsInfo, String logURL,
+      int containerExitCode, ContainerState containerState) {
+    ContainerHistoryData containerHD = new ContainerHistoryData();
+    containerHD.setContainerId(containerId);
+    containerHD.setAllocatedResource(allocatedResource);
+    containerHD.setAssignedNode(assignedNode);
+    containerHD.setPriority(priority);
+    containerHD.setStartTime(startTime);
+    containerHD.setFinishTime(finishTime);
+    containerHD.setDiagnosticsInfo(diagnosticsInfo);
+    containerHD.setLogURL(logURL);
+    containerHD.setContainerExitStatus(containerExitCode);
+    containerHD.setContainerState(containerState);
+    return containerHD;
+  }
+
+  @Public
+  @Unstable
+  public ContainerId getContainerId() {
+    return containerId;
+  }
+
+  @Public
+  @Unstable
+  public void setContainerId(ContainerId containerId) {
+    this.containerId = containerId;
+  }
 
   @Public
   @Unstable
-  ContainerId getContainerId();
+  public Resource getAllocatedResource() {
+    return allocatedResource;
+  }
 
   @Public
   @Unstable
-  void setContainerId(ContainerId containerId);
+  public void setAllocatedResource(Resource resource) {
+    this.allocatedResource = resource;
+  }
 
   @Public
   @Unstable
-  Resource getAllocatedResource();
+  public NodeId getAssignedNode() {
+    return assignedNode;
+  }
 
   @Public
   @Unstable
-  void setAllocatedResource(Resource resource);
+  public void setAssignedNode(NodeId nodeId) {
+    this.assignedNode = nodeId;
+  }
 
   @Public
   @Unstable
-  NodeId getAssignedNode();
+  public Priority getPriority() {
+    return priority;
+  }
 
   @Public
   @Unstable
-  void setAssignedNode(NodeId nodeId);
+  public void setPriority(Priority priority) {
+    this.priority = priority;
+  }
 
   @Public
   @Unstable
-  Priority getPriority();
+  public long getStartTime() {
+    return startTime;
+  }
 
   @Public
   @Unstable
-  void setPriority(Priority priority);
+  public void setStartTime(long startTime) {
+    this.startTime = startTime;
+  }
 
   @Public
   @Unstable
-  long getStartTime();
+  public long getFinishTime() {
+    return finishTime;
+  }
 
   @Public
   @Unstable
-  void setStartTime(long startTime);
+  public void setFinishTime(long finishTime) {
+    this.finishTime = finishTime;
+  }
 
   @Public
   @Unstable
-  long getFinishTime();
+  public String getDiagnosticsInfo() {
+    return diagnosticsInfo;
+  }
 
   @Public
   @Unstable
-  void setFinishTime(long finishTime);
+  public void setDiagnosticsInfo(String diagnosticsInfo) {
+    this.diagnosticsInfo = diagnosticsInfo;
+  }
 
   @Public
   @Unstable
-  String getDiagnosticsInfo();
+  public String getLogURL() {
+    return logURL;
+  }
 
   @Public
   @Unstable
-  void setDiagnosticsInfo(String diagnosticInfo);
+  public void setLogURL(String logURL) {
+    this.logURL = logURL;
+  }
 
   @Public
   @Unstable
-  String getLogURL();
+  public int getContainerExitStatus() {
+    return containerExitStatus;
+  }
 
   @Public
   @Unstable
-  void setLogURL(String logURL);
+  public void setContainerExitStatus(int containerExitStatus) {
+    this.containerExitStatus = containerExitStatus;
+  }
 
   @Public
   @Unstable
-  ContainerState getFinalContainerStatus();
+  public ContainerState getContainerState() {
+    return containerState;
+  }
 
   @Public
   @Unstable
-  void setFinalContainerStatus(ContainerState finalContainerState);
+  public void setContainerState(ContainerState containerState) {
+    this.containerState = containerState;
+  }
 
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java Tue Jan 28 19:50:28 2014
@@ -19,102 +19,189 @@
 package org.apache.hadoop.yarn.server.applicationhistoryservice;
 
 import java.io.IOException;
-import java.util.HashMap;
 
 import junit.framework.Assert;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationAttemptHistoryDataPBImpl;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationHistoryDataPBImpl;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ContainerHistoryDataPBImpl;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-public class TestMemoryApplicationHistoryStore {
-  MemoryApplicationHistoryStore memstore = null;
+public class TestMemoryApplicationHistoryStore extends
+    ApplicationHistoryStoreTestUtils {
 
   @Before
-  public void setup() throws Throwable {
-    memstore = MemoryApplicationHistoryStore.getMemoryStore();
-    writeHistoryApplication();
-    writeHistoryApplicationAttempt();
-    writeContainer();
+  public void setup() {
+    store = new MemoryApplicationHistoryStore();
   }
 
-  public void writeHistoryApplication() throws Throwable {
-    ApplicationHistoryData appData = new ApplicationHistoryDataPBImpl();
-    appData.setApplicationId(ApplicationId.newInstance(1234, 1));
-    memstore.writeApplication(appData);
-  }
-
-  public void writeHistoryApplicationAttempt() throws Throwable {
-    ApplicationAttemptHistoryData appAttemptHistoryData =
-        new ApplicationAttemptHistoryDataPBImpl();
-    appAttemptHistoryData.setApplicationAttemptId(ApplicationAttemptId
-      .newInstance(ApplicationId.newInstance(1234, 1), 1));
-    memstore.writeApplicationAttempt(appAttemptHistoryData);
+  @Test
+  public void testReadWriteApplicationHistory() throws Exception {
+    // Out of order
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    try {
+      writeApplicationFinishData(appId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains(
+          "is stored before the start information"));
+    }
+    // Normal
+    int numApps = 5;
+    for (int i = 1; i <= numApps; ++i) {
+      appId = ApplicationId.newInstance(0, i);
+      writeApplicationStartData(appId);
+      writeApplicationFinishData(appId);
+    }
+    Assert.assertEquals(numApps, store.getAllApplications().size());
+    for (int i = 1; i <= numApps; ++i) {
+      appId = ApplicationId.newInstance(0, i);
+      ApplicationHistoryData data = store.getApplication(appId);
+      Assert.assertNotNull(data);
+      Assert.assertEquals(appId.toString(), data.getApplicationName());
+      Assert.assertEquals(appId.toString(), data.getDiagnosticsInfo());
+    }
+    // Write again
+    appId = ApplicationId.newInstance(0, 1);
+    try {
+      writeApplicationStartData(appId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
+    try {
+      writeApplicationFinishData(appId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
   }
 
-  public void writeContainer() throws Throwable {
-    ContainerHistoryData container = new ContainerHistoryDataPBImpl();
-    container.setContainerId(ContainerId.newInstance(ApplicationAttemptId
-      .newInstance(ApplicationId.newInstance(1234, 1), 1), 1));
-    memstore.writeContainer(container);
+  @Test
+  public void testReadWriteApplicationAttemptHistory() throws Exception {
+    // Out of order
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    try {
+      writeApplicationAttemptFinishData(appAttemptId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains(
+          "is stored before the start information"));
+    }
+    // Normal
+    int numAppAttempts = 5;
+    writeApplicationStartData(appId);
+    for (int i = 1; i <= numAppAttempts; ++i) {
+      appAttemptId =
+          ApplicationAttemptId.newInstance(appId, i);
+      writeApplicationAttemptStartData(appAttemptId);
+      writeApplicationAttemptFinishData(appAttemptId);
+    }
+    Assert.assertEquals(
+        numAppAttempts, store.getApplicationAttempts(appId).size());
+    for (int i = 1; i <= numAppAttempts; ++i) {
+      appAttemptId =
+          ApplicationAttemptId.newInstance(appId, i);
+      ApplicationAttemptHistoryData data =
+          store.getApplicationAttempt(appAttemptId);
+      Assert.assertNotNull(data);
+      Assert.assertEquals(appAttemptId.toString(), data.getHost());
+      Assert.assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo());
+    }
+    writeApplicationFinishData(appId);
+    // Write again
+    appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    try {
+      writeApplicationAttemptStartData(appAttemptId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
+    try {
+      writeApplicationAttemptFinishData(appAttemptId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
   }
 
-  public ContainerHistoryData writeContainer(ApplicationAttemptId appAttemptId,
-      int containerId) throws Throwable {
-    ContainerHistoryData container = new ContainerHistoryDataPBImpl();
-    container
-      .setContainerId(ContainerId.newInstance(appAttemptId, containerId));
-    memstore.writeContainer(container);
-    return container;
-  }
-  
-  @After
-  public void tearDown() {
+  @Test
+  public void testReadWriteContainerHistory() throws Exception {
+    // Out of order
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
+    try {
+      writeContainerFinishData(containerId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains(
+          "is stored before the start information"));
+    }
+    // Normal
+    writeApplicationAttemptStartData(appAttemptId);
+    int numContainers = 5;
+    for (int i = 1; i <= numContainers; ++i) {
+      containerId = ContainerId.newInstance(appAttemptId, i);
+      writeContainerStartData(containerId);
+      writeContainerFinishData(containerId);
+    }
+    Assert.assertEquals(
+        numContainers, store.getContainers(appAttemptId).size());
+    for (int i = 1; i <= numContainers; ++i) {
+      containerId = ContainerId.newInstance(appAttemptId, i);
+      ContainerHistoryData data = store.getContainer(containerId);
+      Assert.assertNotNull(data);
+      Assert.assertEquals(Priority.newInstance(containerId.getId()),
+          data.getPriority());
+      Assert.assertEquals(containerId.toString(), data.getDiagnosticsInfo());
+    }
+    ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId);
+    Assert.assertNotNull(masterContainer);
+    Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1),
+        masterContainer.getContainerId());
+    writeApplicationAttemptFinishData(appAttemptId);
+    // Write again
+    containerId = ContainerId.newInstance(appAttemptId, 1);
+    try {
+      writeContainerStartData(containerId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
+    try {
+      writeContainerFinishData(containerId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
   }
 
   @Test
-  public void testReadApplication() throws Throwable {
-    HashMap<ApplicationId, ApplicationHistoryData> map =
-        (HashMap<ApplicationId, ApplicationHistoryData>) memstore
-          .getAllApplications();
-    Assert.assertEquals(1, map.size());
-    ApplicationHistoryData appData = null;
-    for (ApplicationId appId : map.keySet()) {
-      appData = map.get(appId);
-      Assert.assertEquals("application_1234_0001", appData.getApplicationId()
-        .toString());
-    }
-    HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
-        (HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>) memstore
-          .getApplicationAttempts(appData.getApplicationId());
-    Assert.assertEquals(1, appAttempts.size());
-    ApplicationAttemptHistoryData appAttempt = null;
-    for (ApplicationAttemptId appAttemptId : appAttempts.keySet()) {
-      appAttempt = appAttempts.get(appAttemptId);
-      Assert.assertEquals("appattempt_1234_0001_000001", appAttempt
-        .getApplicationAttemptId().toString());
-    }
-    ContainerHistoryData amContainer =
-        memstore.getContainer(ContainerId.newInstance(ApplicationAttemptId
-          .newInstance(ApplicationId.newInstance(1234, 1), 1), 1));
-    Assert.assertEquals("container_1234_0001_01_000001", amContainer
-      .getContainerId().toString());
-    ContainerHistoryData container2 =
-        writeContainer(appAttempt.getApplicationAttemptId(), 2);
-    HashMap<ContainerId, ContainerHistoryData> containers =
-        (HashMap<ContainerId, ContainerHistoryData>) memstore
-          .getContainers(appAttempt.getApplicationAttemptId());
-    Assert.assertEquals(2, containers.size());
-    Assert.assertEquals("container_1234_0001_01_000002", containers.get(
-      container2.getContainerId()).getContainerId().toString());
+  public void testMassiveWriteContainerHistory() throws IOException {
+    long mb = 1024 * 1024;
+    Runtime runtime = Runtime.getRuntime();
+    long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb;
+    int numContainers = 100000;
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    for (int i = 1; i <= numContainers; ++i) {
+      ContainerId containerId = ContainerId.newInstance(appAttemptId, i);
+      writeContainerStartData(containerId);
+      writeContainerFinishData(containerId);
+    }
+    long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
+    Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 100);
   }
+
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java Tue Jan 28 19:50:28 2014
@@ -33,9 +33,14 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 
@@ -43,6 +48,7 @@ import org.apache.hadoop.yarn.server.res
  * Utility methods to aid serving RM data through the REST and RPC APIs
  */
 public class RMServerUtils {
+
   public static List<RMNode> queryRMNodes(RMContext context,
       EnumSet<NodeState> acceptedStates) {
     // nodes contains nodes that are NEW, RUNNING OR UNHEALTHY
@@ -56,7 +62,7 @@ public class RMServerUtils {
         }
       }
     }
-    
+
     // inactiveNodes contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED
     if (acceptedStates.contains(NodeState.DECOMMISSIONED) ||
         acceptedStates.contains(NodeState.LOST) ||
@@ -69,7 +75,7 @@ public class RMServerUtils {
     }
     return results;
   }
-  
+
   /**
    * Utility method to validate a list resource requests, by insuring that the
    * requested memory/vcore is non-negative and not greater than max
@@ -85,8 +91,9 @@ public class RMServerUtils {
    * @throw <code>InvalidResourceBlacklistRequestException </code> if the
    * resource is not able to be added to the blacklist.
    */
-  public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest) 
-  throws InvalidResourceBlacklistRequestException {
+  public static void validateBlacklistRequest(
+      ResourceBlacklistRequest blacklistRequest)
+      throws InvalidResourceBlacklistRequestException {
     if (blacklistRequest != null) {
       List<String> plus = blacklistRequest.getBlacklistAdditions();
       if (plus != null && plus.contains(ResourceRequest.ANY)) {
@@ -100,10 +107,12 @@ public class RMServerUtils {
    * It will validate to make sure all the containers belong to correct
    * application attempt id. If not then it will throw
    * {@link InvalidContainerReleaseException}
-   * @param containerReleaseList containers to be released as requested by
-   * application master.
-   * @param appAttemptId Application attempt Id
-   * @throws InvalidContainerReleaseException 
+   * 
+   * @param containerReleaseList
+   *          containers to be released as requested by application master.
+   * @param appAttemptId
+   *          Application attempt Id
+   * @throws InvalidContainerReleaseException
    */
   public static void
       validateContainerReleaseRequest(List<ContainerId> containerReleaseList,
@@ -111,9 +120,11 @@ public class RMServerUtils {
           throws InvalidContainerReleaseException {
     for (ContainerId cId : containerReleaseList) {
       if (!appAttemptId.equals(cId.getApplicationAttemptId())) {
-        throw new InvalidContainerReleaseException("Cannot release container : "
-            + cId.toString() + " not belonging to this application attempt : "
-            + appAttemptId);
+        throw new InvalidContainerReleaseException(
+            "Cannot release container : "
+                + cId.toString()
+                + " not belonging to this application attempt : "
+                + appAttemptId);
       }
     }
   }
@@ -157,4 +168,63 @@ public class RMServerUtils {
     }
     return user;
   }
+
+  public static YarnApplicationState createApplicationState(
+      RMAppState rmAppState) {
+    switch (rmAppState) {
+      case NEW:
+        return YarnApplicationState.NEW;
+      case NEW_SAVING:
+        return YarnApplicationState.NEW_SAVING;
+      case SUBMITTED:
+        return YarnApplicationState.SUBMITTED;
+      case ACCEPTED:
+        return YarnApplicationState.ACCEPTED;
+      case RUNNING:
+        return YarnApplicationState.RUNNING;
+      case FINISHING:
+      case FINISHED:
+        return YarnApplicationState.FINISHED;
+      case KILLED:
+        return YarnApplicationState.KILLED;
+      case FAILED:
+        return YarnApplicationState.FAILED;
+      default:
+        throw new YarnRuntimeException("Unknown state passed!");
+      }
+  }
+
+  public static YarnApplicationAttemptState createApplicationAttemptState(
+      RMAppAttemptState rmAppAttemptState) {
+    switch (rmAppAttemptState) {
+      case NEW:
+        return YarnApplicationAttemptState.NEW;
+      case SUBMITTED:
+        return YarnApplicationAttemptState.SUBMITTED;
+      case SCHEDULED:
+        return YarnApplicationAttemptState.SCHEDULED;
+      case ALLOCATED:
+        return YarnApplicationAttemptState.ALLOCATED;
+      case LAUNCHED:
+        return YarnApplicationAttemptState.LAUNCHED;
+      case ALLOCATED_SAVING:
+      case LAUNCHED_UNMANAGED_SAVING:
+        return YarnApplicationAttemptState.ALLOCATED_SAVING;
+      case RUNNING:
+        return YarnApplicationAttemptState.RUNNING;
+      case FINISHING:
+        return YarnApplicationAttemptState.FINISHING;
+      case FINAL_SAVING:
+        return YarnApplicationAttemptState.FINAL_SAVING;
+      case FINISHED:
+        return YarnApplicationAttemptState.FINISHED;
+      case KILLED:
+        return YarnApplicationAttemptState.KILLED;
+      case FAILED:
+        return YarnApplicationAttemptState.FAILED;
+      default:
+        throw new YarnRuntimeException("Unknown state passed!");
+    }
+  }
+
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Tue Jan 28 19:50:28 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
@@ -1069,27 +1070,7 @@ public class RMAppImpl implements RMApp,
     if (rmAppState.equals(RMAppState.KILLING)) {
       rmAppState = stateBeforeKilling;
     }
-    switch (rmAppState) {
-    case NEW:
-      return YarnApplicationState.NEW;
-    case NEW_SAVING:
-      return YarnApplicationState.NEW_SAVING;
-    case SUBMITTED:
-      return YarnApplicationState.SUBMITTED;
-    case ACCEPTED:
-      return YarnApplicationState.ACCEPTED;
-    case RUNNING:
-      return YarnApplicationState.RUNNING;
-    case FINISHING:
-    case FINISHED:
-      return YarnApplicationState.FINISHED;
-    case KILLED:
-      return YarnApplicationState.KILLED;
-    case FAILED:
-      return YarnApplicationState.FAILED;
-    default:
-      throw new YarnRuntimeException("Unknown state passed!");
-    }
+    return RMServerUtils.createApplicationState(rmAppState);
   }
   
   public static boolean isAppInFinalState(RMApp rmApp) {



Mime
View raw message