hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1169484 [1/2] - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/...
Date Sun, 11 Sep 2011 17:26:07 GMT
Author: acmurthy
Date: Sun Sep 11 17:26:06 2011
New Revision: 1169484

URL: http://svn.apache.org/viewvc?rev=1169484&view=rev
Log:
MAPREDUCE-2933. Change allocate call to return ContainerStatus for completed containers rather than Container.

Added:
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
Modified:
    hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeUpdateSchedulerEvent.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Sun Sep 11 17:26:06 2011
@@ -264,6 +264,9 @@ Release 0.23.0 - Unreleased
     MAPREDUCE-2711. Update TestBlockPlacementPolicyRaid for the new namesystem
     and block management APIs.  (szetszwo)
 
+    MAPREDUCE-2933. Change allocate call to return ContainerStatus for
+    completed containers rather than Container. (acmurthy) 
+
   OPTIMIZATIONS
 
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Sun Sep 11 17:26:06 2011
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.yarn.api.records.AMResponse;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.util.RackResolver;
@@ -414,8 +415,8 @@ public class RMContainerAllocator extend
     int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;//first time it would be null
     AMResponse response = makeRemoteRequest();
     int newHeadRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
-    List<Container> newContainers = response.getNewContainerList();
-    List<Container> finishedContainers = response.getFinishedContainerList();
+    List<Container> newContainers = response.getAllocatedContainers();
+    List<ContainerStatus> finishedContainers = response.getCompletedContainersStatuses();
     if (newContainers.size() + finishedContainers.size() > 0 || headRoom != newHeadRoom) {
       //something changed
       recalculateReduceSchedule = true;
@@ -426,12 +427,12 @@ public class RMContainerAllocator extend
         allocatedContainers.add(cont);
         LOG.debug("Received new Container :" + cont);
     }
-    for (Container cont : finishedContainers) {
+    for (ContainerStatus cont : finishedContainers) {
       LOG.info("Received completed container " + cont);
-      TaskAttemptId attemptID = assignedRequests.get(cont.getId());
+      TaskAttemptId attemptID = assignedRequests.get(cont.getContainerId());
       if (attemptID == null) {
         LOG.error("Container complete event for unknown container id "
-            + cont.getId());
+            + cont.getContainerId());
       } else {
         assignedRequests.remove(attemptID);
         if (attemptID.getTaskId().getTaskType().equals(TaskType.MAP)) {
@@ -443,7 +444,7 @@ public class RMContainerAllocator extend
         eventHandler.handle(new TaskAttemptEvent(attemptID,
             TaskAttemptEventType.TA_CONTAINER_COMPLETED));
         // Send the diagnostics
-        String diagnostics = cont.getContainerStatus().getDiagnostics();
+        String diagnostics = cont.getDiagnostics();
         eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID,
             diagnostics));
       }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java Sun Sep 11 17:26:06 2011
@@ -36,7 +36,6 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.AMResponse;
-import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -123,10 +122,11 @@ public abstract class RMContainerRequest
     availableResources = response.getAvailableResources();
 
     LOG.info("getResources() for " + applicationId + ":" + " ask="
-        + ask.size() + " release= " + release.size() + " newContainers="
-        + response.getNewContainerCount() + " finishedContainers="
-        + response.getFinishedContainerCount()
-        + " resourcelimit=" + availableResources);
+        + ask.size() + " release= " + release.size() + 
+        " newContainers=" + response.getAllocatedContainers().size() + 
+        " finishedContainers=" + 
+        response.getCompletedContainersStatuses().size() + 
+        " resourcelimit=" + availableResources);
 
     ask.clear();
     release.clear();

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java Sun Sep 11 17:26:06 2011
@@ -86,32 +86,17 @@ public interface AMResponse {
    */
   @Public
   @Stable
-  public List<Container> getNewContainerList();
+  public List<Container> getAllocatedContainers();
 
-  @Private
-  @Unstable
-  public Container getNewContainer(int index);
-
-  @Private
-  @Unstable
-  public int getNewContainerCount();
-
-  @Private
-  @Unstable
-  public void addAllNewContainers(List<Container> containers);
-
-  @Private
-  @Unstable
-  public void addNewContainer(Container container);
-
-  @Private
-  @Unstable
-  public void removeNewContainer(int index);
+  /**
+   * Set the list of <em>newly allocated</em> <code>Container</code> by the 
+   * <code>ResourceManager</code>.
+   * @param containers list of <em>newly allocated</em> <code>Container</code>
+   */
+  @Public
+  @Stable
+  public void setAllocatedContainers(List<Container> containers);
 
-  @Private
-  @Unstable
-  public void clearNewContainers();
-  
   /**
    * Get the <em>available headroom</em> for resources in the cluster for the 
    * application.
@@ -127,35 +112,18 @@ public interface AMResponse {
   public void setAvailableResources(Resource limit);
   
   /**
-   * Get the list of <em>completed containers</em>.
-   * @return the list of <em>completed containers</em>
+   * Get the list of <em>completed containers' statuses</em>.
+   * @return the list of <em>completed containers' statuses</em>
    */
   @Public
   @Stable
-  public List<Container> getFinishedContainerList();
+  public List<ContainerStatus> getCompletedContainersStatuses();
 
-  @Private
-  @Unstable
-  public Container getFinishedContainer(int index);
-
-  @Private
-  @Unstable
-  public int getFinishedContainerCount();
-  
-
-  @Private
-  @Unstable
-  public void addAllFinishedContainers(List<Container> containers);
-
-  @Private
-  @Unstable
-  public void addFinishedContainer(Container container);
-
-  @Private
-  @Unstable
-  public void removeFinishedContainer(int index);
-
-  @Private
-  @Unstable
-  public void clearFinishedContainers();
+  /**
+   * Set the list of list of <em>completed containers' statuses</em>.
+   * @param containers list of <em>completed containers' statuses</em>
+   */
+  @Public
+  @Stable
+  public void setCompletedContainersStatuses(List<ContainerStatus> containers);
 }
\ No newline at end of file

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java Sun Sep 11 17:26:06 2011
@@ -25,11 +25,13 @@ import java.util.List;
 
 import org.apache.hadoop.yarn.api.records.AMResponse;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.ProtoBase;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.proto.YarnProtos.AMResponseProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.AMResponseProtoOrBuilder;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 
 
@@ -41,8 +43,8 @@ public class AMResponsePBImpl extends Pr
   
   Resource limit;
 
-  private List<Container> newContainersList = null;
-  private List<Container> finishedContainersList = null;
+  private List<Container> allocatedContainers = null;
+  private List<ContainerStatus> completedContainersStatuses = null;
 //  private boolean hasLocalContainerList = false;
   
   
@@ -63,15 +65,17 @@ public class AMResponsePBImpl extends Pr
   }
   
   private synchronized void mergeLocalToBuilder() {
-    if (this.newContainersList != null) {
-      builder.clearNewContainers();
-      Iterable<ContainerProto> iterable = getProtoIterable(this.newContainersList);
-      builder.addAllNewContainers(iterable);
+    if (this.allocatedContainers != null) {
+      builder.clearAllocatedContainers();
+      Iterable<ContainerProto> iterable = 
+          getProtoIterable(this.allocatedContainers);
+      builder.addAllAllocatedContainers(iterable);
     }
-    if (this.finishedContainersList != null) {
-      builder.clearFinishedContainers();
-      Iterable<ContainerProto> iterable = getProtoIterable(this.finishedContainersList);
-      builder.addAllFinishedContainers(iterable);
+    if (this.completedContainersStatuses != null) {
+      builder.clearCompletedContainerStatuses();
+      Iterable<ContainerStatusProto> iterable = 
+          getContainerStatusProtoIterable(this.completedContainersStatuses);
+      builder.addAllCompletedContainerStatuses(iterable);
     }
     if (this.limit != null) {
       builder.setLimit(convertToProtoFormat(this.limit));
@@ -139,42 +143,31 @@ public class AMResponsePBImpl extends Pr
   }
 
   @Override
-  public synchronized List<Container> getNewContainerList() {
+  public synchronized List<Container> getAllocatedContainers() {
     initLocalNewContainerList();
-    return this.newContainersList;
-  }
-  
-  @Override
-  public synchronized Container getNewContainer(int index) {
-    initLocalNewContainerList();
-    return this.newContainersList.get(index);
-  }
-  @Override
-  public synchronized int getNewContainerCount() {
-    initLocalNewContainerList();
-    return this.newContainersList.size();
+    return this.allocatedContainers;
   }
   
   //Once this is called. containerList will never be null - untill a getProto is called.
   private synchronized void initLocalNewContainerList() {
-    if (this.newContainersList != null) {
+    if (this.allocatedContainers != null) {
       return;
     }
     AMResponseProtoOrBuilder p = viaProto ? proto : builder;
-    List<ContainerProto> list = p.getNewContainersList();
-    newContainersList = new ArrayList<Container>();
+    List<ContainerProto> list = p.getAllocatedContainersList();
+    allocatedContainers = new ArrayList<Container>();
 
     for (ContainerProto c : list) {
-      newContainersList.add(convertFromProtoFormat(c));
+      allocatedContainers.add(convertFromProtoFormat(c));
     }
   }
 
   @Override
-  public synchronized void addAllNewContainers(final List<Container> containers) {
+  public synchronized void setAllocatedContainers(final List<Container> containers) {
     if (containers == null) 
       return;
     initLocalNewContainerList();
-    newContainersList.addAll(containers);
+    allocatedContainers.addAll(containers);
   }
 
   private synchronized Iterable<ContainerProto> getProtoIterable(
@@ -207,86 +200,71 @@ public class AMResponsePBImpl extends Pr
       }
     };
   }
-  
-  @Override
-  public synchronized void addNewContainer(Container containers) {
-    initLocalNewContainerList();
-    if (containers == null) 
-      return;
-    this.newContainersList.add(containers);
-  }
-  
-  @Override
-  public synchronized void removeNewContainer(int index) {
-    initLocalNewContainerList();
-    this.newContainersList.remove(index);
-  }
-  @Override
-  public synchronized void clearNewContainers() {
-    initLocalNewContainerList();
-    this.newContainersList.clear();
+
+  private synchronized Iterable<ContainerStatusProto> 
+  getContainerStatusProtoIterable(
+      final List<ContainerStatus> newContainersList) {
+    maybeInitBuilder();
+    return new Iterable<ContainerStatusProto>() {
+      @Override
+      public synchronized Iterator<ContainerStatusProto> iterator() {
+        return new Iterator<ContainerStatusProto>() {
+
+          Iterator<ContainerStatus> iter = newContainersList.iterator();
+
+          @Override
+          public synchronized boolean hasNext() {
+            return iter.hasNext();
+          }
+
+          @Override
+          public synchronized ContainerStatusProto next() {
+            return convertToProtoFormat(iter.next());
+          }
+
+          @Override
+          public synchronized void remove() {
+            throw new UnsupportedOperationException();
+
+          }
+        };
+
+      }
+    };
   }
 
   //// Finished containers
   @Override
-  public synchronized List<Container> getFinishedContainerList() {
+  public synchronized List<ContainerStatus> getCompletedContainersStatuses() {
     initLocalFinishedContainerList();
-    return this.finishedContainersList;
-  }
-  
-  @Override
-  public synchronized Container getFinishedContainer(int index) {
-    initLocalFinishedContainerList();
-    return this.finishedContainersList.get(index);
-  }
-  @Override
-  public synchronized int getFinishedContainerCount() {
-    initLocalFinishedContainerList();
-    return this.finishedContainersList.size();
+    return this.completedContainersStatuses;
   }
   
   //Once this is called. containerList will never be null - untill a getProto is called.
   private synchronized void initLocalFinishedContainerList() {
-    if (this.finishedContainersList != null) {
+    if (this.completedContainersStatuses != null) {
       return;
     }
     AMResponseProtoOrBuilder p = viaProto ? proto : builder;
-    List<ContainerProto> list = p.getFinishedContainersList();
-    finishedContainersList = new ArrayList<Container>();
+    List<ContainerStatusProto> list = p.getCompletedContainerStatusesList();
+    completedContainersStatuses = new ArrayList<ContainerStatus>();
 
-    for (ContainerProto c : list) {
-      finishedContainersList.add(convertFromProtoFormat(c));
+    for (ContainerStatusProto c : list) {
+      completedContainersStatuses.add(convertFromProtoFormat(c));
     }
   }
 
   @Override
-  public synchronized void addAllFinishedContainers(final List<Container> containers) {
+  public synchronized void setCompletedContainersStatuses(
+      final List<ContainerStatus> containers) {
     if (containers == null) 
       return;
     initLocalFinishedContainerList();
-    finishedContainersList.addAll(containers);
+    completedContainersStatuses.addAll(containers);
   }
   
-  @Override
-  public synchronized void addFinishedContainer(Container containers) {
-    initLocalFinishedContainerList();
-    if (containers == null) 
-      return;
-    this.finishedContainersList.add(containers);
-  }
-  
-  @Override
-  public synchronized void removeFinishedContainer(int index) {
-    initLocalFinishedContainerList();
-    this.finishedContainersList.remove(index);
-  }
-  @Override
-  public synchronized void clearFinishedContainers() {
-    initLocalFinishedContainerList();
-    this.finishedContainersList.clear();
-  }
-
-  private synchronized ContainerPBImpl convertFromProtoFormat(ContainerProto p) {
+  private synchronized ContainerPBImpl convertFromProtoFormat(
+      ContainerProto p) {
     return new ContainerPBImpl(p);
   }
 
@@ -294,6 +272,15 @@ public class AMResponsePBImpl extends Pr
     return ((ContainerPBImpl)t).getProto();
   }
 
+  private synchronized ContainerStatusPBImpl convertFromProtoFormat(
+      ContainerStatusProto p) {
+    return new ContainerStatusPBImpl(p);
+  }
+
+  private synchronized ContainerStatusProto convertToProtoFormat(ContainerStatus t) {
+    return ((ContainerStatusPBImpl)t).getProto();
+  }
+
   private synchronized ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
     return new ResourcePBImpl(p);
   }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Sun Sep 11 17:26:06 2011
@@ -177,8 +177,8 @@ message ResourceRequestProto {
 message AMResponseProto {
   optional bool reboot = 1;
   optional int32 response_id = 2;
-  repeated ContainerProto new_containers = 3;
-  repeated ContainerProto finished_containers = 4;
+  repeated ContainerProto allocated_containers = 3;
+  repeated ContainerStatusProto completed_container_statuses = 4;
   optional ResourceProto limit = 5;
 }
 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java Sun Sep 11 17:26:06 2011
@@ -18,10 +18,8 @@
 package org.apache.hadoop.yarn.server.api.records;
 
 import java.util.List;
-import java.util.Map;
 
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 
@@ -31,17 +29,13 @@ public interface NodeStatus {
   public abstract NodeId getNodeId();
   public abstract int getResponseId();
   
-  public abstract Map<ApplicationId, List<Container>> getAllContainers();
-  public abstract List<Container> getContainers(ApplicationId key);
+  public abstract List<ContainerStatus> getContainersStatuses();
+  public abstract void setContainersStatuses(
+      List<ContainerStatus> containersStatuses);
 
   NodeHealthStatus getNodeHealthStatus();
   void setNodeHealthStatus(NodeHealthStatus healthStatus);
 
   public abstract void setNodeId(NodeId nodeId);
   public abstract void setResponseId(int responseId);
-  
-  public abstract void addAllContainers(Map<ApplicationId, List<Container>> containers);
-  public abstract void setContainers(ApplicationId key, List<Container> containers);
-  public abstract void removeContainers(ApplicationId key);
-  public abstract void clearContainers();
 }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java Sun Sep 11 17:26:06 2011
@@ -20,27 +20,19 @@ package org.apache.hadoop.yarn.server.ap
 
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
 
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.ProtoBase;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeHealthStatusPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeHealthStatusProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ApplicationIdContainerListMapProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ContainerListProto;
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProtoOrBuilder;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -51,7 +43,7 @@ public class NodeStatusPBImpl extends Pr
   boolean viaProto = false;
   
   private NodeId nodeId = null;
-  private Map<ApplicationIdProto, List<Container>> containers = null;
+  private List<ContainerStatus> containers = null;
   private NodeHealthStatus nodeHealthStatus = null;
   
   public NodeStatusPBImpl() {
@@ -99,7 +91,40 @@ public class NodeStatusPBImpl extends Pr
     viaProto = false;
   }
     
+  private void addContainersToProto() {
+    maybeInitBuilder();
+    builder.clearContainersStatuses();
+    if (containers == null)
+      return;
+    Iterable<ContainerStatusProto> iterable = new Iterable<ContainerStatusProto>() {
+      @Override
+      public Iterator<ContainerStatusProto> iterator() {
+        return new Iterator<ContainerStatusProto>() {
+  
+          Iterator<ContainerStatus> iter = containers.iterator();
+  
+          @Override
+          public boolean hasNext() {
+            return iter.hasNext();
+          }
+  
+          @Override
+          public ContainerStatusProto next() {
+            return convertToProtoFormat(iter.next());
+          }
+  
+          @Override
+          public void remove() {
+            throw new UnsupportedOperationException();
+  
+          }
+        };
   
+      }
+    };
+    builder.addAllContainersStatuses(iterable);
+  }
+
   @Override
   public int getResponseId() {
     NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
@@ -133,24 +158,17 @@ public class NodeStatusPBImpl extends Pr
   }
   
   @Override
-  public Map<ApplicationId, List<Container>> getAllContainers() {
+  public List<ContainerStatus> getContainersStatuses() {
     initContainers();
-    HashMap<ApplicationId, List<Container>> returnMap = new HashMap<ApplicationId, List<Container>>(
-        this.containers.size());
-    for (Entry<ApplicationIdProto, List<Container>> entry : this.containers.entrySet()) {
-      returnMap.put(convertFromProtoFormat(entry.getKey()), entry.getValue());
-    }
-    return returnMap;
+    return this.containers;
   }
 
   @Override
-  public List<Container> getContainers(ApplicationId applicationId) {
-    initContainers();
-    ApplicationIdProto applicationIdProto = convertToProtoFormat(applicationId);
-    if (this.containers.get(applicationIdProto) == null) {
-      this.containers.put(applicationIdProto, new ArrayList<Container>());
+  public void setContainersStatuses(List<ContainerStatus> containers) {
+    if (containers == null) {
+      builder.clearContainersStatuses();
     }
-    return this.containers.get(applicationIdProto);
+    this.containers = containers;
   }
 
   private void initContainers() {
@@ -158,60 +176,16 @@ public class NodeStatusPBImpl extends Pr
       return;
     }
     NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
-    List<ApplicationIdContainerListMapProto> list = p.getContainersList();
-    this.containers = new HashMap<ApplicationIdProto, List<Container>>();
+    List<ContainerStatusProto> list = p.getContainersStatusesList();
+    this.containers = new ArrayList<ContainerStatus>();
 
-    for (ApplicationIdContainerListMapProto c : list) {
-      this.containers.put(c.getApplicationId(), convertFromProtoFormat(c.getValue()));
+    for (ContainerStatusProto c : list) {
+      this.containers.add(convertFromProtoFormat(c));
     }
     
   }
   
   @Override
-  public void addAllContainers(final Map<ApplicationId, List<Container>> containers) {
-    if (containers == null)
-      return;
-    initContainers();
-    for (Entry<ApplicationId, List<Container>> entry : containers.entrySet()) {
-      this.containers.put(convertToProtoFormat(entry.getKey()), entry.getValue());
-    }
-  }
-  
-  private void addContainersToProto() {
-    maybeInitBuilder();
-    builder.clearContainers();
-    viaProto = false;
-    Iterable<ApplicationIdContainerListMapProto> iterable = new Iterable<ApplicationIdContainerListMapProto>() {
-
-      @Override
-      public Iterator<ApplicationIdContainerListMapProto> iterator() {
-        return new Iterator<ApplicationIdContainerListMapProto>() {
-
-          Iterator<ApplicationIdProto> keyIter = containers.keySet().iterator();
-          @Override
-          public boolean hasNext() {
-            return keyIter.hasNext();
-          }
-
-          @Override
-          public ApplicationIdContainerListMapProto next() {
-            ApplicationIdProto applicationIdProto = keyIter.next();
-            return ApplicationIdContainerListMapProto.newBuilder().setApplicationId(applicationIdProto).setValue(convertToProtoFormat(containers.get(applicationIdProto))).build();
-          }
-
-          @Override
-          public void remove() {
-            throw new UnsupportedOperationException();
-          }
-
-        };
-      }
-      
-    };
-    builder.addAllContainers(iterable);
-  }
-
-  @Override
   public NodeHealthStatus getNodeHealthStatus() {
     NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
     if (nodeHealthStatus != null) {
@@ -233,66 +207,6 @@ public class NodeStatusPBImpl extends Pr
     this.nodeHealthStatus = healthStatus;
   }
 
-  /*
-   * 
-   * @Override
-  public String getApplicationName() {
-    ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
-    if (!p.hasApplicationName()) {
-      return null;
-    }
-    return (p.getApplicationName());
-  }
-
-  @Override
-  public void setApplicationName(String applicationName) {
-    maybeInitBuilder();
-    if (applicationName == null) {
-      builder.clearApplicationName();
-      return;
-    }
-    builder.setApplicationName((applicationName));
-  }
-  */
-  
-  private ContainerListProto convertToProtoFormat(List<Container> src) {
-    ContainerListProto.Builder ret = ContainerListProto.newBuilder();
-    for (Container c : src) {
-      ret.addContainer(((ContainerPBImpl)c).getProto());
-    }
-    return ret.build();
-  }
-  
-  private List<Container> convertFromProtoFormat(ContainerListProto src) {
-    List<Container> ret = new ArrayList<Container>();
-    for (ContainerProto c : src.getContainerList()) {
-      ret.add(convertFromProtoFormat(c));
-    }
-    return ret;
-  }
-
-  private Container convertFromProtoFormat(ContainerProto src) {
-    return new ContainerPBImpl(src);
-  }
-  
-  @Override
-  public void setContainers(ApplicationId applicationId, List<Container> containers) {
-    initContainers();
-    this.containers.put(convertToProtoFormat(applicationId), containers);
-  }
-
-  @Override
-  public void removeContainers(ApplicationId applicationId) {
-    initContainers();
-    this.containers.remove(convertToProtoFormat(applicationId));
-  }
-  
-  @Override
-  public void clearContainers() {
-    initContainers();
-    this.containers.clear();
-  }
-
   private NodeIdProto convertToProtoFormat(NodeId nodeId) {
     return ((NodeIdPBImpl)nodeId).getProto();
   }
@@ -301,14 +215,6 @@ public class NodeStatusPBImpl extends Pr
     return new NodeIdPBImpl(proto);
   }
 
-  private ApplicationIdProto convertToProtoFormat(ApplicationId applicationId) {
-    return ((ApplicationIdPBImpl)applicationId).getProto();
-  }
-  
-  private ApplicationId convertFromProtoFormat(ApplicationIdProto proto) {
-    return new ApplicationIdPBImpl(proto);
-  }
-
   private NodeHealthStatusProto convertToProtoFormat(
       NodeHealthStatus healthStatus) {
     return ((NodeHealthStatusPBImpl) healthStatus).getProto();
@@ -317,4 +223,12 @@ public class NodeStatusPBImpl extends Pr
   private NodeHealthStatus convertFromProtoFormat(NodeHealthStatusProto proto) {
     return new NodeHealthStatusPBImpl(proto);
   }
+
+  private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto c) {
+    return new ContainerStatusPBImpl(c);
+  }
+  
+  private ContainerStatusProto convertToProtoFormat(ContainerStatus c) {
+    return ((ContainerStatusPBImpl)c).getProto();
+  }
 }  

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto Sun Sep 11 17:26:06 2011
@@ -26,7 +26,7 @@ import "yarn_protos.proto";
 message NodeStatusProto {
   optional NodeIdProto node_id = 1;
   optional int32 response_id = 2;
-  repeated ApplicationIdContainerListMapProto containers = 3;
+  repeated ContainerStatusProto containersStatuses = 3;
   optional NodeHealthStatusProto nodeHealthStatus = 4;
 }
 
@@ -41,12 +41,3 @@ message HeartbeatResponseProto {
   repeated ApplicationIdProto applications_to_cleanup = 4;
 }
 
-message ContainerListProto {
-  repeated ContainerProto container = 1;
-}
-
-message ApplicationIdContainerListMapProto {
-  optional ApplicationIdProto application_id = 1;
-  optional ContainerListProto value = 2;
-}
-

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Sun Sep 11 17:26:06 2011
@@ -30,13 +30,13 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.NodeHealthCheckerService;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityInfo;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -73,7 +73,6 @@ public class NodeStatusUpdaterImpl exten
   private String rmAddress;
   private Resource totalResource;
   private String containerManagerBindAddress;
-  private String nodeHttpAddress;
   private String hostName;
   private int containerManagerPort;
   private int httpPort;
@@ -127,7 +126,6 @@ public class NodeStatusUpdaterImpl exten
       this.httpPort = httpBindAddress.getPort();
       this.containerManagerBindAddress =
           this.hostName + ":" + this.containerManagerPort;
-      this.nodeHttpAddress = this.hostName + ":" + this.httpPort;
       LOG.info("Configured ContainerManager Address is "
           + this.containerManagerBindAddress);
       // Registration has to be in start so that ContainerManager can get the
@@ -195,35 +193,28 @@ public class NodeStatusUpdaterImpl exten
     nodeStatus.setNodeId(this.nodeId);
 
     int numActiveContainers = 0;
+    List<ContainerStatus> containersStatuses = new ArrayList<ContainerStatus>();
     for (Iterator<Entry<ContainerId, Container>> i =
         this.context.getContainers().entrySet().iterator(); i.hasNext();) {
       Entry<ContainerId, Container> e = i.next();
       ContainerId containerId = e.getKey();
       Container container = e.getValue();
 
-      List<org.apache.hadoop.yarn.api.records.Container> applicationContainers = nodeStatus
-          .getContainers(container.getContainerID().getAppId());
-      if (applicationContainers == null) {
-        applicationContainers = new ArrayList<org.apache.hadoop.yarn.api.records.Container>();
-        nodeStatus.setContainers(container.getContainerID().getAppId(),
-            applicationContainers);
-      }
-
       // Clone the container to send it to the RM
-      org.apache.hadoop.yarn.api.records.Container c = container.cloneAndGetContainer();
-      c.setNodeId(this.nodeId);
-      c.setNodeHttpAddress(this.nodeHttpAddress); // TODO: don't set everytime.
-      applicationContainers.add(c);
+      org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus = 
+          container.cloneAndGetContainerStatus();
+      containersStatuses.add(containerStatus);
       ++numActiveContainers;
-      LOG.info("Sending out status for container: " + c);
+      LOG.info("Sending out status for container: " + containerStatus);
 
-      if (c.getState() == ContainerState.COMPLETE) {
+      if (containerStatus.getState() == ContainerState.COMPLETE) {
         // Remove
         i.remove();
 
         LOG.info("Removed completed container " + containerId);
       }
     }
+    nodeStatus.setContainersStatuses(containersStatuses);
 
     LOG.debug(this.containerManagerBindAddress + " sending out status for " + numActiveContainers
         + " containers");

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java Sun Sep 11 17:26:06 2011
@@ -40,8 +40,6 @@ public interface Container extends Event
 
   Map<Path,String> getLocalizedResources();
 
-  org.apache.hadoop.yarn.api.records.Container cloneAndGetContainer();
-
   ContainerStatus cloneAndGetContainerStatus();
 
   String toString();

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Sun Sep 11 17:26:06 2011
@@ -327,24 +327,6 @@ public class ContainerImpl implements Co
   }
 
   @Override
-  public
-      org.apache.hadoop.yarn.api.records.Container cloneAndGetContainer() {
-    this.readLock.lock();
-    try {
-      org.apache.hadoop.yarn.api.records.Container c =
-        recordFactory.newRecordInstance(
-            org.apache.hadoop.yarn.api.records.Container.class);
-      c.setId(this.launchContext.getContainerId());
-      c.setResource(this.launchContext.getResource());
-      c.setState(getCurrentState());
-      c.setContainerStatus(cloneAndGetContainerStatus());
-      return c;
-    } finally {
-      this.readLock.unlock();
-    }
-  }
-
-  @Override
   public ContainerLaunchContext getLaunchContext() {
     this.readLock.lock();
     try {

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Sun Sep 11 17:26:06 2011
@@ -21,6 +21,10 @@ package org.apache.hadoop.yarn.server.no
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentMap;
 
 import org.apache.commons.logging.Log;
@@ -32,6 +36,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -98,13 +103,30 @@ public class TestNodeStatusUpdater {
     ContainerId firstContainerID = recordFactory.newRecordInstance(ContainerId.class);
     ContainerId secondContainerID = recordFactory.newRecordInstance(ContainerId.class);
 
+    private Map<ApplicationId, List<ContainerStatus>> getAppToContainerStatusMap(
+        List<ContainerStatus> containers) {
+      Map<ApplicationId, List<ContainerStatus>> map =
+          new HashMap<ApplicationId, List<ContainerStatus>>();
+      for (ContainerStatus cs : containers) {
+        ApplicationId applicationId = cs.getContainerId().getAppId();
+        List<ContainerStatus> appContainers = map.get(applicationId);
+        if (appContainers == null) {
+          appContainers = new ArrayList<ContainerStatus>();
+          map.put(applicationId, appContainers);
+        }
+        appContainers.add(cs);
+      }
+      return map;
+    }
     @Override
     public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) throws YarnRemoteException {
       NodeStatus nodeStatus = request.getNodeStatus();
       LOG.info("Got heartbeat number " + heartBeatID);
       nodeStatus.setResponseId(heartBeatID++);
+      Map<ApplicationId, List<ContainerStatus>> appToContainers =
+          getAppToContainerStatusMap(nodeStatus.getContainersStatuses());
       if (heartBeatID == 1) {
-        Assert.assertEquals(0, nodeStatus.getAllContainers().size());
+        Assert.assertEquals(0, nodeStatus.getContainersStatuses().size());
 
         // Give a container to the NM.
         applicationID.setId(heartBeatID);
@@ -121,11 +143,9 @@ public class TestNodeStatusUpdater {
       } else if (heartBeatID == 2) {
         // Checks on the RM end
         Assert.assertEquals("Number of applications should only be one!", 1,
-            nodeStatus.getAllContainers().size());
+            nodeStatus.getContainersStatuses().size());
         Assert.assertEquals("Number of container for the app should be one!",
-            1, nodeStatus.getContainers(applicationID).size());
-        Assert.assertEquals(2, nodeStatus.getContainers(applicationID).get(0)
-            .getResource().getMemory());
+            1, appToContainers.get(applicationID).size());
 
         // Checks on the NM end
         ConcurrentMap<ContainerId, Container> activeContainers =
@@ -147,13 +167,9 @@ public class TestNodeStatusUpdater {
       } else if (heartBeatID == 3) {
         // Checks on the RM end
         Assert.assertEquals("Number of applications should only be one!", 1,
-            nodeStatus.getAllContainers().size());
+            appToContainers.size());
         Assert.assertEquals("Number of container for the app should be two!",
-            2, nodeStatus.getContainers(applicationID).size());
-        Assert.assertEquals(2, nodeStatus.getContainers(applicationID).get(0)
-            .getResource().getMemory());
-        Assert.assertEquals(3, nodeStatus.getContainers(applicationID).get(1)
-            .getResource().getMemory());
+            2, appToContainers.get(applicationID).size());
 
         // Checks on the NM end
         ConcurrentMap<ContainerId, Container> activeContainers =

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Sun Sep 11 17:26:06 2011
@@ -232,8 +232,8 @@ public class ApplicationMasterService ex
       RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
 
       AMResponse response = recordFactory.newRecordInstance(AMResponse.class);
-      response.addAllNewContainers(allocation.getContainers());
-      response.addAllFinishedContainers(appAttempt
+      response.setAllocatedContainers(allocation.getContainers());
+      response.setCompletedContainersStatuses(appAttempt
           .pullJustFinishedContainers());
       response.setResponseId(lastResponse.getResponseId() + 1);
       response.setAvailableResources(allocation.getResourceLimit());

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Sun Sep 11 17:26:06 2011
@@ -20,10 +20,6 @@ package org.apache.hadoop.yarn.server.re
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.crypto.SecretKey;
 
@@ -31,7 +27,6 @@ import org.apache.avro.ipc.Server;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.security.SecurityInfo;
@@ -252,7 +247,7 @@ public class ResourceTrackerService exte
       // 4. Send status to RMNode, saving the latest response.
       this.rmContext.getDispatcher().getEventHandler().handle(
           new RMNodeStatusEvent(nodeId, remoteNodeStatus.getNodeHealthStatus(),
-              remoteNodeStatus.getAllContainers(), latestResponse));
+              remoteNodeStatus.getContainersStatuses(), latestResponse));
 
       nodeHeartBeatResponse.setHeartbeatResponse(latestResponse);
       return nodeHeartBeatResponse;

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Sun Sep 11 17:26:06 2011
@@ -24,6 +24,7 @@ import java.util.Set;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.event.EventHandler;
 
@@ -47,9 +48,9 @@ public interface RMAppAttempt extends Ev
 
   Set<NodeId> getRanNodes();
 
-  List<Container> pullJustFinishedContainers();
+  List<ContainerStatus> pullJustFinishedContainers();
 
-  List<Container> getJustFinishedContainers();
+  List<ContainerStatus> getJustFinishedContainers();
 
   Container getMasterContainer();
 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Sun Sep 11 17:26:06 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -97,8 +98,8 @@ public class RMAppAttemptImpl implements
   //nodes on while this attempt's containers ran
   private final Set<NodeId> ranNodes = 
     new HashSet<NodeId>();
-  private final List<Container> justFinishedContainers = 
-    new ArrayList<Container>();
+  private final List<ContainerStatus> justFinishedContainers = 
+    new ArrayList<ContainerStatus>();
   private Container masterContainer;
 
   private float progress = 0;
@@ -333,7 +334,7 @@ public class RMAppAttemptImpl implements
   }
 
   @Override
-  public List<Container> getJustFinishedContainers() {
+  public List<ContainerStatus> getJustFinishedContainers() {
     this.readLock.lock();
     try {
       return this.justFinishedContainers;
@@ -343,11 +344,11 @@ public class RMAppAttemptImpl implements
   }
 
   @Override
-  public List<Container> pullJustFinishedContainers() {
+  public List<ContainerStatus> pullJustFinishedContainers() {
     this.writeLock.lock();
 
     try {
-      List<Container> returnList = new ArrayList<Container>(
+      List<ContainerStatus> returnList = new ArrayList<ContainerStatus>(
           this.justFinishedContainers.size());
       returnList.addAll(this.justFinishedContainers);
       this.justFinishedContainers.clear();
@@ -705,11 +706,13 @@ public class RMAppAttemptImpl implements
 
       RMAppAttemptContainerFinishedEvent containerFinishedEvent
         = (RMAppAttemptContainerFinishedEvent) event;
-      Container container = containerFinishedEvent.getContainer();
+      ContainerStatus containerStatus = 
+          containerFinishedEvent.getContainerStatus();
 
       // Is this container the AmContainer? If the finished container is same as
       // the AMContainer, AppAttempt fails
-      if (appAttempt.masterContainer.getId().equals(container.getId())) {
+      if (appAttempt.masterContainer.getId().equals(
+          containerStatus.getContainerId())) {
         new FinalTransition(RMAppAttemptState.FAILED).transition(
             appAttempt, containerFinishedEvent);
         return RMAppAttemptState.FAILED;
@@ -718,7 +721,7 @@ public class RMAppAttemptImpl implements
       // Normal container.
 
       // Put it in completedcontainers list
-      appAttempt.justFinishedContainers.add(container);
+      appAttempt.justFinishedContainers.add(containerStatus);
       return RMAppAttemptState.RUNNING;
     }
   }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java Sun Sep 11 17:26:06 2011
@@ -19,22 +19,22 @@
 package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
 
 public class RMAppAttemptContainerFinishedEvent extends RMAppAttemptEvent {
 
-  private final Container container;
+  private final ContainerStatus containerStatus;
 
   public RMAppAttemptContainerFinishedEvent(ApplicationAttemptId appAttemptId, 
-      Container container) {
+      ContainerStatus containerStatus) {
     super(appAttemptId, RMAppAttemptEventType.CONTAINER_FINISHED);
-    this.container = container;
+    this.containerStatus = containerStatus;
   }
 
-  public Container getContainer() {
-    return this.container;
+  public ContainerStatus getContainerStatus() {
+    return this.containerStatus;
   }
 
 }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java Sun Sep 11 17:26:06 2011
@@ -26,8 +26,8 @@ public class RMContainerFinishedEvent ex
   private final ContainerStatus remoteContainerStatus;
 
   public RMContainerFinishedEvent(ContainerId containerId,
-      ContainerStatus containerStatus) {
-    super(containerId, RMContainerEventType.FINISHED);
+      ContainerStatus containerStatus, RMContainerEventType event) {
+    super(containerId, event);
     this.remoteContainerStatus = containerStatus;
   }
 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java Sun Sep 11 17:26:06 2011
@@ -92,7 +92,7 @@ public class RMContainerImpl implements 
 
     // Transitions from RUNNING state
     .addTransition(RMContainerState.RUNNING, RMContainerState.COMPLETED,
-        RMContainerEventType.FINISHED, new ContainerCompletedTransition())
+        RMContainerEventType.FINISHED, new FinishedTransition())
     .addTransition(RMContainerState.RUNNING, RMContainerState.KILLED,
         RMContainerEventType.KILL, new KillTransition())
 
@@ -273,10 +273,16 @@ public class RMContainerImpl implements 
 
     @Override
     public void transition(RMContainerImpl container, RMContainerEvent event) {
+      RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
+
+      // Update container-status for diagnostics. Today we completely
+      // replace it on finish. We may just need to update diagnostics.
+      container.container.setContainerStatus(finishedEvent
+          .getRemoteContainerStatus());
 
       // Inform AppAttempt
       container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
-          container.appAttemptId, container.container));
+          container.appAttemptId, container.container.getContainerStatus()));
     }
   }
 
@@ -312,22 +318,4 @@ public class RMContainerImpl implements 
     }
   }
 
-  private static final class ContainerCompletedTransition extends
-      FinishedTransition {
-
-    @Override
-    public void transition(RMContainerImpl container, RMContainerEvent event) {
-
-      RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
-
-      // Update container-status for diagnostics. Today we completely
-      // replace it on finish. We may just need to update diagnostics.
-      // ^TODO
-      container.container.setContainerStatus(finishedEvent
-          .getRemoteContainerStatus());
-
-      // Inform appAttempt
-      super.transition(container, event);
-    }
-  }
 }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java Sun Sep 11 17:26:06 2011
@@ -23,7 +23,6 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -36,9 +35,9 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -87,8 +86,8 @@ public class RMNodeImpl implements RMNod
       .newRecordInstance(NodeHealthStatus.class);
   
   /* set of containers that have just launched */
-  private final Map<ContainerId, Container> justLaunchedContainers = 
-    new HashMap<ContainerId, Container>();
+  private final Map<ContainerId, ContainerStatus> justLaunchedContainers = 
+    new HashMap<ContainerId, ContainerStatus>();
   
 
   /* set of containers that need to be cleaned */
@@ -355,43 +354,29 @@ public class RMNodeImpl implements RMNod
 
       // Filter the map to only obtain just launched containers and finished
       // containers.
-      Map<ApplicationId, List<Container>> remoteAppContainersMap = statusEvent
-          .getContainersCollection();
-      Map<ApplicationId, List<Container>> containersMapForScheduler = new HashMap<ApplicationId, List<Container>>(
-          remoteAppContainersMap.size());
-      for (Entry<ApplicationId, List<Container>> entrySet : remoteAppContainersMap
-          .entrySet()) {
-
-        ApplicationId appId = entrySet.getKey();
-        List<Container> remoteContainerList = entrySet.getValue();
-
-        if (!containersMapForScheduler.containsKey(appId)) {
-          containersMapForScheduler.put(appId, new ArrayList<Container>(
-              remoteContainerList.size()));
-        }
-        List<Container> entryForThisApp = containersMapForScheduler
-            .get(appId);
-
-        for (Container remoteContainer : remoteContainerList) {
-
-          // Process running containers
-          ContainerId containerId = remoteContainer.getId();
-          if (remoteContainer.getState() == ContainerState.RUNNING) {
-            if (!rmNode.justLaunchedContainers.containsKey(containerId)) {
-              // Just launched container. RM knows about it the first time.
-              rmNode.justLaunchedContainers.put(containerId, remoteContainer);
-              entryForThisApp.add(remoteContainer);
-            }
-          } else {
-            // A finished container
-            rmNode.justLaunchedContainers.remove(containerId);
-            entryForThisApp.add(remoteContainer);
+      List<ContainerStatus> newlyLaunchedContainers = 
+          new ArrayList<ContainerStatus>();
+      List<ContainerStatus> completedContainers = 
+          new ArrayList<ContainerStatus>();
+      for (ContainerStatus remoteContainer : statusEvent.getContainers()) {
+        // Process running containers
+        ContainerId containerId = remoteContainer.getContainerId();
+        if (remoteContainer.getState() == ContainerState.RUNNING) {
+          if (!rmNode.justLaunchedContainers.containsKey(containerId)) {
+            // Just launched container. RM knows about it the first time.
+            rmNode.justLaunchedContainers.put(containerId, remoteContainer);
+            newlyLaunchedContainers.add(remoteContainer);
           }
+        } else {
+          // A finished container
+          rmNode.justLaunchedContainers.remove(containerId);
+          completedContainers.add(remoteContainer);
         }
       }
 
       rmNode.context.getDispatcher().getEventHandler().handle(
-          new NodeUpdateSchedulerEvent(rmNode, containersMapForScheduler));
+          new NodeUpdateSchedulerEvent(rmNode, newlyLaunchedContainers, 
+              completedContainers));
 
       return RMNodeState.RUNNING;
     }

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java Sun Sep 11 17:26:06 2011
@@ -19,10 +19,8 @@
 package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
 
 import java.util.List;
-import java.util.Map;
 
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
@@ -30,11 +28,11 @@ import org.apache.hadoop.yarn.server.api
 public class RMNodeStatusEvent extends RMNodeEvent {
 
   private final NodeHealthStatus nodeHealthStatus;
-  private Map<ApplicationId, List<Container>> containersCollection;
+  private List<ContainerStatus> containersCollection;
   private final HeartbeatResponse latestResponse;
 
   public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus,
-      Map<ApplicationId, List<Container>> collection,
+      List<ContainerStatus> collection,
       HeartbeatResponse latestResponse) {
     super(nodeId, RMNodeEventType.STATUS_UPDATE);
     this.nodeHealthStatus = nodeHealthStatus;
@@ -46,7 +44,7 @@ public class RMNodeStatusEvent extends R
     return this.nodeHealthStatus;
   }
 
-  public Map<ApplicationId, List<Container>> getContainersCollection() {
+  public List<ContainerStatus> getContainers() {
     return this.containersCollection;
   }
 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java Sun Sep 11 17:26:06 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -161,26 +162,21 @@ public class SchedulerApp {
             RMContainerEventType.LAUNCHED));
   }
 
-  public synchronized void killContainers(
-      SchedulerApp application) {
-  }
-
   synchronized public void containerCompleted(RMContainer rmContainer,
-      RMContainerEventType event) {
+      ContainerStatus containerStatus, RMContainerEventType event) {
     
     Container container = rmContainer.getContainer();
     ContainerId containerId = container.getId();
     
     // Inform the container
-    if (event.equals(RMContainerEventType.FINISHED)) {
-      // Have to send diagnostics for finished containers.
-      rmContainer.handle(new RMContainerFinishedEvent(containerId,
-          container.getContainerStatus()));
-    } else {
-      rmContainer.handle(new RMContainerEvent(containerId, event));
-    }
+    rmContainer.handle(
+        new RMContainerFinishedEvent(
+            containerId,
+            containerStatus, 
+            event)
+        );
     LOG.info("Completed container: " + rmContainer.getContainerId() + 
-        " in state: " + rmContainer.getState());
+        " in state: " + rmContainer.getState() + " event:" + event);
     
     // Remove from the list of containers
     liveContainers.remove(rmContainer.getContainerId());

Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1169484&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java Sun Sep 11 17:26:06 2011
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+
+/**
+ * Utilities shared by schedulers. 
+ */
+@Private
+@Unstable
+public class SchedulerUtils {
+  
+  private static final RecordFactory recordFactory = 
+      RecordFactoryProvider.getRecordFactory(null);
+
+  public static final String RELEASED_CONTAINER = 
+      "Container released by application";
+  
+  public static final String LOST_CONTAINER = 
+      "Container released on a *lost* node";
+  
+  public static final String COMPLETED_APPLICATION = 
+      "Container of a completed application";
+  
+  public static final String EXPIRED_CONTAINER =
+      "Container expired since it unused";
+  
+  public static final String UNRESERVED_CONTAINER =
+      "Container reservation no longer required.";
+  
+  /**
+   * Utility to create a {@link ContainerStatus} during exceptional
+   * circumstances.
+   * 
+   * @param containerId {@link ContainerId} of returned/released/lost container.
+   * @param diagnostics diagnostic message
+   * @return <code>ContainerStatus</code> for an returned/released/lost 
+   *         container
+   */
+  public static ContainerStatus createAbnormalContainerStatus(
+      ContainerId containerId, String diagnostics) {
+    ContainerStatus containerStatus = 
+        recordFactory.newRecordInstance(ContainerStatus.class);
+    containerStatus.setContainerId(containerId);
+    containerStatus.setDiagnostics(diagnostics);
+    containerStatus.setExitStatus("ABORTED");
+    containerStatus.setState(ContainerState.COMPLETE);
+    return containerStatus;
+  }
+
+
+}

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java Sun Sep 11 17:26:06 2011
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -165,11 +166,14 @@ extends org.apache.hadoop.yarn.server.re
    * @param node node on which the container completed
    * @param container completed container, 
    *                  <code>null</code> if it was just a reservation
+   * @param containerStatus <code>ContainerStatus</code> for the completed 
+   *                        container
    * @param event event to be sent to the container
    */
   public void completedContainer(Resource clusterResource,
       SchedulerApp application, SchedulerNode node, 
-      RMContainer container, RMContainerEventType event);
+      RMContainer container, ContainerStatus containerStatus, 
+      RMContainerEventType event);
 
   /**
    * Get the number of applications in the queue.



Mime
View raw message