hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1506393 [3/3] - in /hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project: ./ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/y...
Date Wed, 24 Jul 2013 03:48:20 GMT
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1506393&r1=1506392&r2=1506393&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
Wed Jul 24 03:48:19 2013
@@ -40,8 +40,9 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -227,9 +228,6 @@ public class TestContainersMonitor exten
     commands.add(scriptFile.getAbsolutePath());
     containerLaunchContext.setCommands(commands);
     Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1);
-    StartContainerRequest startRequest =
-        recordFactory.newRecordInstance(StartContainerRequest.class);
-    startRequest.setContainerLaunchContext(containerLaunchContext);
     ContainerTokenIdentifier containerIdentifier =
         new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
           r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER);
@@ -237,8 +235,14 @@ public class TestContainersMonitor exten
         BuilderUtils.newContainerToken(context.getNodeId(),
           containerManager.getContext().getContainerTokenSecretManager()
             .createPassword(containerIdentifier), containerIdentifier);
-    startRequest.setContainerToken(containerToken);
-    containerManager.startContainer(startRequest);
+    StartContainerRequest scRequest =
+        StartContainerRequest.newInstance(containerLaunchContext,
+          containerToken);
+    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+    list.add(scRequest);
+    StartContainersRequest allRequests =
+        StartContainersRequest.newInstance(list);
+    containerManager.startContainers(allRequests);
     
     int timeoutSecs = 0;
     while (!processStartFile.exists() && timeoutSecs++ < 20) {
@@ -260,11 +264,12 @@ public class TestContainersMonitor exten
     BaseContainerManagerTest.waitForContainerState(containerManager, cId,
         ContainerState.COMPLETE, 60);
 
-    GetContainerStatusRequest gcsRequest =
-        recordFactory.newRecordInstance(GetContainerStatusRequest.class);
-    gcsRequest.setContainerId(cId);
+    List<ContainerId> containerIds = new ArrayList<ContainerId>();
+    containerIds.add(cId);
+    GetContainerStatusesRequest gcsRequest =
+        GetContainerStatusesRequest.newInstance(containerIds);
     ContainerStatus containerStatus =
-        containerManager.getContainerStatus(gcsRequest).getStatus();
+        containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
     Assert.assertEquals(ExitCode.TERMINATED.getExitCode(),
         containerStatus.getExitStatus());
     String expectedMsgPattern =

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1506393&r1=1506392&r2=1506393&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
Wed Jul 24 03:48:19 2013
@@ -22,6 +22,8 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
@@ -33,11 +35,15 @@ import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
@@ -46,8 +52,6 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -68,8 +72,6 @@ public class AMLauncher implements Runna
 
   private final RMAppAttempt application;
   private final Configuration conf;
-  private final RecordFactory recordFactory = 
-      RecordFactoryProvider.getRecordFactory(null);
   private final AMLauncherEventType eventType;
   private final RMContext rmContext;
   private final Container masterContainer;
@@ -102,22 +104,42 @@ public class AMLauncher implements Runna
         + " for AM " + application.getAppAttemptId());  
     ContainerLaunchContext launchContext =
         createAMContainerLaunchContext(applicationContext, masterContainerID);
-    StartContainerRequest request = 
-        recordFactory.newRecordInstance(StartContainerRequest.class);
-    request.setContainerLaunchContext(launchContext);
-    request.setContainerToken(masterContainer.getContainerToken());
-    containerMgrProxy.startContainer(request);
-    LOG.info("Done launching container " + masterContainer
-        + " for AM " + application.getAppAttemptId());
+
+    StartContainerRequest scRequest =
+        StartContainerRequest.newInstance(launchContext,
+          masterContainer.getContainerToken());
+    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+    list.add(scRequest);
+    StartContainersRequest allRequests =
+        StartContainersRequest.newInstance(list);
+
+    StartContainersResponse response =
+        containerMgrProxy.startContainers(allRequests);
+    if (response.getFailedRequests() != null
+        && response.getFailedRequests().containsKey(masterContainerID)) {
+      Throwable t =
+          response.getFailedRequests().get(masterContainerID).deSerialize();
+      parseAndThrowException(t);
+    } else {
+      LOG.info("Done launching container " + masterContainer + " for AM "
+          + application.getAppAttemptId());
+    }
   }
   
   private void cleanup() throws IOException, YarnException {
     connect();
     ContainerId containerId = masterContainer.getId();
-    StopContainerRequest stopRequest = 
-        recordFactory.newRecordInstance(StopContainerRequest.class);
-    stopRequest.setContainerId(containerId);
-    containerMgrProxy.stopContainer(stopRequest);
+    List<ContainerId> containerIds = new ArrayList<ContainerId>();
+    containerIds.add(containerId);
+    StopContainersRequest stopRequest =
+        StopContainersRequest.newInstance(containerIds);
+    StopContainersResponse response =
+        containerMgrProxy.stopContainers(stopRequest);
+    if (response.getFailedRequests() != null
+        && response.getFailedRequests().containsKey(containerId)) {
+      Throwable t = response.getFailedRequests().get(containerId).deSerialize();
+      parseAndThrowException(t);
+    }
   }
 
   // Protected. For tests.
@@ -254,4 +276,15 @@ public class AMLauncher implements Runna
       break;
     }
   }
+
+  private void parseAndThrowException(Throwable t) throws YarnException,
+      IOException {
+    if (t instanceof YarnException) {
+      throw (YarnException) t;
+    } else if (t instanceof InvalidToken) {
+      throw (InvalidToken) t;
+    } else {
+      throw (IOException) t;
+    }
+  }
 }

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1506393&r1=1506392&r2=1506393&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
Wed Jul 24 03:48:19 2013
@@ -35,7 +35,8 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.net.NetworkTopology;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -212,9 +213,11 @@ public class Application {
     NodeManager nodeManager = task.getNodeManager();
     ContainerId containerId = task.getContainerId();
     task.stop();
-    StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
-    stopRequest.setContainerId(containerId);
-    nodeManager.stopContainer(stopRequest);
+    List<ContainerId> containerIds = new ArrayList<ContainerId>();
+    containerIds.add(containerId);
+    StopContainersRequest stopRequest =
+        StopContainersRequest.newInstance(containerIds);
+    nodeManager.stopContainers(stopRequest);
     
     Resources.subtractFrom(used, requestSpec.get(task.getPriority()));
     
@@ -339,10 +342,15 @@ public class Application {
             updateResourceRequests(requests.get(priority), type, task);
 
             // Launch the container
-            StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
-            startRequest.setContainerLaunchContext(createCLC());
-            startRequest.setContainerToken(container.getContainerToken());
-            nodeManager.startContainer(startRequest);
+            StartContainerRequest scRequest =
+                StartContainerRequest.newInstance(createCLC(),
+                  container.getContainerToken());
+            List<StartContainerRequest> list =
+                new ArrayList<StartContainerRequest>();
+            list.add(scRequest);
+            StartContainersRequest allRequests =
+                StartContainersRequest.newInstance(list);
+            nodeManager.startContainers(allRequests);
             break;
           }
         }

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1506393&r1=1506392&r2=1506393&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
Wed Jul 24 03:48:19 2013
@@ -31,12 +31,13 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -158,61 +159,60 @@ public class NodeManager implements Cont
   }
 
   @Override
-  synchronized public StartContainerResponse startContainer(
-      StartContainerRequest request) 
+  synchronized public StartContainersResponse startContainers(
+      StartContainersRequest requests) 
   throws YarnException {
 
-    Token containerToken = request.getContainerToken();
-    ContainerTokenIdentifier tokenId = null;
+    for (StartContainerRequest request : requests.getStartContainerRequests()) {
+      Token containerToken = request.getContainerToken();
+      ContainerTokenIdentifier tokenId = null;
+
+      try {
+        tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
+      } catch (IOException e) {
+        throw RPCUtil.getRemoteException(e);
+      }
 
-    try {
-      tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
-    } catch (IOException e) {
-      throw RPCUtil.getRemoteException(e);
-    }
+      ContainerId containerID = tokenId.getContainerID();
+      ApplicationId applicationId =
+          containerID.getApplicationAttemptId().getApplicationId();
+
+      List<Container> applicationContainers = containers.get(applicationId);
+      if (applicationContainers == null) {
+        applicationContainers = new ArrayList<Container>();
+        containers.put(applicationId, applicationContainers);
+      }
 
-    ContainerId containerID = tokenId.getContainerID();
-    ApplicationId applicationId =
-        containerID.getApplicationAttemptId().getApplicationId();
-
-    List<Container> applicationContainers = containers.get(applicationId);
-    if (applicationContainers == null) {
-      applicationContainers = new ArrayList<Container>();
-      containers.put(applicationId, applicationContainers);
-    }
-    
-    // Sanity check
-    for (Container container : applicationContainers) {
-      if (container.getId().compareTo(containerID)
-          == 0) {
-        throw new IllegalStateException(
-            "Container " + containerID +
-            " already setup on node " + containerManagerAddress);
+      // Sanity check
+      for (Container container : applicationContainers) {
+        if (container.getId().compareTo(containerID) == 0) {
+          throw new IllegalStateException("Container " + containerID
+              + " already setup on node " + containerManagerAddress);
+        }
       }
-    }
 
-    Container container =
-        BuilderUtils.newContainer(containerID,
-            this.nodeId, nodeHttpAddress,
-            tokenId.getResource(),
-            null, null                                 // DKDC - Doesn't matter
+      Container container =
+          BuilderUtils.newContainer(containerID, this.nodeId, nodeHttpAddress,
+            tokenId.getResource(), null, null // DKDC - Doesn't matter
             );
 
-    ContainerStatus containerStatus =
-        BuilderUtils.newContainerStatus(container.getId(), ContainerState.NEW,
-            "", -1000);
-    applicationContainers.add(container);
-    containerStatusMap.put(container, containerStatus);
-    Resources.subtractFrom(available, tokenId.getResource());
-    Resources.addTo(used, tokenId.getResource());
-    
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("startContainer:" + " node=" + containerManagerAddress
-        + " application=" + applicationId + " container=" + container
-        + " available=" + available + " used=" + used);
-    }
+      ContainerStatus containerStatus =
+          BuilderUtils.newContainerStatus(container.getId(),
+            ContainerState.NEW, "", -1000);
+      applicationContainers.add(container);
+      containerStatusMap.put(container, containerStatus);
+      Resources.subtractFrom(available, tokenId.getResource());
+      Resources.addTo(used, tokenId.getResource());
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("startContainer:" + " node=" + containerManagerAddress
+            + " application=" + applicationId + " container=" + container
+            + " available=" + available + " used=" + used);
+      }
 
-    StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class);
+    }
+    StartContainersResponse response =
+        StartContainersResponse.newInstance(null, null, null);
     return response;
   }
 
@@ -225,76 +225,79 @@ public class NodeManager implements Cont
   }
   
   @Override
-  synchronized public StopContainerResponse stopContainer(StopContainerRequest request) 
+  synchronized public StopContainersResponse stopContainers(StopContainersRequest request)

   throws YarnException {
-    ContainerId containerID = request.getContainerId();
-    String applicationId = String.valueOf(
-        containerID.getApplicationAttemptId().getApplicationId().getId());
-    
-    // Mark the container as COMPLETE
-    List<Container> applicationContainers = containers.get(applicationId);
-    for (Container c : applicationContainers) {
-      if (c.getId().compareTo(containerID) == 0) {
-        ContainerStatus containerStatus = containerStatusMap.get(c);
-        containerStatus.setState(ContainerState.COMPLETE);
-        containerStatusMap.put(c, containerStatus);
+    for (ContainerId containerID : request.getContainerIds()) {
+      String applicationId =
+          String.valueOf(containerID.getApplicationAttemptId()
+            .getApplicationId().getId());
+
+      // Mark the container as COMPLETE
+      List<Container> applicationContainers = containers.get(applicationId);
+      for (Container c : applicationContainers) {
+        if (c.getId().compareTo(containerID) == 0) {
+          ContainerStatus containerStatus = containerStatusMap.get(c);
+          containerStatus.setState(ContainerState.COMPLETE);
+          containerStatusMap.put(c, containerStatus);
+        }
       }
-    }
-    
-    // Send a heartbeat
-    try {
-      heartbeat();
-    } catch (IOException ioe) {
-      throw RPCUtil.getRemoteException(ioe);
-    }
-    
-    // Remove container and update status
-    int ctr = 0;
-    Container container = null;
-    for (Iterator<Container> i=applicationContainers.iterator(); i.hasNext();) {
-      container = i.next();
-      if (container.getId().compareTo(containerID) == 0) {
-        i.remove();
-        ++ctr;
+
+      // Send a heartbeat
+      try {
+        heartbeat();
+      } catch (IOException ioe) {
+        throw RPCUtil.getRemoteException(ioe);
       }
-    }
-    
-    if (ctr != 1) {
-      throw new IllegalStateException("Container " + containerID + 
-          " stopped " + ctr + " times!");
-    }
-    
-    Resources.addTo(available, container.getResource());
-    Resources.subtractFrom(used, container.getResource());
-
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("stopContainer:" + " node=" + containerManagerAddress
-        + " application=" + applicationId + " container=" + containerID
-        + " available=" + available + " used=" + used);
-    }
 
-    StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class);
-    return response;
+      // Remove container and update status
+      int ctr = 0;
+      Container container = null;
+      for (Iterator<Container> i = applicationContainers.iterator(); i
+        .hasNext();) {
+        container = i.next();
+        if (container.getId().compareTo(containerID) == 0) {
+          i.remove();
+          ++ctr;
+        }
+      }
+
+      if (ctr != 1) {
+        throw new IllegalStateException("Container " + containerID
+            + " stopped " + ctr + " times!");
+      }
+
+      Resources.addTo(available, container.getResource());
+      Resources.subtractFrom(used, container.getResource());
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("stopContainer:" + " node=" + containerManagerAddress
+            + " application=" + applicationId + " container=" + containerID
+            + " available=" + available + " used=" + used);
+      }
+    }
+    return StopContainersResponse.newInstance(null,null);
   }
 
   @Override
-  synchronized public GetContainerStatusResponse getContainerStatus(GetContainerStatusRequest
request) throws YarnException {
-    ContainerId containerId = request.getContainerId();
-    List<Container> appContainers = 
-        containers.get(
-            containerId.getApplicationAttemptId().getApplicationId());
-    Container container = null;
-    for (Container c : appContainers) {
-      if (c.getId().equals(containerId)) {
-        container = c;
+  synchronized public GetContainerStatusesResponse getContainerStatuses(
+      GetContainerStatusesRequest request) throws YarnException {
+    List<ContainerStatus> statuses = new ArrayList<ContainerStatus>();
+    for (ContainerId containerId : request.getContainerIds()) {
+      List<Container> appContainers =
+          containers.get(containerId.getApplicationAttemptId()
+            .getApplicationId());
+      Container container = null;
+      for (Container c : appContainers) {
+        if (c.getId().equals(containerId)) {
+          container = c;
+        }
+      }
+      if (container != null
+          && containerStatusMap.get(container).getState() != null) {
+        statuses.add(containerStatusMap.get(container));
       }
     }
-    GetContainerStatusResponse response = 
-        recordFactory.newRecordInstance(GetContainerStatusResponse.class);
-    if (container != null && containerStatusMap.get(container).getState() != null)
{
-      response.setStatus(containerStatusMap.get(container));
-    }
-    return response;
+    return GetContainerStatusesResponse.newInstance(statuses, null);
   }
 
   public static org.apache.hadoop.yarn.server.api.records.NodeStatus 

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1506393&r1=1506392&r2=1506393&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
Wed Jul 24 03:48:19 2013
@@ -36,14 +36,14 @@ import org.apache.hadoop.security.Creden
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -98,23 +98,23 @@ public class TestAMAuthorization {
     }
 
     @Override
-    public StartContainerResponse
-        startContainer(StartContainerRequest request)
+    public StartContainersResponse
+        startContainers(StartContainersRequest request)
             throws YarnException {
-      containerTokens = request.getContainerLaunchContext().getTokens();
-      return null;
+      containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens();
+      return StartContainersResponse.newInstance(null, null, null);
     }
 
     @Override
-    public StopContainerResponse stopContainer(StopContainerRequest request)
+    public StopContainersResponse stopContainers(StopContainersRequest request)
         throws YarnException {
-      return null;
+      return StopContainersResponse.newInstance(null, null);
     }
 
     @Override
-    public GetContainerStatusResponse getContainerStatus(
-        GetContainerStatusRequest request) throws YarnException {
-      return null;
+    public GetContainerStatusesResponse getContainerStatuses(
+        GetContainerStatusesRequest request) throws YarnException {
+      return GetContainerStatusesResponse.newInstance(null, null);
     }
 
     public Credentials getContainerCredentials() throws IOException {

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1506393&r1=1506392&r2=1506393&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
Wed Jul 24 03:48:19 2013
@@ -19,7 +19,10 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
@@ -27,16 +30,18 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SerializedException;
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -69,9 +74,10 @@ public class TestApplicationMasterLaunch
     int maxAppAttempts;
 
     @Override
-    public StartContainerResponse
-        startContainer(StartContainerRequest request)
+    public StartContainersResponse
+        startContainers(StartContainersRequest requests)
             throws YarnException {
+      StartContainerRequest request = requests.getStartContainerRequests().get(0);
       LOG.info("Container started by MyContainerManager: " + request);
       launched = true;
       Map<String, String> env =
@@ -95,11 +101,13 @@ public class TestApplicationMasterLaunch
           Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
       maxAppAttempts =
           Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
-      return null;
+      return StartContainersResponse.newInstance(
+        new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
+        new HashMap<ContainerId, SerializedException>());
     }
 
     @Override
-    public StopContainerResponse stopContainer(StopContainerRequest request)
+    public StopContainersResponse stopContainers(StopContainersRequest request)
         throws YarnException {
       LOG.info("Container cleaned up by MyContainerManager");
       cleanedup = true;
@@ -107,11 +115,10 @@ public class TestApplicationMasterLaunch
     }
 
     @Override
-    public GetContainerStatusResponse getContainerStatus(
-        GetContainerStatusRequest request) throws YarnException {
+    public GetContainerStatusesResponse getContainerStatuses(
+        GetContainerStatusesRequest request) throws YarnException {
       return null;
     }
-
   }
 
   @Test

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java?rev=1506393&r1=1506392&r2=1506393&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
Wed Jul 24 03:48:19 2013
@@ -19,7 +19,9 @@
 package org.apache.hadoop.yarn.server.resourcemanager.security;
 
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 import java.io.IOException;
 import java.lang.annotation.Annotation;
@@ -50,6 +52,8 @@ import org.apache.hadoop.yarn.api.Contai
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.event.Dispatcher;
@@ -158,6 +162,9 @@ public class TestClientToAMTokens {
 
     ContainerManagementProtocol containerManager =
         mock(ContainerManagementProtocol.class);
+    StartContainersResponse mockResponse = mock(StartContainersResponse.class);
+    when(containerManager.startContainers((StartContainersRequest) any()))
+      .thenReturn(mockResponse);
     final DrainDispatcher dispatcher = new DrainDispatcher();
 
     MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1506393&r1=1506392&r2=1506393&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Wed Jul 24 03:48:19 2013
@@ -23,6 +23,8 @@ import static org.junit.Assert.fail;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.List;
 
 import junit.framework.Assert;
 
@@ -32,15 +34,20 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.SerializedException;
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -328,18 +335,21 @@ public class TestContainerManagerSecurit
           ContainerId containerId,
           ApplicationAttemptId appAttemptId, NodeId nodeId,
           boolean isExceptionExpected) throws Exception {
-    GetContainerStatusRequest request =
-        Records.newRecord(GetContainerStatusRequest.class);
-    request.setContainerId(containerId);
-    
+    List<ContainerId> containerIds = new ArrayList<ContainerId>();
+    containerIds.add(containerId);
+    GetContainerStatusesRequest request =
+        GetContainerStatusesRequest.newInstance(containerIds);
     ContainerManagementProtocol proxy = null;
-    
     try {
       proxy =
           getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
               appAttemptId.toString());
-      proxy.getContainerStatus(request);
-      
+      GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request);
+      if (statuses.getFailedRequests() != null
+          && statuses.getFailedRequests().containsKey(containerId)) {
+        parseAndThrowException(statuses.getFailedRequests().get(containerId)
+          .deSerialize());
+      }
     } finally {
       if (proxy != null) {
         rpc.stopProxy(proxy, conf);
@@ -352,17 +362,21 @@ public class TestContainerManagerSecurit
       org.apache.hadoop.yarn.api.records.Token containerToken,
       NodeId nodeId, String user) throws Exception {
 
-    StartContainerRequest request =
-        Records.newRecord(StartContainerRequest.class);
-    request.setContainerToken(containerToken);
     ContainerLaunchContext context =
         Records.newRecord(ContainerLaunchContext.class);
-    request.setContainerLaunchContext(context);
-
+    StartContainerRequest scRequest =
+        StartContainerRequest.newInstance(context,containerToken);
+    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+    list.add(scRequest);
+    StartContainersRequest allRequests =
+        StartContainersRequest.newInstance(list);
     ContainerManagementProtocol proxy = null;
     try {
       proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
-      proxy.startContainer(request);
+      StartContainersResponse response = proxy.startContainers(allRequests);
+      for(SerializedException ex : response.getFailedRequests().values()){
+        parseAndThrowException(ex.deSerialize());
+      }
     } finally {
       if (proxy != null) {
         rpc.stopProxy(proxy, conf);
@@ -370,6 +384,17 @@ public class TestContainerManagerSecurit
     }
   }
 
+  private void parseAndThrowException(Throwable t) throws YarnException,
+      IOException {
+    if (t instanceof YarnException) {
+      throw (YarnException) t;
+    } else if (t instanceof InvalidToken) {
+      throw (InvalidToken) t;
+    } else {
+      throw (IOException) t;
+    }
+  }
+
   protected ContainerManagementProtocol getContainerManagementProtocolProxy(
       final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken,
       NodeId nodeId, String user) {



Mime
View raw message