hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rkan...@apache.org
Subject hadoop git commit: MAPREDUCE-6677. LocalContainerAllocator doesn't specify resource of the containers allocated (haibochen via rkanter)
Date Thu, 05 May 2016 05:39:40 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 5b5383317 -> cca8e9b90


MAPREDUCE-6677. LocalContainerAllocator doesn't specify resource of the containers allocated
(haibochen via rkanter)

(cherry picked from commit ecce3b7d53ab624c9931273716326c76641ea11d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cca8e9b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cca8e9b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cca8e9b9

Branch: refs/heads/branch-2
Commit: cca8e9b90c38680b5268126554e758d83523fd67
Parents: 5b53833
Author: Robert Kanter <rkanter@apache.org>
Authored: Wed May 4 22:19:20 2016 -0700
Committer: Robert Kanter <rkanter@apache.org>
Committed: Wed May 4 22:19:42 2016 -0700

----------------------------------------------------------------------
 .../v2/app/local/LocalContainerAllocator.java   |  2 +
 .../app/local/TestLocalContainerAllocator.java  | 43 ++++++++++++++++++++
 .../apache/hadoop/mapreduce/v2/TestMRJobs.java  |  9 ----
 3 files changed, 45 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cca8e9b9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
index 7437357..269e5f4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.client.ClientRMProxy;
@@ -177,6 +178,7 @@ public class LocalContainerAllocator extends RMCommunicator
       Container container = recordFactory.newRecordInstance(Container.class);
       container.setId(cID);
       NodeId nodeId = NodeId.newInstance(this.nmHost, this.nmPort);
+      container.setResource(Resource.newInstance(0, 0));
       container.setNodeId(nodeId);
       container.setContainerToken(null);
       container.setNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cca8e9b9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
index 38df8f0..0f7dc87 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapreduce.v2.app.local;
 
 import static org.mockito.Matchers.isA;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import java.io.IOException;
@@ -29,10 +31,15 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.app.ClusterInfo;
 import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
 import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -46,12 +53,14 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -60,6 +69,7 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
+import org.mockito.ArgumentCaptor;
 
 public class TestLocalContainerAllocator {
 
@@ -170,6 +180,39 @@ public class TestLocalContainerAllocator {
         ugiToken.getService());
   }
 
+  @Test
+  public void testAllocatedContainerResourceIsNotNull() {
+    ArgumentCaptor<TaskAttemptContainerAssignedEvent> containerAssignedCaptor
+        = ArgumentCaptor.forClass(TaskAttemptContainerAssignedEvent.class);
+    @SuppressWarnings("unchecked")
+    EventHandler<Event> eventHandler = mock(EventHandler.class);
+    AppContext context = mock(AppContext.class) ;
+    when(context.getEventHandler()).thenReturn(eventHandler);
+    ContainerId containerId = ContainerId.fromString(
+        "container_1427562107907_0002_01_000001");
+    LocalContainerAllocator containerAllocator = new LocalContainerAllocator(
+        mock(ClientService.class), context, "localhost", -1, -1, containerId);
+
+    ContainerAllocatorEvent containerAllocatorEvent =
+        createContainerRequestEvent();
+    containerAllocator.handle(containerAllocatorEvent);
+
+    verify(eventHandler, times(1)).handle(containerAssignedCaptor.capture());
+    Container container = containerAssignedCaptor.getValue().getContainer();
+    Resource containerResource = container.getResource();
+    Assert.assertNotNull(containerResource);
+    Assert.assertEquals(containerResource.getMemory(), 0);
+    Assert.assertEquals(containerResource.getVirtualCores(), 0);
+  }
+
+  private static ContainerAllocatorEvent createContainerRequestEvent() {
+    TaskAttemptId taskAttemptId = mock(TaskAttemptId.class);
+    TaskId taskId = mock(TaskId.class);
+    when(taskAttemptId.getTaskId()).thenReturn(taskId);
+    return new ContainerAllocatorEvent(taskAttemptId,
+        ContainerAllocator.EventType.CONTAINER_REQ);
+  }
+
   private static class StubbedLocalContainerAllocator
     extends LocalContainerAllocator {
     private ApplicationMasterProtocol scheduler;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cca8e9b9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
index e685c65..ba05d9d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
@@ -471,12 +471,6 @@ public class TestMRJobs {
         .getValue());
     Assert.assertEquals(numSleepReducers,
         counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES).getValue());
-    Assert
-        .assertTrue(counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS) != null
-            && counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue() != 0);
-    Assert
-        .assertTrue(counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS) != null
-            && counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue() != 0);
   }
   
   protected void verifyTaskProgress(Job job) throws InterruptedException,
@@ -548,9 +542,6 @@ public class TestMRJobs {
         .getValue());
     Assert.assertEquals(3, counters.findCounter(JobCounter.TOTAL_LAUNCHED_MAPS)
         .getValue());
-    Assert
-        .assertTrue(counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS) != null
-            && counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue() != 0);
   }
 
   @Test (timeout = 60000)


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message