hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject [42/50] [abbrv] hadoop git commit: YARN-6050. AMs can't be scheduled on racks or nodes (rkanter)
Date Wed, 29 Mar 2017 03:54:19 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
new file mode 100644
index 0000000..078b8fd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
@@ -0,0 +1,297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class TestRMServerUtils {
+  @Test
+  public void testGetApplicableNodeCountForAMLocality() throws Exception {
+    List<NodeId> rack1Nodes = new ArrayList<>();
+    for (int i = 0; i < 29; i++) {
+      rack1Nodes.add(NodeId.newInstance("host" + i, 1234));
+    }
+    NodeId node1 = NodeId.newInstance("node1", 1234);
+    NodeId node2 = NodeId.newInstance("node2", 1234);
+    rack1Nodes.add(node2);
+
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, false);
+    ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+    Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100);
+    Mockito.when(scheduler.getNodeIds("/rack1")).thenReturn(rack1Nodes);
+    Mockito.when(scheduler.getNodeIds("node1"))
+        .thenReturn(Collections.singletonList(node1));
+    Mockito.when(scheduler.getNodeIds("node2"))
+        .thenReturn(Collections.singletonList(node2));
+    RMContext rmContext = Mockito.mock(RMContext.class);
+    Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+
+    ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY,
+        true, null);
+    List<ResourceRequest> reqs = new ArrayList<>();
+    reqs.add(anyReq);
+    Assert.assertEquals(100,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    ResourceRequest rackReq = createResourceRequest("/rack1", true, null);
+    reqs.add(rackReq);
+    Assert.assertEquals(30,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    anyReq.setRelaxLocality(false);
+    Assert.assertEquals(30,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    rackReq.setRelaxLocality(false);
+    Assert.assertEquals(100,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    ResourceRequest node1Req = createResourceRequest("node1", false, null);
+    reqs.add(node1Req);
+    Assert.assertEquals(100,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node1Req.setRelaxLocality(true);
+    Assert.assertEquals(1,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    rackReq.setRelaxLocality(true);
+    Assert.assertEquals(31,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    ResourceRequest node2Req = createResourceRequest("node2", false, null);
+    reqs.add(node2Req);
+    Assert.assertEquals(31,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node2Req.setRelaxLocality(true);
+    Assert.assertEquals(31,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    rackReq.setRelaxLocality(false);
+    Assert.assertEquals(2,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node1Req.setRelaxLocality(false);
+    Assert.assertEquals(1,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node2Req.setRelaxLocality(false);
+    Assert.assertEquals(100,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+  }
+
+  @Test
+  public void testGetApplicableNodeCountForAMLabels() throws Exception {
+    Set<NodeId> noLabelNodes = new HashSet<>();
+    for (int i = 0; i < 80; i++) {
+      noLabelNodes.add(NodeId.newInstance("host" + i, 1234));
+    }
+    Set<NodeId> label1Nodes = new HashSet<>();
+    for (int i = 80; i < 90; i++) {
+      label1Nodes.add(NodeId.newInstance("host" + i, 1234));
+    }
+    label1Nodes.add(NodeId.newInstance("host101", 0));
+    label1Nodes.add(NodeId.newInstance("host102", 0));
+    Map<String, Set<NodeId>> label1NodesMap = new HashMap<>();
+    label1NodesMap.put("label1", label1Nodes);
+
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+    Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100);
+    RMContext rmContext = Mockito.mock(RMContext.class);
+    Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+    RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class);
+    Mockito.when(labMan.getNodesWithoutALabel()).thenReturn(noLabelNodes);
+    Mockito.when(labMan.getLabelsToNodes(Collections.singleton("label1")))
+        .thenReturn(label1NodesMap);
+    Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan);
+
+    ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY,
+        true, null);
+    List<ResourceRequest> reqs = new ArrayList<>();
+    reqs.add(anyReq);
+    Assert.assertEquals(80,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    anyReq.setNodeLabelExpression("label1");
+    Assert.assertEquals(10,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+  }
+
+  @Test
+  public void testGetApplicableNodeCountForAMLocalityAndLabels()
+      throws Exception {
+    List<NodeId> rack1Nodes = new ArrayList<>();
+    for (int i = 0; i < 29; i++) {
+      rack1Nodes.add(NodeId.newInstance("host" + i, 1234));
+    }
+    NodeId node1 = NodeId.newInstance("node1", 1234);
+    NodeId node2 = NodeId.newInstance("node2", 1234);
+    rack1Nodes.add(node2);
+    Set<NodeId> noLabelNodes = new HashSet<>();
+    for (int i = 0; i < 19; i++) {
+      noLabelNodes.add(rack1Nodes.get(i));
+    }
+    noLabelNodes.add(node2);
+    for (int i = 29; i < 89; i++) {
+      noLabelNodes.add(NodeId.newInstance("host" + i, 1234));
+    }
+    Set<NodeId> label1Nodes = new HashSet<>();
+    label1Nodes.add(node1);
+    for (int i = 89; i < 93; i++) {
+      label1Nodes.add(NodeId.newInstance("host" + i, 1234));
+    }
+    for (int i = 19; i < 29; i++) {
+      label1Nodes.add(rack1Nodes.get(i));
+    }
+    label1Nodes.add(NodeId.newInstance("host101", 0));
+    label1Nodes.add(NodeId.newInstance("host102", 0));
+    Map<String, Set<NodeId>> label1NodesMap = new HashMap<>();
+    label1NodesMap.put("label1", label1Nodes);
+
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+    Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100);
+    Mockito.when(scheduler.getNodeIds("/rack1")).thenReturn(rack1Nodes);
+    Mockito.when(scheduler.getNodeIds("node1"))
+        .thenReturn(Collections.singletonList(node1));
+    Mockito.when(scheduler.getNodeIds("node2"))
+        .thenReturn(Collections.singletonList(node2));
+    RMContext rmContext = Mockito.mock(RMContext.class);
+    Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+    RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class);
+    Mockito.when(labMan.getNodesWithoutALabel()).thenReturn(noLabelNodes);
+    Mockito.when(labMan.getLabelsToNodes(Collections.singleton("label1")))
+        .thenReturn(label1NodesMap);
+    Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan);
+
+    ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY,
+        true, null);
+    List<ResourceRequest> reqs = new ArrayList<>();
+    reqs.add(anyReq);
+    Assert.assertEquals(80,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    ResourceRequest rackReq = createResourceRequest("/rack1", true, null);
+    reqs.add(rackReq);
+    Assert.assertEquals(20,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    anyReq.setRelaxLocality(false);
+    Assert.assertEquals(20,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    rackReq.setRelaxLocality(false);
+    Assert.assertEquals(80,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    ResourceRequest node1Req = createResourceRequest("node1", false, null);
+    reqs.add(node1Req);
+    Assert.assertEquals(80,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node1Req.setRelaxLocality(true);
+    Assert.assertEquals(0,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    rackReq.setRelaxLocality(true);
+    Assert.assertEquals(20,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    ResourceRequest node2Req = createResourceRequest("node2", false, null);
+    reqs.add(node2Req);
+    Assert.assertEquals(20,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node2Req.setRelaxLocality(true);
+    Assert.assertEquals(20,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    rackReq.setRelaxLocality(false);
+    Assert.assertEquals(1,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node1Req.setRelaxLocality(false);
+    Assert.assertEquals(1,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node2Req.setRelaxLocality(false);
+    Assert.assertEquals(80,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    anyReq.setNodeLabelExpression("label1");
+    rackReq.setNodeLabelExpression("label1");
+    node1Req.setNodeLabelExpression("label1");
+    node2Req.setNodeLabelExpression("label1");
+    anyReq.setRelaxLocality(true);
+    reqs = new ArrayList<>();
+    reqs.add(anyReq);
+    Assert.assertEquals(15,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    rackReq.setRelaxLocality(true);
+    reqs.add(rackReq);
+    Assert.assertEquals(10,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    anyReq.setRelaxLocality(false);
+    Assert.assertEquals(10,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    rackReq.setRelaxLocality(false);
+    Assert.assertEquals(15,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    node1Req.setRelaxLocality(false);
+    reqs.add(node1Req);
+    Assert.assertEquals(15,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node1Req.setRelaxLocality(true);
+    Assert.assertEquals(1,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    rackReq.setRelaxLocality(true);
+    Assert.assertEquals(11,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+    node2Req.setRelaxLocality(false);
+    reqs.add(node2Req);
+    Assert.assertEquals(11,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node2Req.setRelaxLocality(true);
+    Assert.assertEquals(11,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    rackReq.setRelaxLocality(false);
+    Assert.assertEquals(1,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node1Req.setRelaxLocality(false);
+    Assert.assertEquals(0,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+    node2Req.setRelaxLocality(false);
+    Assert.assertEquals(15,
+        RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+  }
+
+  private ResourceRequest createResourceRequest(String resource,
+      boolean relaxLocality, String nodeLabel) {
+    return ResourceRequest.newInstance(Priority.newInstance(0),
+        resource, Resource.newInstance(1, 1), 1, relaxLocality, nodeLabel);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 9be52c6..5246eb7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -57,7 +57,7 @@ import com.google.common.collect.Lists;
 public abstract class MockAsm extends MockApps {
 
   public static class ApplicationBase implements RMApp {
-    ResourceRequest amReq;
+    List<ResourceRequest> amReqs;
     @Override
     public String getUser() {
       throw new UnsupportedOperationException("Not supported yet.");
@@ -204,8 +204,8 @@ public abstract class MockAsm extends MockApps {
     }
     
     @Override
-    public ResourceRequest getAMResourceRequest() {
-      return this.amReq; 
+    public List<ResourceRequest> getAMResourceRequests() {
+      return this.amReqs;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 55e93c1..7005bca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -526,7 +526,8 @@ public class TestSystemMetricsPublisher {
     when(app.getAppNodeLabelExpression()).thenCallRealMethod();
     ResourceRequest amReq = mock(ResourceRequest.class);
     when(amReq.getNodeLabelExpression()).thenReturn("high-mem");
-    when(app.getAMResourceRequest()).thenReturn(amReq);
+    when(app.getAMResourceRequests())
+        .thenReturn(Collections.singletonList(amReq));
     when(app.getAmNodeLabelExpression()).thenCallRealMethod();
     when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10));
     when(app.getCallerContext())

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 118b6bc..9290ff8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -62,14 +63,14 @@ public class MockRMApp implements RMApp {
   StringBuilder diagnostics = new StringBuilder();
   RMAppAttempt attempt;
   int maxAppAttempts = 1;
-  ResourceRequest amReq;
+  List<ResourceRequest> amReqs;
 
   public MockRMApp(int newid, long time, RMAppState newState) {
     finish = time;
     id = MockApps.newAppID(newid);
     state = newState;
-    amReq = ResourceRequest.newInstance(Priority.UNDEFINED, "0.0.0.0",
-        Resource.newInstance(0, 0), 1);
+    amReqs = Collections.singletonList(ResourceRequest.newInstance(
+        Priority.UNDEFINED, "0.0.0.0", Resource.newInstance(0, 0), 1));
   }
 
   public MockRMApp(int newid, long time, RMAppState newState, String userName) {
@@ -276,8 +277,8 @@ public class MockRMApp implements RMApp {
   }
   
   @Override
-  public ResourceRequest getAMResourceRequest() {
-    return this.amReq; 
+  public List<ResourceRequest> getAMResourceRequests() {
+    return this.amReqs;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 4884851..5aa7af9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -30,8 +30,10 @@ import static org.mockito.Mockito.verify;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
@@ -271,7 +273,8 @@ public class TestRMAppTransitions {
     submissionContext.setAMContainerSpec(mock(ContainerLaunchContext.class));
     RMApp application = new RMAppImpl(applicationId, rmContext, conf, name,
         user, queue, submissionContext, scheduler, masterService,
-        System.currentTimeMillis(), "YARN", null, mock(ResourceRequest.class));
+        System.currentTimeMillis(), "YARN", null,
+        new ArrayList<ResourceRequest>());
 
     testAppStartState(applicationId, user, name, queue, application);
     this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
@@ -1024,9 +1027,9 @@ public class TestRMAppTransitions {
             submissionContext.getQueue(), submissionContext, scheduler, null,
             appState.getSubmitTime(), submissionContext.getApplicationType(),
             submissionContext.getApplicationTags(),
-            BuilderUtils.newResourceRequest(
+            Collections.singletonList(BuilderUtils.newResourceRequest(
                 RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
-                submissionContext.getResource(), 1));
+                submissionContext.getResource(), 1)));
     Assert.assertEquals(RMAppState.NEW, application.getState());
 
     RMAppEvent recoverEvent =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index ced5bd9..9a4b6dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -328,9 +328,9 @@ public class TestRMAppAttemptTransitions {
     applicationAttempt =
         new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler,
             masterService, submissionContext, new Configuration(),
-            BuilderUtils.newResourceRequest(
+            Collections.singletonList(BuilderUtils.newResourceRequest(
                 RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
-                submissionContext.getResource(), 1), application);
+                submissionContext.getResource(), 1)), application);
 
     when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
     when(application.getApplicationId()).thenReturn(applicationId);
@@ -1108,9 +1108,9 @@ public class TestRMAppAttemptTransitions {
         new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
             spyRMContext, scheduler,masterService,
             submissionContext, myConf,
-            BuilderUtils.newResourceRequest(
+            Collections.singletonList(BuilderUtils.newResourceRequest(
                 RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
-                submissionContext.getResource(), 1), application);
+                submissionContext.getResource(), 1)), application);
 
     //submit, schedule and allocate app attempt
     myApplicationAttempt.handle(
@@ -1584,9 +1584,9 @@ public class TestRMAppAttemptTransitions {
     applicationAttempt =
         new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext,
           scheduler, masterService, submissionContext, new Configuration(),
-          BuilderUtils.newResourceRequest(
+            Collections.singletonList(BuilderUtils.newResourceRequest(
               RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
-              submissionContext.getResource(), 1), application);
+              submissionContext.getResource(), 1)), application);
     when(submissionContext.getKeepContainersAcrossApplicationAttempts())
       .thenReturn(true);
     when(submissionContext.getMaxAppAttempts()).thenReturn(1);
@@ -1645,9 +1645,10 @@ public class TestRMAppAttemptTransitions {
     applicationAttempt =
         new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
             spyRMContext, scheduler, masterService, submissionContext,
-            new Configuration(), ResourceRequest.newInstance(
-                Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3,
-                false, "label-expression"), application);
+            new Configuration(), Collections.singletonList(
+                ResourceRequest.newInstance(Priority.UNDEFINED, "host1",
+                    Resource.newInstance(3333, 1), 3,
+                false, "label-expression")), application);
     new RMAppAttemptImpl.ScheduleTransition().transition(
         (RMAppAttemptImpl) applicationAttempt, null);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index bb0a123..8aca235 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.when;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -613,7 +614,8 @@ public class TestApplicationLimits {
     ResourceRequest amResourceRequest = mock(ResourceRequest.class);
     Resource amResource = Resources.createResource(0, 0);
     when(amResourceRequest.getCapability()).thenReturn(amResource);
-    when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+    when(rmApp.getAMResourceRequests()).thenReturn(
+        Collections.singletonList(amResourceRequest));
     Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
     when(spyRMContext.getRMApps()).thenReturn(spyApps);
     RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index b70a359..0aac2ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.when;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -639,7 +640,8 @@ public class TestApplicationLimitsByPartition {
     ResourceRequest amResourceRequest = mock(ResourceRequest.class);
     Resource amResource = Resources.createResource(0, 0);
     when(amResourceRequest.getCapability()).thenReturn(amResource);
-    when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+    when(rmApp.getAMResourceRequests()).thenReturn(
+        Collections.singletonList(amResourceRequest));
     Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
     when(spyRMContext.getRMApps()).thenReturn(spyApps);
     RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 293bac2..ff0f7cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -3205,7 +3205,7 @@ public class TestCapacityScheduler {
     RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName);
 
     assertEquals("RMApp does not containes minimum allocation",
-        minAllocResource, rmApp.getAMResourceRequest().getCapability());
+        minAllocResource, rmApp.getAMResourceRequests().get(0).getCapability());
 
     ResourceScheduler scheduler = rm.getRMContext().getScheduler();
     LeafQueue queueA =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 3fbbae3..1162b9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -151,7 +151,8 @@ public class TestLeafQueue {
     amResourceRequest = mock(ResourceRequest.class);
     when(amResourceRequest.getCapability()).thenReturn(
       Resources.createResource(0, 0));
-    when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+    when(rmApp.getAMResourceRequests()).thenReturn(
+        Collections.singletonList(amResourceRequest));
     Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
     when(spyRMContext.getRMApps()).thenReturn(spyApps);
     

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index baf7434..9bf79f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3204,6 +3204,84 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     assertEquals(1, app.getLiveContainers().size());
   }
 
+  @Test
+  public void testAMStrictLocalityRack() throws IOException {
+    testAMStrictLocality(false, false);
+  }
+
+  @Test
+  public void testAMStrictLocalityNode() throws IOException {
+    testAMStrictLocality(true, false);
+  }
+
+  @Test
+  public void testAMStrictLocalityRackInvalid() throws IOException {
+    testAMStrictLocality(false, true);
+  }
+
+  @Test
+  public void testAMStrictLocalityNodeInvalid() throws IOException {
+    testAMStrictLocality(true, true);
+  }
+
+  private void testAMStrictLocality(boolean node, boolean invalid)
+      throws IOException {
+    scheduler.init(conf);
+    scheduler.start();
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1,
+        "127.0.0.1");
+    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+    scheduler.handle(nodeEvent1);
+
+    RMNode node2 = MockNodes.newNodeInfo(2, Resources.createResource(1024), 2,
+        "127.0.0.2");
+    NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+    scheduler.handle(nodeEvent2);
+
+    List<ResourceRequest> reqs = new ArrayList<>();
+    ResourceRequest nodeRequest = createResourceRequest(1024,
+        node2.getHostName(), 1, 1, true);
+    if (node && invalid) {
+      nodeRequest.setResourceName("invalid");
+    }
+    ResourceRequest rackRequest = createResourceRequest(1024,
+        node2.getRackName(), 1, 1, !node);
+    if (!node && invalid) {
+      rackRequest.setResourceName("invalid");
+    }
+    ResourceRequest anyRequest = createResourceRequest(1024,
+        ResourceRequest.ANY, 1, 1, false);
+    reqs.add(anyRequest);
+    reqs.add(rackRequest);
+    if (node) {
+      reqs.add(nodeRequest);
+    }
+
+    ApplicationAttemptId attId1 =
+        createSchedulingRequest("queue1", "user1", reqs);
+
+    scheduler.update();
+
+    NodeUpdateSchedulerEvent node2UpdateEvent =
+        new NodeUpdateSchedulerEvent(node2);
+
+    FSAppAttempt app = scheduler.getSchedulerApp(attId1);
+
+    // node2 should get the container
+    scheduler.handle(node2UpdateEvent);
+    if (invalid) {
+      assertEquals(0, app.getLiveContainers().size());
+      assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers());
+      assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+    } else {
+      assertEquals(1, app.getLiveContainers().size());
+      assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers());
+      assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+    }
+  }
+
   /**
    * Strict locality requests shouldn't reserve resources on another node.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bae6720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 30f25e9..fb9e8ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1213,8 +1213,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
     assertEquals(app1.getApplicationId().toString(), appInfo.getAppId());
     assertEquals(app1.getName(), appInfo.getName());
     assertEquals(app1.createApplicationState(), appInfo.getState());
-    assertEquals(app1.getAMResourceRequest().getCapability().getMemorySize(),
-        appInfo.getAllocatedMB());
+    assertEquals(app1.getAMResourceRequests().get(0).getCapability()
+            .getMemorySize(), appInfo.getAllocatedMB());
 
     rm.stop();
   }
@@ -1427,7 +1427,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
       expectedNumberOfElements++;
       appNodeLabelExpression = info.getString("appNodeLabelExpression");
     }
-    if (app.getAMResourceRequest().getNodeLabelExpression() != null) {
+    if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
       expectedNumberOfElements++;
       amNodeLabelExpression = info.getString("amNodeLabelExpression");
     }
@@ -1534,7 +1534,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
         app.getApplicationSubmissionContext().getNodeLabelExpression(),
         appNodeLabelExpression);
     assertEquals("unmanagedApplication doesn't match",
-        app.getAMResourceRequest().getNodeLabelExpression(),
+        app.getAMResourceRequests().get(0).getNodeLabelExpression(),
         amNodeLabelExpression);
     assertEquals("amRPCAddress",
         AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
@@ -1561,7 +1561,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
       String nodeLabelExpression, int numContainers, boolean relaxLocality,
       int priority, String resourceName, long memory, long vCores,
       String executionType, boolean enforceExecutionType) {
-    ResourceRequest request = app.getAMResourceRequest();
+    ResourceRequest request = app.getAMResourceRequests().get(0);
     assertEquals("nodeLabelExpression doesn't match",
         request.getNodeLabelExpression(), nodeLabelExpression);
     assertEquals("numContainers doesn't match", request.getNumContainers(),


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message