hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From haiboc...@apache.org
Subject [39/50] [abbrv] hadoop git commit: YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo Chen.
Date Tue, 05 Jun 2018 19:41:57 GMT
YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo
Chen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ecf3c51f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ecf3c51f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ecf3c51f

Branch: refs/heads/YARN-1011
Commit: ecf3c51ff3a4d20cf9db845cc1270162507598b4
Parents: 52ec505
Author: Miklos Szegedi <szegedim@apache.org>
Authored: Tue Nov 28 14:01:30 2017 -0800
Committer: Haibo Chen <haibochen@apache.org>
Committed: Mon Jun 4 21:01:33 2018 -0700

----------------------------------------------------------------------
 .../hadoop/yarn/server/utils/BuilderUtils.java  | 15 +++-
 .../yarn/server/utils/TestBuilderUtils.java     | 54 ++++++++++++++
 .../server/resourcemanager/RMAppManager.java    |  6 +-
 .../server/resourcemanager/TestAppManager.java  | 76 +++++++++++++++-----
 4 files changed, 131 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecf3c51f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index c906b7e..fd9a9fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -349,15 +349,28 @@ public class BuilderUtils {
     return priority;
   }
 
+  public static ResourceRequest newAMResourceRequest(Priority priority,
+      String hostName, Resource capability) {
+    return newResourceRequest(priority, hostName, capability, 1,
+        ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
+  }
+
   public static ResourceRequest newResourceRequest(Priority priority,
       String hostName, Resource capability, int numContainers) {
+    return newResourceRequest(priority, hostName, capability,
+        numContainers, ExecutionTypeRequest.newInstance());
+  }
+
+  public static ResourceRequest newResourceRequest(Priority priority,
+      String hostName, Resource capability, int numContainers,
+      ExecutionTypeRequest executionTypeRequest) {
     ResourceRequest request = recordFactory
         .newRecordInstance(ResourceRequest.class);
     request.setPriority(priority);
     request.setResourceName(hostName);
     request.setCapability(capability);
     request.setNumContainers(numContainers);
-    request.setExecutionTypeRequest(ExecutionTypeRequest.newInstance());
+    request.setExecutionTypeRequest(executionTypeRequest);
     return request;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecf3c51f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
new file mode 100644
index 0000000..4d5ab79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.utils;
+
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Unit tests for BuilderUtils.
+ */
+public class TestBuilderUtils {
+  @Test
+  public void testNewAMResourceRequest() {
+    final Priority priority = Priority.newInstance(0);
+    final String hostName = "ResourceA";
+    final Resource capacity = Resource.newInstance(1024, 1);
+
+    ResourceRequest resourceRequest =
+        BuilderUtils.newAMResourceRequest(priority, hostName, capacity);
+
+    Assert.assertEquals("unexpected priority", priority,
+        resourceRequest.getPriority());
+    Assert.assertEquals("unexpected host name", hostName,
+        resourceRequest.getResourceName());
+    Assert.assertEquals("unexpected capacity", capacity,
+        resourceRequest.getCapability());
+    Assert.assertEquals("unexpected number of containers", 1,
+        resourceRequest.getNumContainers());
+    Assert.assertEquals("unexpected execution type",
+        ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true),
+        resourceRequest.getExecutionTypeRequest());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecf3c51f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 3e64cfc..357f1c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -483,8 +483,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
       if (amReqs == null || amReqs.isEmpty()) {
         if (submissionContext.getResource() != null) {
           amReqs = Collections.singletonList(BuilderUtils
-              .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
-                  ResourceRequest.ANY, submissionContext.getResource(), 1));
+              .newAMResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+                  ResourceRequest.ANY, submissionContext.getResource()));
         } else {
           throw new InvalidResourceRequestException("Invalid resource request, "
               + "no resources requested");
@@ -515,7 +515,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
         for (ResourceRequest amReq : amReqs) {
           amReq.setCapability(anyReq.getCapability());
           amReq.setExecutionTypeRequest(
-              ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+              ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
           amReq.setNumContainers(1);
           amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecf3c51f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index e79ba08..c95ed85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -700,15 +700,45 @@ public class TestAppManager{
   public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
     asContext.setResource(Resources.createResource(1024));
     asContext.setAMContainerResourceRequest(
-        ResourceRequest.newInstance(Priority.newInstance(0),
-            ResourceRequest.ANY, Resources.createResource(1024), 1, true));
+        ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+            .resourceName(ResourceRequest.ANY)
+            .capability(Resources.createResource(1024))
+            .numContainers(1)
+            .relaxLocality(true)
+            .executionTypeRequest(ExecutionTypeRequest.newInstance(
+                ExecutionType.GUARANTEED, true))
+        .build());
     List<ResourceRequest> reqs = new ArrayList<>();
-    reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
-        ResourceRequest.ANY, Resources.createResource(1025), 1, false));
-    reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
-        "/rack", Resources.createResource(1025), 1, false));
-    reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
-        "/rack/node", Resources.createResource(1025), 1, true));
+    reqs.add(
+        ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+            .resourceName(ResourceRequest.ANY)
+            .capability(Resources.createResource(1025))
+            .numContainers(1)
+            .relaxLocality(false)
+            .executionTypeRequest(ExecutionTypeRequest.newInstance(
+                ExecutionType.GUARANTEED, true))
+        .build()
+    );
+    reqs.add(
+        ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+            .resourceName("/rack")
+            .capability(Resources.createResource(1025))
+            .numContainers(1)
+            .relaxLocality(false)
+            .executionTypeRequest(ExecutionTypeRequest.newInstance(
+                ExecutionType.GUARANTEED, true))
+        .build()
+    );
+    reqs.add(
+        ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+            .resourceName("/rack/node")
+            .capability(Resources.createResource(1025))
+            .numContainers(1)
+            .relaxLocality(true)
+            .executionTypeRequest(ExecutionTypeRequest.newInstance(
+                ExecutionType.GUARANTEED, true))
+        .build()
+    );
     asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
     // getAMContainerResourceRequest uses the first entry of
     // getAMContainerResourceRequests
@@ -730,9 +760,15 @@ public class TestAppManager{
     asContext.setResource(Resources.createResource(1024));
     asContext.setAMContainerResourceRequests(null);
     ResourceRequest req =
-        ResourceRequest.newInstance(Priority.newInstance(0),
-            ResourceRequest.ANY, Resources.createResource(1025), 1, true);
-    req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+        ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+            .resourceName(ResourceRequest.ANY)
+            .capability(Resources.createResource(1025))
+            .numContainers(1)
+            .relaxLocality(true)
+            .nodeLabelExpression(RMNodeLabelsManager.NO_LABEL)
+            .executionTypeRequest(ExecutionTypeRequest.newInstance(
+                ExecutionType.GUARANTEED, true))
+        .build();
     asContext.setAMContainerResourceRequest(cloneResourceRequest(req));
     // getAMContainerResourceRequests uses a singleton list of
     // getAMContainerResourceRequest
@@ -753,9 +789,16 @@ public class TestAppManager{
 
     // setResource
     Assert.assertEquals(Collections.singletonList(
-        ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
-        ResourceRequest.ANY, Resources.createResource(1024), 1, true,
-            "")),
+        ResourceRequest.newBuilder()
+            .priority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY)
+            .resourceName(ResourceRequest.ANY)
+            .capability(Resources.createResource(1024))
+            .numContainers(1)
+            .relaxLocality(true)
+            .nodeLabelExpression("")
+            .executionTypeRequest(ExecutionTypeRequest.newInstance(
+                ExecutionType.GUARANTEED, true))
+        .build()),
         app.getAMResourceRequests());
   }
 
@@ -797,7 +840,7 @@ public class TestAppManager{
     for (ResourceRequest req : reqs) {
       req.setCapability(anyReq.getCapability());
       req.setExecutionTypeRequest(
-          ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+          ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
       req.setNumContainers(1);
       req.setPriority(Priority.newInstance(0));
     }
@@ -1110,7 +1153,8 @@ public class TestAppManager{
         req.getNodeLabelExpression() != null
             ? new String(req.getNodeLabelExpression()) : null,
         ExecutionTypeRequest.newInstance(
-            req.getExecutionTypeRequest().getExecutionType()));
+            req.getExecutionTypeRequest().getExecutionType(),
+            req.getExecutionTypeRequest().getEnforceExecutionType()));
   }
 
   private static List<ResourceRequest> cloneResourceRequests(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message