hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asur...@apache.org
Subject hadoop git commit: YARN-5351. ResourceRequest should take ExecutionType into account during comparison. (Konstantinos Karanasos via asuresh)
Date Wed, 27 Jul 2016 02:08:38 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 49969b16c -> 2d8d183b1


YARN-5351. ResourceRequest should take ExecutionType into account during comparison. (Konstantinos
Karanasos via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d8d183b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d8d183b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d8d183b

Branch: refs/heads/trunk
Commit: 2d8d183b1992b82c4d8dd3d6b41a1964685d909e
Parents: 49969b1
Author: Arun Suresh <asuresh@apache.org>
Authored: Tue Jul 26 19:08:30 2016 -0700
Committer: Arun Suresh <asuresh@apache.org>
Committed: Tue Jul 26 19:08:30 2016 -0700

----------------------------------------------------------------------
 .../yarn/api/records/ExecutionTypeRequest.java  |   9 +-
 .../yarn/api/records/ResourceRequest.java       |  23 +++-
 .../yarn/client/api/impl/TestAMRMClient.java    | 131 ++++++++++++++++++-
 .../impl/pb/ExecutionTypeRequestPBImpl.java     |   5 +
 4 files changed, 159 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d8d183b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ExecutionTypeRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ExecutionTypeRequest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ExecutionTypeRequest.java
index f553a44..e3ee8b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ExecutionTypeRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ExecutionTypeRequest.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.yarn.util.Records;
  */
 @Public
 @Evolving
-public abstract class ExecutionTypeRequest {
+public abstract class ExecutionTypeRequest
+    implements Comparable<ExecutionTypeRequest> {
 
   @Public
   @Evolving
@@ -41,6 +42,12 @@ public abstract class ExecutionTypeRequest {
 
   @Public
   @Evolving
+  public static ExecutionTypeRequest newInstance(ExecutionType execType) {
+    return newInstance(execType, false);
+  }
+
+  @Public
+  @Evolving
   public static ExecutionTypeRequest newInstance(ExecutionType execType,
       boolean ensureExecutionType) {
     ExecutionTypeRequest executionTypeRequest =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d8d183b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 50a7619..07f132c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -117,6 +117,10 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest>
{
         ret = h1.compareTo(h2);
       }
       if (ret == 0) {
+        ret = r1.getExecutionTypeRequest()
+            .compareTo(r2.getExecutionTypeRequest());
+      }
+      if (ret == 0) {
         ret = r1.getCapability().compareTo(r2.getCapability());
       }
       return ret;
@@ -414,7 +418,8 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest>
{
       if (other.getExecutionTypeRequest() != null) {
         return false;
       }
-    } else if (!execTypeRequest.equals(other.getExecutionTypeRequest())) {
+    } else if (!execTypeRequest.getExecutionType()
+        .equals(other.getExecutionTypeRequest().getExecutionType())) {
       return false;
     }
     if (getNodeLabelExpression() == null) {
@@ -441,12 +446,18 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest>
{
       int hostNameComparison =
           this.getResourceName().compareTo(other.getResourceName());
       if (hostNameComparison == 0) {
-        int capabilityComparison =
-            this.getCapability().compareTo(other.getCapability());
-        if (capabilityComparison == 0) {
-          return this.getNumContainers() - other.getNumContainers();
+        int execTypeReqComparison = this.getExecutionTypeRequest()
+            .compareTo(other.getExecutionTypeRequest());
+        if (execTypeReqComparison == 0) {
+          int capabilityComparison =
+              this.getCapability().compareTo(other.getCapability());
+          if (capabilityComparison == 0) {
+            return this.getNumContainers() - other.getNumContainers();
+          } else {
+            return capabilityComparison;
+          }
         } else {
-          return capabilityComparison;
+          return execTypeReqComparison;
         }
       } else {
         return hostNameComparison;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d8d183b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 99bfca5..57cdbfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -39,7 +39,6 @@ import java.util.Set;
 import java.util.TreeSet;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
@@ -62,6 +61,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.NMToken;
@@ -120,7 +120,7 @@ public class TestAMRMClient {
   static String[] nodes;
   static String[] racks;
   private final static int DEFAULT_ITERATION = 3;
-  
+
   @BeforeClass
   public static void setup() throws Exception {
     // start minicluster
@@ -335,6 +335,133 @@ public class TestAMRMClient {
       }
     }
   }
+
+  /**
+   * Test fit of both GUARANTEED and OPPORTUNISTIC containers.
+   */
+  @Test (timeout=60000)
+  public void testAMRMClientMatchingFitExecType()
+      throws YarnException, IOException {
+    AMRMClient<ContainerRequest> amClient = null;
+    try {
+      // start am rm client
+      amClient = AMRMClient.<ContainerRequest>createAMRMClient();
+      amClient.init(conf);
+      amClient.start();
+      amClient.registerApplicationMaster("Host", 10000, "");
+
+      Resource capability1 = Resource.newInstance(1024, 2);
+      Resource capability2 = Resource.newInstance(1024, 1);
+      Resource capability3 = Resource.newInstance(1000, 2);
+      Resource capability4 = Resource.newInstance(1000, 2);
+      Resource capability5 = Resource.newInstance(2000, 2);
+      Resource capability6 = Resource.newInstance(2000, 3);
+      Resource capability7 = Resource.newInstance(6000, 3);
+
+      // Add 2 GUARANTEED and 7 OPPORTUNISTIC requests.
+      ContainerRequest storedGuarContainer1 =
+          new ContainerRequest(capability1, nodes, racks, priority);
+      ContainerRequest storedGuarContainer2 =
+          new ContainerRequest(capability2, nodes, racks, priority);
+      ContainerRequest storedOpportContainer1 =
+          new ContainerRequest(capability1, nodes, racks, priority, true, null,
+              ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC));
+      ContainerRequest storedOpportContainer2 =
+          new ContainerRequest(capability2, nodes, racks, priority, true, null,
+              ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC));
+      ContainerRequest storedOpportContainer3 =
+          new ContainerRequest(capability3, nodes, racks, priority, true, null,
+              ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC));
+      ContainerRequest storedOpportContainer4 =
+          new ContainerRequest(capability4, nodes, racks, priority, true, null,
+              ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC));
+      ContainerRequest storedOpportContainer5 =
+          new ContainerRequest(capability5, nodes, racks, priority, true, null,
+              ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC));
+      ContainerRequest storedOpportContainer6 =
+          new ContainerRequest(capability6, nodes, racks, priority, true, null,
+              ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC));
+      ContainerRequest storedOpportContainer7 =
+          new ContainerRequest(capability7, nodes, racks, priority2,
+              false, null,
+              ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC));
+      amClient.addContainerRequest(storedGuarContainer1);
+      amClient.addContainerRequest(storedGuarContainer2);
+      amClient.addContainerRequest(storedOpportContainer1);
+      amClient.addContainerRequest(storedOpportContainer2);
+      amClient.addContainerRequest(storedOpportContainer3);
+      amClient.addContainerRequest(storedOpportContainer4);
+      amClient.addContainerRequest(storedOpportContainer5);
+      amClient.addContainerRequest(storedOpportContainer6);
+      amClient.addContainerRequest(storedOpportContainer7);
+
+      // Make sure 3 entries are generated in the ask list for each added
+      // container request of a given capability, locality, execution type and
+      // priority (one node-local, one rack-local, and one ANY).
+      assertEquals(24,
+          (((AMRMClientImpl<ContainerRequest>) amClient).ask.size()));
+
+      // test exact matching of GUARANTEED containers
+      List<? extends Collection<ContainerRequest>> matches;
+      ContainerRequest storedRequest;
+      Resource testCapability1 = Resource.newInstance(1024,  2);
+      matches = amClient
+          .getMatchingRequests(priority, node, ExecutionType.GUARANTEED,
+              testCapability1);
+      verifyMatches(matches, 1);
+      storedRequest = matches.get(0).iterator().next();
+      assertEquals(storedGuarContainer1, storedRequest);
+      amClient.removeContainerRequest(storedGuarContainer1);
+
+      // test exact matching of OPPORTUNISTIC containers
+      matches = amClient.getMatchingRequests(priority, node,
+          ExecutionType.OPPORTUNISTIC, testCapability1);
+      verifyMatches(matches, 1);
+      storedRequest = matches.get(0).iterator().next();
+      assertEquals(storedOpportContainer1, storedRequest);
+      amClient.removeContainerRequest(storedOpportContainer1);
+
+      // exact OPPORTUNISTIC matching with order maintained
+      Resource testCapability2 = Resource.newInstance(1000, 2);
+      matches = amClient.getMatchingRequests(priority, node,
+          ExecutionType.OPPORTUNISTIC, testCapability2);
+      verifyMatches(matches, 2);
+      // must be returned in the order they were made
+      int i = 0;
+      for(ContainerRequest storedRequest1 : matches.get(0)) {
+        if(i++ == 0) {
+          assertEquals(storedOpportContainer3, storedRequest1);
+        } else {
+          assertEquals(storedOpportContainer4, storedRequest1);
+        }
+      }
+      amClient.removeContainerRequest(storedOpportContainer3);
+
+      // matching with larger container
+      Resource testCapability3 = Resource.newInstance(4000, 4);
+      matches = amClient.getMatchingRequests(priority, node,
+          ExecutionType.OPPORTUNISTIC, testCapability3);
+      assert(matches.size() == 4);
+
+      // verify requests without relaxed locality are only returned at specific
+      // locations
+      Resource testCapability4 = Resource.newInstance(6000, 3);
+      matches = amClient.getMatchingRequests(priority2, ResourceRequest.ANY,
+          ExecutionType.OPPORTUNISTIC, testCapability4);
+      assert(matches.size() == 0);
+      matches = amClient.getMatchingRequests(priority2, node,
+          ExecutionType.OPPORTUNISTIC, testCapability4);
+      assert(matches.size() == 1);
+
+      amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
+          null, null);
+
+    } finally {
+      if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
+        amClient.stop();
+      }
+    }
+  }
   
   private void verifyMatches(
                   List<? extends Collection<ContainerRequest>> matches,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d8d183b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ExecutionTypeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ExecutionTypeRequestPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ExecutionTypeRequestPBImpl.java
index 0037dd3..f6a5032 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ExecutionTypeRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ExecutionTypeRequestPBImpl.java
@@ -86,6 +86,11 @@ public class ExecutionTypeRequestPBImpl extends ExecutionTypeRequest {
   }
 
   @Override
+  public int compareTo(ExecutionTypeRequest other) {
+    return this.getExecutionType().compareTo(other.getExecutionType());
+  }
+
+  @Override
   public String toString() {
     return "{Execution Type: " + getExecutionType()
         + ", Enforce Execution Type: " + getEnforceExecutionType() + "}";


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message