hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asur...@apache.org
Subject [04/19] hadoop git commit: YARN-4335. Allow ResourceRequests to specify ExecutionType of a request ask (kkaranasos via asuresh)
Date Thu, 19 May 2016 06:37:32 GMT
YARN-4335. Allow ResourceRequests to specify ExecutionType of a request ask (kkaranasos via
asuresh)

(cherry picked from commit 8ffabfdf4fec104cdaa68133385cfe9cf88b95a8)
(cherry picked from commit b2a654c5ee6524f81c971ea0b70e58ea0a455f1d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68964bba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68964bba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68964bba

Branch: refs/heads/branch-2
Commit: 68964bbaa2db444ac5a70ffe5f1bec240a4bfe4a
Parents: 2c6c701
Author: Arun Suresh <asuresh@apache.org>
Authored: Thu Jan 7 15:18:59 2016 -0800
Committer: Arun Suresh <asuresh@apache.org>
Committed: Wed May 18 22:02:43 2016 -0700

----------------------------------------------------------------------
 .../yarn/api/records/ResourceRequest.java       | 41 ++++++++++++++++++-
 .../src/main/proto/yarn_protos.proto            |  1 +
 .../hadoop/yarn/client/api/AMRMClient.java      | 43 ++++++++++++++++++--
 .../records/impl/pb/ResourceRequestPBImpl.java  | 21 ++++++++++
 4 files changed, 102 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/68964bba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 790120a..8c1fd8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -79,6 +79,15 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest>
{
   public static ResourceRequest newInstance(Priority priority, String hostName,
       Resource capability, int numContainers, boolean relaxLocality,
       String labelExpression) {
+    return newInstance(priority, hostName, capability, numContainers,
+        relaxLocality, labelExpression, ExecutionType.GUARANTEED);
+  }
+
+  @Public
+  @Stable
+  public static ResourceRequest newInstance(Priority priority, String hostName,
+      Resource capability, int numContainers, boolean relaxLocality, String
+      labelExpression, ExecutionType execType) {
     ResourceRequest request = Records.newRecord(ResourceRequest.class);
     request.setPriority(priority);
     request.setResourceName(hostName);
@@ -86,6 +95,7 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest>
{
     request.setNumContainers(numContainers);
     request.setRelaxLocality(relaxLocality);
     request.setNodeLabelExpression(labelExpression);
+    request.setExecutionType(execType);
     return request;
   }
 
@@ -221,7 +231,28 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest>
{
   @Public
   @Stable
   public abstract boolean getRelaxLocality();
-  
+
+  /**
+   * Set the <code>ExecutionType</code> of the requested container.
+   *
+   * @param execType
+   *          ExecutionType of the requested container
+   */
+  @Public
+  @Stable
+  public abstract void setExecutionType(ExecutionType execType);
+
+  /**
+   * Get whether locality relaxation is enabled with this
+   * <code>ResourceRequest</code>. Defaults to true.
+   *
+   * @return whether locality relaxation is enabled with this
+   * <code>ResourceRequest</code>.
+   */
+  @Public
+  @Stable
+  public abstract ExecutionType getExecutionType();
+
   /**
    * <p>For a request at a network hierarchy level, set whether locality can be relaxed
    * to that level and beyond.<p>
@@ -322,6 +353,14 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest>
{
         return false;
     } else if (!priority.equals(other.getPriority()))
       return false;
+    ExecutionType executionType = getExecutionType();
+    if (executionType == null) {
+      if (other.getExecutionType() != null) {
+        return false;
+      }
+    } else if (executionType != other.getExecutionType()) {
+      return false;
+    }
     if (getNodeLabelExpression() == null) {
       if (other.getNodeLabelExpression() != null) {
         return false;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68964bba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3546b88..e5d31dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -304,6 +304,7 @@ message ResourceRequestProto {
   optional int32 num_containers = 4;
   optional bool relax_locality = 5 [default = true];
   optional string node_label_expression = 6;
+  optional ExecutionTypeProto executionType = 7 [default = GUARANTEED];
 }
 
 enum AMCommandProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68964bba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index e0bf2d3..73c2725 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
 
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -108,6 +109,7 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest>
extends
     final Priority priority;
     final boolean relaxLocality;
     final String nodeLabelsExpression;
+    final ExecutionType executionType;
     
     /**
      * Instantiates a {@link ContainerRequest} with the given constraints and
@@ -152,6 +154,33 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest>
extends
         String[] racks, Priority priority, boolean relaxLocality) {
       this(capability, nodes, racks, priority, relaxLocality, null);
     }
+
+    /**
+     * Instantiates a {@link ContainerRequest} with the given constraints.
+     *
+     * @param capability
+     *          The {@link Resource} to be requested for each container.
+     * @param nodes
+     *          Any hosts to request that the containers are placed on.
+     * @param racks
+     *          Any racks to request that the containers are placed on. The
+     *          racks corresponding to any hosts requested will be automatically
+     *          added to this list.
+     * @param priority
+     *          The priority at which to request the containers. Higher
+     *          priorities have lower numerical values.
+     * @param relaxLocality
+     *          If true, containers for this request may be assigned on hosts
+     *          and racks other than the ones explicitly requested.
+     * @param nodeLabelsExpression
+     *          Set node labels to allocate resource, now we only support
+     *          asking for only a single node label
+     */
+    public ContainerRequest(Resource capability, String[] nodes, String[] racks,
+        Priority priority, boolean relaxLocality, String nodeLabelsExpression) {
+      this(capability, nodes, racks, priority, relaxLocality, null,
+          ExecutionType.GUARANTEED);
+    }
           
     /**
      * Instantiates a {@link ContainerRequest} with the given constraints.
@@ -173,10 +202,12 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest>
extends
      * @param nodeLabelsExpression
      *          Set node labels to allocate resource, now we only support
      *          asking for only a single node label
+     * @param executionType
+     *          Set the execution type of the container request.
      */
-    public ContainerRequest(Resource capability, String[] nodes,
-        String[] racks, Priority priority, boolean relaxLocality,
-        String nodeLabelsExpression) {
+    public ContainerRequest(Resource capability, String[] nodes, String[] racks,
+        Priority priority, boolean relaxLocality, String nodeLabelsExpression,
+        ExecutionType executionType) {
       // Validate request
       Preconditions.checkArgument(capability != null,
           "The Resource to be requested for each container " +
@@ -194,6 +225,7 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest>
extends
       this.priority = priority;
       this.relaxLocality = relaxLocality;
       this.nodeLabelsExpression = nodeLabelsExpression;
+      this.executionType = executionType;
     }
     
     public Resource getCapability() {
@@ -220,10 +252,15 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest>
extends
       return nodeLabelsExpression;
     }
     
+    public ExecutionType getExecutionType() {
+      return executionType;
+    }
+
     public String toString() {
       StringBuilder sb = new StringBuilder();
       sb.append("Capability[").append(capability).append("]");
       sb.append("Priority[").append(priority).append("]");
+      sb.append("ExecutionType[").append(executionType).append("]");
       return sb.toString();
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68964bba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index fc09cef..53ae2cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -206,4 +207,24 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
     }
     builder.setNodeLabelExpression(nodeLabelExpression);
   }
+
+  @Override
+  public ExecutionType getExecutionType() {
+    ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasExecutionType()) {
+      return null;
+    }
+    return ProtoUtils.convertFromProtoFormat(p.getExecutionType());
+  }
+
+  @Override
+  public void setExecutionType(ExecutionType execType) {
+    maybeInitBuilder();
+    if (execType == null) {
+      builder.clearExecutionType();
+      return;
+    }
+    builder.setExecutionType(ProtoUtils.convertToProtoFormat(execType));
+  }
+
 }
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message