hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [26/49] hadoop git commit: YARN-4887. Add allocation request ID to AM-RM protocol for identifying resource-requests explicitly. (Subru Krishnan via wangda)
Date Thu, 16 Jun 2016 18:32:50 GMT
YARN-4887. Add allocation request ID to AM-RM protocol for identifying resource-requests explicitly.
(Subru Krishnan via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eec835ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eec835ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eec835ec

Branch: refs/heads/HDFS-7240
Commit: eec835ec17f77bf713d90dbc2f77dea45f3ee660
Parents: e3ba9ad
Author: Wangda Tan <wangda@apache.org>
Authored: Mon Jun 13 21:57:33 2016 -0700
Committer: Wangda Tan <wangda@apache.org>
Committed: Mon Jun 13 21:57:33 2016 -0700

----------------------------------------------------------------------
 .../hadoop/yarn/api/records/Container.java      | 45 +++++++++++++++++
 .../yarn/api/records/ResourceRequest.java       | 52 ++++++++++++++++++++
 .../src/main/proto/yarn_protos.proto            |  2 +
 .../api/records/impl/pb/ContainerPBImpl.java    | 14 ++++++
 .../records/impl/pb/ResourceRequestPBImpl.java  | 16 +++++-
 5 files changed, 128 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
index 9a62935..707a71d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api.records;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
@@ -189,4 +190,48 @@ public abstract class Container implements Comparable<Container>
{
   @Private
   @Unstable
   public abstract void setExecutionType(ExecutionType executionType);
+
+  /**
+   * Get the optional <em>ID</em> corresponding to the original {@code
+   * ResourceRequest{@link #getAllocationRequestId()}}s which is satisfied by
+   * this allocated {@code Container}.
+   * <p>
+   * The scheduler may return multiple {@code AllocateResponse}s corresponding
+   * to the same ID as and when scheduler allocates {@code Container}s.
+   * <b>Applications</b> can continue to completely ignore the returned ID in
+   * the response and use the allocation for any of their outstanding requests.
+   * <p>
+   *
+   * @return the <em>ID</em> corresponding to the original  allocation request
+   * which is satisfied by this allocation.
+   */
+  @Public
+  @Evolving
+  public long getAllocationRequestId() {
+    throw new UnsupportedOperationException();
+  }
+
+  /**
+   * Set the optional <em>ID</em> corresponding to the original {@code
+   * ResourceRequest{@link #setAllocationRequestId(long)}
+   * etAllocationRequestId()}}s which is satisfied by this allocated {@code
+   * Container}.
+   * <p>
+   * The scheduler may return multiple {@code AllocateResponse}s corresponding
+   * to the same ID as and when scheduler allocates {@code Container}s.
+   * <b>Applications</b> can continue to completely ignore the returned ID in
+   * the response and use the allocation for any of their outstanding requests.
+   * If the ID is not set, scheduler will continue to work as previously and all
+   * allocated {@code Container}(s) will have the default ID, -1.
+   * <p>
+   *
+   * @param allocationRequestID the <em>ID</em> corresponding to the original
+   *                            allocation request which is satisfied by this
+   *                            allocation.
+   */
+  @Private
+  @Evolving
+  public void setAllocationRequestId(long allocationRequestID) {
+    throw new UnsupportedOperationException();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index fbe7e58..50a7619 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -312,6 +312,58 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest>
{
   @Public
   @Evolving
   public abstract void setNodeLabelExpression(String nodelabelExpression);
+
+  /**
+   * Get the optional <em>ID</em> corresponding to this allocation request. This
+   * ID is an identifier for different {@code ResourceRequest}s from the <b>same
+   * application</b>. The allocated {@code Container}(s) received as part of the
+   * {@code AllocateResponse} response will have the ID corresponding to the
+   * original {@code ResourceRequest} for which the RM made the allocation.
+   * <p>
+   * The scheduler may return multiple {@code AllocateResponse}s corresponding
+   * to the same ID as and when scheduler allocates {@code Container}(s).
+   * <b>Applications</b> can continue to completely ignore the returned ID in
+   * the response and use the allocation for any of their outstanding requests.
+   * <p>
+   * If one wishes to replace an entire {@code ResourceRequest} corresponding to
+   * a specific ID, they can simply cancel the corresponding {@code
+   * ResourceRequest} and submit a new one afresh.
+   *
+   * @return the <em>ID</em> corresponding to this allocation request.
+   */
+  @Public
+  @Evolving
+  public long getAllocationRequestId() {
+    throw new UnsupportedOperationException();
+  }
+
+  /**
+   * Set the optional <em>ID</em> corresponding to this allocation request. This
+   * ID is an identifier for different {@code ResourceRequest}s from the <b>same
+   * application</b>. The allocated {@code Container}(s) received as part of the
+   * {@code AllocateResponse} response will have the ID corresponding to the
+   * original {@code ResourceRequest} for which the RM made the allocation.
+   * <p>
+   * The scheduler may return multiple {@code AllocateResponse}s corresponding
+   * to the same ID as and when scheduler allocates {@code Container}(s).
+   * <b>Applications</b> can continue to completely ignore the returned ID in
+   * the response and use the allocation for any of their outstanding requests.
+   * <p>
+   * If one wishes to replace an entire {@code ResourceRequest} corresponding to
+   * a specific ID, they can simply cancel the corresponding {@code
+   * ResourceRequest} and submit a new one afresh.
+   * <p>
+   * If the ID is not set, scheduler will continue to work as previously and all
+   * allocated {@code Container}(s) will have the default ID, -1.
+   *
+   * @param allocationRequestID the <em>ID</em> corresponding to this allocation
+   *                            request.
+   */
+  @Public
+  @Evolving
+  public void setAllocationRequestId(long allocationRequestID) {
+    throw new UnsupportedOperationException();
+  }
   
   @Override
   public int hashCode() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 0649f8e..c84f4e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -93,6 +93,7 @@ message ContainerProto {
   optional PriorityProto priority = 5;
   optional hadoop.common.TokenProto container_token = 6;
   optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED];
+  optional int64 allocation_request_id = 8 [default = -1];
 }
 
 message ContainerReportProto {
@@ -302,6 +303,7 @@ message ResourceRequestProto {
   optional bool relax_locality = 5 [default = true];
   optional string node_label_expression = 6;
   optional ExecutionTypeRequestProto execution_type_request = 7;
+  optional int64 allocation_request_id = 8 [default = -1];
 }
 
 message ExecutionTypeRequestProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
index bd2d937..91b3e5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
@@ -262,6 +262,18 @@ public class ContainerPBImpl extends Container {
     builder.setExecutionType(convertToProtoFormat(executionType));
   }
 
+  @Override
+  public long getAllocationRequestId() {
+    ContainerProtoOrBuilder p = viaProto ? proto : builder;
+    return (p.getAllocationRequestId());
+  }
+
+  @Override
+  public void setAllocationRequestId(long allocationRequestID) {
+    maybeInitBuilder();
+    builder.setAllocationRequestId(allocationRequestID);
+  }
+
   private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
     return new ContainerIdPBImpl(p);
   }
@@ -315,6 +327,8 @@ public class ContainerPBImpl extends Container {
     StringBuilder sb = new StringBuilder();
     sb.append("Container: [");
     sb.append("ContainerId: ").append(getId()).append(", ");
+    sb.append("AllocationRequestId: ").append(getAllocationRequestId())
+        .append(", ");
     sb.append("NodeId: ").append(getNodeId()).append(", ");
     sb.append("NodeHttpAddress: ").append(getNodeHttpAddress()).append(", ");
     sb.append("Resource: ").append(getResource()).append(", ");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index b0c4b97..9890296 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -192,6 +192,18 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
     builder.setRelaxLocality(relaxLocality);
   }
 
+  @Override
+  public long getAllocationRequestId() {
+    ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
+    return (p.getAllocationRequestId());
+  }
+
+  @Override
+  public void setAllocationRequestId(long allocationRequestID) {
+    maybeInitBuilder();
+    builder.setAllocationRequestId(allocationRequestID);
+  }
+
   private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
     return new PriorityPBImpl(p);
   }
@@ -210,7 +222,9 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
   
   @Override
   public String toString() {
-    return "{Priority: " + getPriority() + ", Capability: " + getCapability()
+    return "{AllocationRequestId: " + getAllocationRequestId()
+        + ", Priority: " + getPriority()
+        + ", Capability: " + getCapability()
         + ", # Containers: " + getNumContainers()
         + ", Location: " + getResourceName()
         + ", Relax Locality: " + getRelaxLocality()


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message