hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asur...@apache.org
Subject [4/4] hadoop git commit: YARN-2885. Create AMRMProxy request interceptor and ContainerAllocator to distribute OPPORTUNISTIC containers to appropriate Nodes (asuresh)
Date Thu, 11 Feb 2016 16:56:28 GMT
YARN-2885. Create AMRMProxy request interceptor and ContainerAllocator to distribute OPPORTUNISTIC
containers to appropriate Nodes (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e127296e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e127296e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e127296e

Branch: refs/heads/yarn-2877
Commit: e127296e91032a4d35360856e929a88f70cc963f
Parents: 3006e23
Author: Arun Suresh <asuresh@apache.org>
Authored: Thu Feb 11 08:52:59 2016 -0800
Committer: Arun Suresh <asuresh@apache.org>
Committed: Thu Feb 11 08:52:59 2016 -0800

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES-yarn-2877.txt         |  3 +++
 .../hadoop/yarn/conf/YarnConfiguration.java       | 18 +++++++++---------
 .../yarn/api/records/impl/pb/ProtoUtils.java      |  6 ++++--
 .../server/api/DistributedSchedulerProtocol.java  |  7 ++++---
 .../hadoop/yarn/server/api/ServerRMProxy.java     |  2 +-
 .../amrmproxy/AbstractRequestInterceptor.java     |  4 ++--
 .../nodemanager/amrmproxy/RequestInterceptor.java |  1 -
 .../OpportunisticContainerAllocator.java          |  3 ++-
 8 files changed, 25 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e127296e/hadoop-yarn-project/CHANGES-yarn-2877.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES-yarn-2877.txt b/hadoop-yarn-project/CHANGES-yarn-2877.txt
index d29ff0f..a147866 100644
--- a/hadoop-yarn-project/CHANGES-yarn-2877.txt
+++ b/hadoop-yarn-project/CHANGES-yarn-2877.txt
@@ -13,3 +13,6 @@ yarn-2877 distributed scheduling (Unreleased)
     YARN-4335. Allow ResourceRequests to specify ExecutionType of a request
     ask (kkaranasos via asuresh)
 
+    YARN-2885. Create AMRMProxy request interceptor and ContainerAllocator
+    to distribute OPPORTUNISTIC containers to appropriate Nodes (asuresh)
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e127296e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9b54675..edae3eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -291,43 +291,43 @@ public class YarnConfiguration extends Configuration {
   /** ACL used in case none is found. Allows nothing. */
   public static final String DEFAULT_YARN_APP_ACL = " ";
 
-  /** Is Distributed Scheduling Enabled */
+  /** Is Distributed Scheduling Enabled. */
   public static String DIST_SCHEDULING_ENABLED =
       YARN_PREFIX + "distributed-scheduling.enabled";
   public static final boolean DIST_SCHEDULING_ENABLED_DEFAULT = false;
 
-  /** Mininum allocatable container memory for Distributed Scheduling */
+  /** Mininum allocatable container memory for Distributed Scheduling. */
   public static String DIST_SCHEDULING_MIN_MEMORY =
       YARN_PREFIX + "distributed-scheduling.min-memory";
   public static final int DIST_SCHEDULING_MIN_MEMORY_DEFAULT = 512;
 
-  /** Mininum allocatable container vcores for Distributed Scheduling */
+  /** Mininum allocatable container vcores for Distributed Scheduling. */
   public static String DIST_SCHEDULING_MIN_VCORES =
       YARN_PREFIX + "distributed-scheduling.min-vcores";
   public static final int DIST_SCHEDULING_MIN_VCORES_DEFAULT = 1;
 
-  /** Maximum allocatable container memory for Distributed Scheduling */
+  /** Maximum allocatable container memory for Distributed Scheduling. */
   public static String DIST_SCHEDULING_MAX_MEMORY =
       YARN_PREFIX + "distributed-scheduling.max-memory";
   public static final int DIST_SCHEDULING_MAX_MEMORY_DEFAULT = 2048;
 
-  /** Maximum allocatable container vcores for Distributed Scheduling */
+  /** Maximum allocatable container vcores for Distributed Scheduling. */
   public static String DIST_SCHEDULING_MAX_VCORES =
       YARN_PREFIX + "distributed-scheduling.max-vcores";
   public static final int DIST_SCHEDULING_MAX_VCORES_DEFAULT = 4;
 
-  /** Incremental allocatable container memory for Distributed Scheduling */
+  /** Incremental allocatable container memory for Distributed Scheduling. */
   public static String DIST_SCHEDULING_INCR_MEMORY =
       YARN_PREFIX + "distributed-scheduling.incr-memory";
   public static final int DIST_SCHEDULING_INCR_MEMORY_DEFAULT = 512;
 
-  /** Incremental allocatable container vcores for Distributed Scheduling */
+  /** Incremental allocatable container vcores for Distributed Scheduling. */
   public static String DIST_SCHEDULING_INCR_VCORES =
       YARN_PREFIX + "distributed-scheduling.incr-vcores";
   public static final int DIST_SCHEDULING_INCR_VCORES_DEFAULT = 1;
 
-  /** Container token expiry for container allocated via
-   * Distributed Scheduling */
+  /** Container token expiry for container allocated via Distributed
+   * Scheduling. */
   public static String DIST_SCHEDULING_CONTAINER_TOKEN_EXPIRY_MS =
       YARN_PREFIX + "distributed-scheduling.container-token-expiry";
   public static final int DIST_SCHEDULING_CONTAINER_TOKEN_EXPIRY_MS_DEFAULT =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e127296e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index ccdfc64..9d683f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -300,11 +300,13 @@ public class ProtoUtils {
   /*
    * Resource
    */
-  public static synchronized YarnProtos.ResourceProto convertToProtoFormat(Resource r) {
+  public static synchronized YarnProtos.ResourceProto convertToProtoFormat(
+      Resource r) {
     return ((ResourcePBImpl) r).getProto();
   }
 
-  public static Resource convertFromProtoFormat(YarnProtos.ResourceProto resource) {
+  public static Resource convertFromProtoFormat(
+      YarnProtos.ResourceProto resource) {
     return new ResourcePBImpl(resource);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e127296e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocol.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocol.java
index 47de8cb..490c25b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocol.java
@@ -38,11 +38,12 @@ import java.io.IOException;
  * required to perform Distributed Scheduling.
  * </p>
  */
-public interface DistributedSchedulerProtocol extends ApplicationMasterProtocol {
+public interface DistributedSchedulerProtocol
+    extends ApplicationMasterProtocol {
 
   /**
    * <p> Extends the <code>registerApplicationMaster</code> to wrap the
response
-   * with additional metadata</p>
+   * with additional metadata.</p>
    *
    * @param request ApplicationMaster registration request
    * @return A <code>DistSchedRegisterResponse</code> that contains a standard
@@ -60,7 +61,7 @@ public interface DistributedSchedulerProtocol extends ApplicationMasterProtocol
 
   /**
    * <p> Extends the <code>allocate</code> to wrap the response with additional
-   * metadata</p>
+   * metadata.</p>
    *
    * @param request ApplicationMaster allocate request
    * @return A <code>DistSchedAllocateResponse</code> that contains a standard

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e127296e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
index e3ac924..c23e27c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
@@ -78,7 +78,7 @@ public class ServerRMProxy<T> extends RMProxy<T> {
         YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
-    } else if (protocol == DistributedSchedulerProtocol.class ) {
+    } else if (protocol == DistributedSchedulerProtocol.class) {
       return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
           YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
           YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e127296e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AbstractRequestInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AbstractRequestInterceptor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AbstractRequestInterceptor.java
index 2cf185f..36bc5ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AbstractRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AbstractRequestInterceptor.java
@@ -110,7 +110,7 @@ public abstract class AbstractRequestInterceptor implements
 
   /**
    * Default implementation that invokes the distributed scheduling version
-   * of the register method
+   * of the register method.
    *
    * @param request ApplicationMaster allocate request
    * @return
@@ -126,7 +126,7 @@ public abstract class AbstractRequestInterceptor implements
 
   /**
    * Default implementation that invokes the distributed scheduling version
-   * of the allocate method
+   * of the allocate method.
    *
    * @param request ApplicationMaster registration request
    * @return

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e127296e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/RequestInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/RequestInterceptor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/RequestInterceptor.java
index 8ec2b4d..7a73563 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/RequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/RequestInterceptor.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
 
 import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocol;
 
 /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e127296e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java
index 1fd6529..7b2a258 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java
@@ -108,8 +108,9 @@ public class OpportunisticContainerAllocator {
     List<String> topKNodesLeft = new ArrayList<>();
     for (String s : allNodes.keySet()) {
       // Bias away from whatever we have already allocated and respect blacklist
-      if (nodesAllocated.contains(s) || blacklist.contains(s))
+      if (nodesAllocated.contains(s) || blacklist.contains(s)) {
         continue;
+      }
       topKNodesLeft.add(s);
     }
     int numAllocated = 0;


Mime
View raw message