hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xg...@apache.org
Subject [15/21] hadoop git commit: YARN-6066. Opportunistic containers Minor fixes : API annotations, parameter name changes, checkstyles. (asuresh)
Date Mon, 09 Jan 2017 21:30:10 GMT
YARN-6066. Opportunistic containers Minor fixes : API annotations, parameter name changes,
checkstyles. (asuresh)

(cherry picked from commit 4985217de453a04ddffd7b52644bdc8d153f753c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85826f6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85826f6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85826f6c

Branch: refs/heads/YARN-5734
Commit: 85826f6ca5a6d06b711a6805f7a1a6788852db05
Parents: 679478d
Author: Arun Suresh <asuresh@apache.org>
Authored: Fri Dec 30 08:46:10 2016 -0800
Committer: Arun Suresh <asuresh@apache.org>
Committed: Sun Jan 8 09:04:20 2017 -0800

----------------------------------------------------------------------
 .../v2/app/rm/RMContainerAllocator.java         | 20 ++++++++++----------
 .../apache/hadoop/mapreduce/MRJobConfig.java    |  6 +++---
 .../hadoop/mapred/TestMROpportunisticMaps.java  |  2 +-
 .../hadoop/yarn/api/records/ContainerState.java |  2 ++
 .../hadoop/yarn/conf/YarnConfiguration.java     |  9 +++++++++
 .../site/markdown/OpportunisticContainers.md    |  4 ++--
 6 files changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index 2b63b9e..31bc380 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -237,10 +237,10 @@ public class RMContainerAllocator extends RMContainerRequestor
     // Init startTime to current time. If all goes well, it will be reset after
     // first attempt to contact RM.
     retrystartTime = System.currentTimeMillis();
-    this.scheduledRequests.setNumOpportunisticMapsPer100(
-        conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE,
-            MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE));
-    LOG.info(this.scheduledRequests.getNumOpportunisticMapsPer100() +
+    this.scheduledRequests.setNumOpportunisticMapsPercent(
+        conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT,
+            MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT));
+    LOG.info(this.scheduledRequests.getNumOpportunisticMapsPercent() +
         "% of the mappers will be scheduled using OPPORTUNISTIC containers");
   }
 
@@ -1056,14 +1056,14 @@ public class RMContainerAllocator extends RMContainerRequestor
     final Map<TaskAttemptId, ContainerRequest> maps =
       new LinkedHashMap<TaskAttemptId, ContainerRequest>();
     int mapsMod100 = 0;
-    int numOpportunisticMapsPer100 = 0;
+    int numOpportunisticMapsPercent = 0;
 
-    void setNumOpportunisticMapsPer100(int numMaps) {
-      this.numOpportunisticMapsPer100 = numMaps;
+    void setNumOpportunisticMapsPercent(int numMaps) {
+      this.numOpportunisticMapsPercent = numMaps;
     }
 
-    int getNumOpportunisticMapsPer100() {
-      return this.numOpportunisticMapsPer100;
+    int getNumOpportunisticMapsPercent() {
+      return this.numOpportunisticMapsPercent;
     }
 
     @VisibleForTesting
@@ -1110,7 +1110,7 @@ public class RMContainerAllocator extends RMContainerRequestor
         maps.put(event.getAttemptID(), request);
         addContainerReq(request);
       } else {
-        if (mapsMod100 < numOpportunisticMapsPer100) {
+        if (mapsMod100 < numOpportunisticMapsPercent) {
           request =
               new ContainerRequest(event, PRIORITY_OPPORTUNISTIC_MAP,
                   mapNodeLabelExpression);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 32bbe96..7add7df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -1005,9 +1005,9 @@ public interface MRJobConfig {
    * requested by the AM will be opportunistic. If the total number of maps
    * for the job is less than 'x', then ALL maps will be OPPORTUNISTIC
    */
-  public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE =
-      "mapreduce.job.num-opportunistic-maps-percentage";
-  public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE = 0;
+  public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENT =
+      "mapreduce.job.num-opportunistic-maps-percent";
+  public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0;
 
   /**
    * A comma-separated list of properties whose value will be redacted.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
index 462ff04..eed731f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
@@ -145,7 +145,7 @@ public class TestMROpportunisticMaps {
     job.setNumReduceTasks(numReducers);
 
     // All OPPORTUNISTIC
-    job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE, percent);
+    job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT, percent);
     job.setInt("mapreduce.map.maxattempts", 1);
     job.setInt("mapreduce.reduce.maxattempts", 1);
     job.setInt("mapred.test.num_lines", numLines);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
index 4efd8c1..696fe06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.api.records;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 
 /**
@@ -37,5 +38,6 @@ public enum ContainerState {
   COMPLETE,
 
   /** Scheduled (awaiting resources) at the NM. */
+  @InterfaceStability.Unstable
   SCHEDULED
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d0ade22..7dd5ce3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -314,6 +314,7 @@ public class YarnConfiguration extends Configuration {
 
   /** Setting that controls whether opportunistic container allocation
    *  is enabled or not. */
+  @Unstable
   public static final String OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED =
       RM_PREFIX + "opportunistic-container-allocation.enabled";
   public static final boolean
@@ -321,12 +322,14 @@ public class YarnConfiguration extends Configuration {
 
   /** Number of nodes to be used by the Opportunistic Container allocator for
    * dispatching containers during container allocation. */
+  @Unstable
   public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
       RM_PREFIX + "opportunistic-container-allocation.nodes-used";
   public static final int DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
       10;
 
   /** Frequency for computing least loaded NMs. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS =
       RM_PREFIX + "nm-container-queuing.sorting-nodes-interval-ms";
   public static final long
@@ -334,6 +337,7 @@ public class YarnConfiguration extends Configuration {
 
   /** Comparator for determining node load for scheduling of opportunistic
    * containers. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
       RM_PREFIX + "nm-container-queuing.load-comparator";
   public static final String DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
@@ -341,6 +345,7 @@ public class YarnConfiguration extends Configuration {
 
   /** Value of standard deviation used for calculation of queue limit
    * thresholds. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_LIMIT_STDEV =
       RM_PREFIX + "nm-container-queuing.queue-limit-stdev";
   public static final float DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV =
@@ -349,6 +354,7 @@ public class YarnConfiguration extends Configuration {
   /** Min length of container queue at NodeManager. This is a cluster-wide
    * configuration that acts as the lower-bound of optimal queue length
    * calculated by the NodeQueueLoadMonitor */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH =
       RM_PREFIX + "nm-container-queuing.min-queue-length";
   public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH = 5;
@@ -356,17 +362,20 @@ public class YarnConfiguration extends Configuration {
   /** Max length of container queue at NodeManager. This is a cluster-wide
    * configuration that acts as the upper-bound of optimal queue length
    * calculated by the NodeQueueLoadMonitor */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH =
       RM_PREFIX + "nm-container-queuing.max-queue-length";
   public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH = 15;
 
   /** Min queue wait time for a container at a NodeManager. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
       RM_PREFIX + "nm-container-queuing.min-queue-wait-time-ms";
   public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
       10;
 
   /** Max queue wait time for a container queue at a NodeManager. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =
       RM_PREFIX + "nm-container-queuing.max-queue-wait-time-ms";
   public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
index ac26d88..223930e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
@@ -72,10 +72,10 @@ By default, allocation of opportunistic containers is performed centrally
throug
 
 The following command can be used to run a sample pi map-reduce job, executing 40% of mappers
using opportunistic containers (substitute `3.0.0-alpha2-SNAPSHOT` below with the version
of Hadoop you are using):
 ```
-$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar
pi -Dmapreduce.job.num-opportunistic-maps-percentage="40" 50 100
+$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar
pi -Dmapreduce.job.num-opportunistic-maps-percent="40" 50 100
 ```
 
-By changing the value of `mapreduce.job.num-opportunistic-maps-percentage` in the above command,
we can specify the percentage of mappers that can be executed through opportunistic containers.
+By changing the value of `mapreduce.job.num-opportunistic-maps-percent` in the above command,
we can specify the percentage of mappers that can be executed through opportunistic containers.
 
 
 ###<a name="Opportunistic_Containers_in_Web_UI"></a>Opportunistic Containers
in Web UI


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message