hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [48/50] [abbrv] hadoop git commit: YARN-6953. Clean up ResourceUtils.setMinimumAllocationForMandatoryResources() and setMaximumAllocationForMandatoryResources() (Contributed by Manikandan R via Daniel Templeton)
Date Wed, 15 Nov 2017 19:55:43 GMT
YARN-6953. Clean up ResourceUtils.setMinimumAllocationForMandatoryResources() and setMaximumAllocationForMandatoryResources()
(Contributed by Manikandan R via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e094eb74
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e094eb74
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e094eb74

Branch: refs/heads/HDFS-7240
Commit: e094eb74b9e7d8c3c6f1990445d248b062cc230b
Parents: 69043ba
Author: Daniel Templeton <templedf@apache.org>
Authored: Wed Nov 15 09:55:40 2017 -0800
Committer: Daniel Templeton <templedf@apache.org>
Committed: Wed Nov 15 10:03:29 2017 -0800

----------------------------------------------------------------------
 .../yarn/util/resource/ResourceUtils.java       | 108 +++++++------------
 1 file changed, 38 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e094eb74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 3deace8..c168337 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -142,74 +142,44 @@ public class ResourceUtils {
     }
   }
 
-  private static void setMinimumAllocationForMandatoryResources(
+  private static void setAllocationForMandatoryResources(
       Map<String, ResourceInformation> res, Configuration conf) {
-    String[][] resourceTypesKeys = {
-        {ResourceInformation.MEMORY_MB.getName(),
-            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
-            String.valueOf(
-                YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
-            ResourceInformation.MEMORY_MB.getName()},
-        {ResourceInformation.VCORES.getName(),
-            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
-            String.valueOf(
-                YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
-            ResourceInformation.VCORES.getName()}};
-    for (String[] arr : resourceTypesKeys) {
-      String resourceTypesKey =
-          YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
-      long minimumResourceTypes = conf.getLong(resourceTypesKey, -1);
-      long minimumConf = conf.getLong(arr[1], -1);
-      long minimum;
-      if (minimumResourceTypes != -1) {
-        minimum = minimumResourceTypes;
-        if (minimumConf != -1) {
-          LOG.warn("Using minimum allocation for memory specified in "
-              + "resource-types config file with key "
-              + minimumResourceTypes + ", ignoring minimum specified using "
-              + arr[1]);
-        }
-      } else {
-        minimum = conf.getLong(arr[1], Long.parseLong(arr[2]));
-      }
-      ResourceInformation ri = res.get(arr[3]);
-      ri.setMinimumAllocation(minimum);
-    }
-  }
-
-  private static void setMaximumAllocationForMandatoryResources(
-      Map<String, ResourceInformation> res, Configuration conf) {
-    String[][] resourceTypesKeys = {
-        {ResourceInformation.MEMORY_MB.getName(),
-            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-            String.valueOf(
-                YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
-            ResourceInformation.MEMORY_MB.getName()},
-        {ResourceInformation.VCORES.getName(),
-            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-            String.valueOf(
-                YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
-            ResourceInformation.VCORES.getName()}};
-    for (String[] arr : resourceTypesKeys) {
-      String resourceTypesKey =
-          YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
-      long maximumResourceTypes = conf.getLong(resourceTypesKey, -1);
-      long maximumConf = conf.getLong(arr[1], -1);
-      long maximum;
-      if (maximumResourceTypes != -1) {
-        maximum = maximumResourceTypes;
-        if (maximumConf != -1) {
-          LOG.warn("Using maximum allocation for memory specified in "
-              + "resource-types config file with key "
-              + maximumResourceTypes + ", ignoring maximum specified using "
-              + arr[1]);
-        }
-      } else {
-        maximum = conf.getLong(arr[1], Long.parseLong(arr[2]));
-      }
-      ResourceInformation ri = res.get(arr[3]);
-      ri.setMaximumAllocation(maximum);
+    ResourceInformation mem = res.get(ResourceInformation.MEMORY_MB.getName());
+    mem.setMinimumAllocation(getAllocation(conf,
+        YarnConfiguration.RESOURCE_TYPES + "." +
+            mem.getName() + MINIMUM_ALLOCATION,
+        YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
+    mem.setMaximumAllocation(getAllocation(conf,
+        YarnConfiguration.RESOURCE_TYPES + "." +
+            mem.getName() + MAXIMUM_ALLOCATION,
+        YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
+
+    ResourceInformation cpu = res.get(ResourceInformation.VCORES.getName());
+
+    cpu.setMinimumAllocation(getAllocation(conf,
+        YarnConfiguration.RESOURCE_TYPES + "." +
+            cpu.getName() + MINIMUM_ALLOCATION,
+        YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES));
+    cpu.setMaximumAllocation(getAllocation(conf,
+        YarnConfiguration.RESOURCE_TYPES + "." +
+        cpu.getName() + MAXIMUM_ALLOCATION,
+        YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES));
+  }
+
+  private static long getAllocation(Configuration conf,
+      String resourceTypesKey, String schedulerKey, long schedulerDefault) {
+    long value = conf.getLong(resourceTypesKey, -1L);
+    if (value == -1) {
+      LOG.debug("Mandatory Resource '" + resourceTypesKey + "' is not "
+          + "configured in resource-types config file. Setting allocation "
+          + "specified using '" + schedulerKey + "'");
+      value = conf.getLong(schedulerKey, schedulerDefault);
     }
+    return value;
   }
 
   @VisibleForTesting
@@ -275,8 +245,7 @@ public class ResourceUtils {
     checkMandatoryResources(resourceInformationMap);
     addMandatoryResources(resourceInformationMap);
 
-    setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
-    setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
+    setAllocationForMandatoryResources(resourceInformationMap, conf);
 
     initializeResourcesFromResourceInformationMap(resourceInformationMap);
   }
@@ -473,8 +442,7 @@ public class ResourceUtils {
               conf);
           addMandatoryResources(nodeResources);
           checkMandatoryResources(nodeResources);
-          setMinimumAllocationForMandatoryResources(nodeResources, conf);
-          setMaximumAllocationForMandatoryResources(nodeResources, conf);
+          setAllocationForMandatoryResources(nodeResources, conf);
           readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
           initializedNodeResources = true;
         }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message