hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From templ...@apache.org
Subject [37/50] [abbrv] hadoop git commit: YARN-6933. [YARN-3926] ResourceUtils.DISALLOWED_NAMES check is duplicated. Contributed by Manikandan R.
Date Mon, 23 Oct 2017 18:20:38 GMT
YARN-6933. [YARN-3926] ResourceUtils.DISALLOWED_NAMES check is duplicated. Contributed by Manikandan
R.

(cherry picked from commit 805095496dfd8e50f71b70cf20845e954d3ba47c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5daaf859
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5daaf859
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5daaf859

Branch: refs/heads/resource-types
Commit: 5daaf8598e93707b3f371b57275d451122659c24
Parents: 95e7121
Author: Sunil G <sunilg@apache.org>
Authored: Wed Sep 6 18:51:14 2017 +0530
Committer: Daniel Templeton <templedf@apache.org>
Committed: Mon Oct 23 11:19:25 2017 -0700

----------------------------------------------------------------------
 .../yarn/util/resource/ResourceUtils.java       | 38 ++++++++++----------
 .../resource-types/resource-types-error-2.xml   |  4 +++
 .../resource-types/resource-types-error-3.xml   |  2 +-
 3 files changed, 23 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5daaf859/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 94c2e97..e3e25d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -42,10 +42,8 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
@@ -63,13 +61,6 @@ public class ResourceUtils {
   private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
   private static final String VCORES = ResourceInformation.VCORES.getName();
 
-  private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
-  static {
-    DISALLOWED_NAMES.add("memory");
-    DISALLOWED_NAMES.add(MEMORY);
-    DISALLOWED_NAMES.add(VCORES);
-  }
-
   private static volatile boolean initializedResources = false;
   private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
       new ConcurrentHashMap<String, Integer>();
@@ -85,9 +76,21 @@ public class ResourceUtils {
   private ResourceUtils() {
   }
 
-  private static void checkMandatatoryResources(
+  private static void checkMandatoryResources(
       Map<String, ResourceInformation> resourceInformationMap)
       throws YarnRuntimeException {
+    /*
+     * Supporting 'memory' also as invalid resource name, in addition to
+     * 'MEMORY' for historical reasons
+     */
+    String key = "memory";
+    if (resourceInformationMap.containsKey(key)) {
+      LOG.warn("Attempt to define resource '" + key +
+          "', but it is not allowed.");
+      throw new YarnRuntimeException("Attempt to re-define mandatory resource '"
+          + key + "'.");
+    }
+
     if (resourceInformationMap.containsKey(MEMORY)) {
       ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
       String memUnits = ResourceInformation.MEMORY_MB.getUnits();
@@ -113,7 +116,7 @@ public class ResourceUtils {
     }
   }
 
-  private static void addManadtoryResources(
+  private static void addMandatoryResources(
       Map<String, ResourceInformation> res) {
     ResourceInformation ri;
     if (!res.containsKey(MEMORY)) {
@@ -229,11 +232,6 @@ public class ResourceUtils {
               "Incomplete configuration for resource type '" + resourceName
                   + "'. One of name, units or type is configured incorrectly.");
         }
-        if (DISALLOWED_NAMES.contains(resourceName)) {
-          throw new YarnRuntimeException(
-              "Resource type cannot be named '" + resourceName
-                  + "'. That name is disallowed.");
-        }
         ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
         LOG.info("Adding resource type - name = " + resourceName + ", units = "
             + resourceUnits + ", type = " + resourceTypeName);
@@ -246,8 +244,8 @@ public class ResourceUtils {
                 minimumAllocation, maximumAllocation));
       }
     }
-    checkMandatatoryResources(resourceInformationMap);
-    addManadtoryResources(resourceInformationMap);
+    checkMandatoryResources(resourceInformationMap);
+    addMandatoryResources(resourceInformationMap);
     setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
     setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
     resourceTypes = Collections.unmodifiableMap(resourceInformationMap);
@@ -454,8 +452,8 @@ public class ResourceUtils {
         if (!initializedNodeResources) {
           Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
               conf);
-          addManadtoryResources(nodeResources);
-          checkMandatatoryResources(nodeResources);
+          addMandatoryResources(nodeResources);
+          checkMandatoryResources(nodeResources);
           setMinimumAllocationForMandatoryResources(nodeResources, conf);
           setMaximumAllocationForMandatoryResources(nodeResources, conf);
           readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5daaf859/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
index ca428eb..fa43b6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
@@ -26,4 +26,8 @@ limitations under the License. See accompanying LICENSE file.
    <value>G</value>
  </property>
 
+ <property>
+   <name>yarn.resource-types.vcores.units</name>
+   <value>Az</value>
+ </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5daaf859/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
index 08b8a6d..539d657 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
@@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file.
 
  <property>
    <name>yarn.resource-types</name>
-   <value>vcores,resource1</value>
+   <value>resource1,resource1</value>
  </property>
 
  <property>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message