hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From templ...@apache.org
Subject [13/50] [abbrv] hadoop git commit: YARN-5242. Update DominantResourceCalculator to consider all resource types in calculations. Contributed by Varun Vasudev.
Date Thu, 19 Oct 2017 22:40:51 GMT
YARN-5242. Update DominantResourceCalculator to consider all resource types in calculations.
Contributed by Varun Vasudev.

(cherry picked from commit 9e4ba6aff595b7a5b53d016c97c8334e10f1f2cb)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34aa5613
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34aa5613
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34aa5613

Branch: refs/heads/resource-types
Commit: 34aa5613ec2cec10d40b8fe204ae6d62153bf005
Parents: 5458369
Author: Rohith Sharma K S <rohithsharmaks@apache.org>
Authored: Tue Jul 26 14:13:03 2016 +0530
Committer: Daniel Templeton <templedf@apache.org>
Committed: Thu Oct 19 15:39:58 2017 -0700

----------------------------------------------------------------------
 .../hadoop/yarn/api/records/Resource.java       |  7 ++++++
 .../api/records/impl/pb/ResourcePBImpl.java     |  2 +-
 .../resource/DominantResourceCalculator.java    | 23 ++++++++++++++++----
 .../yarn/util/resource/ResourceUtils.java       |  5 +++--
 .../hadoop/yarn/util/resource/Resources.java    |  6 +----
 5 files changed, 31 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/34aa5613/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 1de7b2f..a25c8ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -327,6 +327,8 @@ public abstract class Resource implements Comparable<Resource> {
     otherResources = other.getResources();
     long diff = thisResources.size() - otherResources.size();
     if (diff == 0) {
+      // compare memory and vcores first(in that order) to preserve
+      // existing behaviour
       if (thisResources.keySet().equals(otherResources.keySet())) {
         diff = this.getMemorySize() - other.getMemorySize();
         if (diff == 0) {
@@ -335,6 +337,11 @@ public abstract class Resource implements Comparable<Resource>
{
         if (diff == 0) {
           for (Map.Entry<String, ResourceInformation> entry : thisResources
               .entrySet()) {
+            if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
+                || entry.getKey()
+                .equals(ResourceInformation.VCORES.getName())) {
+              continue;
+            }
             diff =
                 entry.getValue().compareTo(otherResources.get(entry.getKey()));
             if (diff != 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/34aa5613/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index ab18f2d..5f32648 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -242,7 +242,7 @@ public class ResourcePBImpl extends Resource {
         builder.addResourceValueMap(e);
       }
     }
-    builder.setMemory(this.getMemory());
+    builder.setMemory(this.getMemorySize());
     builder.setVirtualCores(this.getVirtualCores());
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/34aa5613/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 0412c0f..3c4413c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -397,10 +397,25 @@ public class DominantResourceCalculator extends ResourceCalculator {
   }
 
   @Override
-  public boolean fitsIn(Resource cluster,
-      Resource smaller, Resource bigger) {
-    return smaller.getMemorySize() <= bigger.getMemorySize()
-        && smaller.getVirtualCores() <= bigger.getVirtualCores();
+  public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) {
+    for (String resource : resourceNames) {
+      try {
+        ResourceInformation sResourceInformation =
+            smaller.getResourceInformation(resource);
+        ResourceInformation bResourceInformation =
+            bigger.getResourceInformation(resource);
+        Long sResourceValue = UnitsConversionUtil
+            .convert(sResourceInformation.getUnits(),
+                bResourceInformation.getUnits(),
+                sResourceInformation.getValue());
+        if(sResourceValue > bResourceInformation.getValue()) {
+          return false;
+        }
+      } catch (YarnException ye) {
+        return false;
+      }
+    }
+    return true;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/34aa5613/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 4f56fbb..109287a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -178,7 +178,6 @@ public class ResourceUtils {
       synchronized (ResourceUtils.class) {
         if (lock == null) {
           synchronized (ResourceUtils.class) {
-            lock = new Object();
             Map<String, ResourceInformation> resources = new HashMap<>();
             if (conf == null) {
               conf = new YarnConfiguration();
@@ -187,10 +186,12 @@ public class ResourceUtils {
               addResourcesFileToConf(resourceFile, conf);
               LOG.debug("Found " + resourceFile + ", adding to configuration");
               initializeResourcesMap(conf, resources);
+              lock = new Object();
             } catch (FileNotFoundException fe) {
               LOG.info("Unable to find '" + resourceFile
                   + "'. Falling back to memory and vcores as resources", fe);
               initializeResourcesMap(conf, resources);
+              lock = new Object();
             }
           }
         }
@@ -268,12 +269,12 @@ public class ResourceUtils {
       synchronized (ResourceUtils.class) {
         if (nodeLock == null) {
           synchronized (ResourceUtils.class) {
-            nodeLock = new Object();
             Map<String, ResourceInformation> nodeResources =
                 initializeNodeResourceInformation(conf);
             addManadtoryResources(nodeResources);
             checkMandatatoryResources(nodeResources);
             readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
+            nodeLock = new Object();
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/34aa5613/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 836f87e..73776e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -134,16 +134,12 @@ public class Resources {
 
     private Map<String, ResourceInformation> initResourceMap() {
       Map<String, ResourceInformation> tmp = new HashMap<>();
-      // Due to backwards compat, the max value for memory and vcores
-      // needs to be Integer.MAX_VALUE
-      int max = resourceValue > Integer.MAX_VALUE ? Integer.MAX_VALUE :
-          resourceValue.intValue();
       Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
       if (types != null) {
         for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
           tmp.put(entry.getKey(),
               ResourceInformation.newInstance(entry.getValue()));
-          tmp.get(entry.getKey()).setValue((long) max);
+          tmp.get(entry.getKey()).setValue(resourceValue);
         }
       }
       return tmp;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message