Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id EAF64200C06 for ; Fri, 27 Jan 2017 19:41:38 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id E9802160B40; Fri, 27 Jan 2017 18:41:38 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id DBFBA160B5B for ; Fri, 27 Jan 2017 19:41:37 +0100 (CET) Received: (qmail 17462 invoked by uid 500); 27 Jan 2017 18:40:07 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 15624 invoked by uid 99); 27 Jan 2017 18:40:07 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Jan 2017 18:40:07 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1D74DDFCCC; Fri, 27 Jan 2017 18:40:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Fri, 27 Jan 2017 18:40:51 -0000 Message-Id: In-Reply-To: <7fccdec439f94cbd8d0afe3fe9340207@git.apache.org> References: <7fccdec439f94cbd8d0afe3fe9340207@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [46/50] [abbrv] hadoop git commit: YARN-5242. Update DominantResourceCalculator to consider all resource types in calculations. Contributed by Varun Vasudev. archived-at: Fri, 27 Jan 2017 18:41:39 -0000 YARN-5242. Update DominantResourceCalculator to consider all resource types in calculations. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39d2350b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39d2350b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39d2350b Branch: refs/heads/YARN-3926 Commit: 39d2350b6dc578c33e1dab6fbb1eb44715d275c6 Parents: 1d70259 Author: Rohith Sharma K S Authored: Tue Jul 26 14:13:03 2016 +0530 Committer: Varun Vasudev Committed: Fri Jan 27 23:23:36 2017 +0530 ---------------------------------------------------------------------- .../hadoop/yarn/api/records/Resource.java | 7 ++++++ .../api/records/impl/pb/ResourcePBImpl.java | 2 +- .../resource/DominantResourceCalculator.java | 23 ++++++++++++++++---- .../yarn/util/resource/ResourceUtils.java | 5 +++-- .../hadoop/yarn/util/resource/Resources.java | 6 +---- 5 files changed, 31 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/39d2350b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 3880bb0..4bbca5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -291,6 +291,8 @@ public abstract class Resource implements Comparable { otherResources = other.getResources(); long diff = thisResources.size() - otherResources.size(); if (diff == 0) { + // compare memory and vcores first(in that order) to preserve + // existing behaviour if (thisResources.keySet().equals(otherResources.keySet())) { diff = this.getMemorySize() - other.getMemorySize(); if (diff == 0) { @@ -299,6 +301,11 @@ public abstract class Resource implements Comparable { if (diff == 0) { for (Map.Entry entry : thisResources .entrySet()) { + if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) + || entry.getKey() + .equals(ResourceInformation.VCORES.getName())) { + continue; + } diff = entry.getValue().compareTo(otherResources.get(entry.getKey())); if (diff != 0) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/39d2350b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 8cf7291..36e5beb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -228,7 +228,7 @@ public class ResourcePBImpl extends Resource { builder.addResourceValueMap(e); } } - builder.setMemory(this.getMemory()); + builder.setMemory(this.getMemorySize()); builder.setVirtualCores(this.getVirtualCores()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39d2350b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 6dbe754..6c5406b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -387,9 +387,24 @@ public class DominantResourceCalculator extends ResourceCalculator { } @Override - public boolean fitsIn(Resource cluster, - Resource smaller, Resource bigger) { - return smaller.getMemorySize() <= bigger.getMemorySize() - && smaller.getVirtualCores() <= bigger.getVirtualCores(); + public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { + for (String resource : resourceNames) { + try { + ResourceInformation sResourceInformation = + smaller.getResourceInformation(resource); + ResourceInformation bResourceInformation = + bigger.getResourceInformation(resource); + Long sResourceValue = UnitsConversionUtil + .convert(sResourceInformation.getUnits(), + bResourceInformation.getUnits(), + sResourceInformation.getValue()); + if(sResourceValue > bResourceInformation.getValue()) { + return false; + } + } catch (YarnException ye) { + return false; + } + } + return true; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39d2350b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 4f56fbb..109287a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -178,7 +178,6 @@ public class ResourceUtils { synchronized (ResourceUtils.class) { if (lock == null) { synchronized (ResourceUtils.class) { - lock = new Object(); Map resources = new HashMap<>(); if (conf == null) { conf = new YarnConfiguration(); @@ -187,10 +186,12 @@ public class ResourceUtils { addResourcesFileToConf(resourceFile, conf); LOG.debug("Found " + resourceFile + ", adding to configuration"); initializeResourcesMap(conf, resources); + lock = new Object(); } catch (FileNotFoundException fe) { LOG.info("Unable to find '" + resourceFile + "'. Falling back to memory and vcores as resources", fe); initializeResourcesMap(conf, resources); + lock = new Object(); } } } @@ -268,12 +269,12 @@ public class ResourceUtils { synchronized (ResourceUtils.class) { if (nodeLock == null) { synchronized (ResourceUtils.class) { - nodeLock = new Object(); Map nodeResources = initializeNodeResourceInformation(conf); addManadtoryResources(nodeResources); checkMandatatoryResources(nodeResources); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); + nodeLock = new Object(); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39d2350b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 12a91ef..a597295 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -134,16 +134,12 @@ public class Resources { private Map initResourceMap() { Map tmp = new HashMap<>(); - // Due to backwards compat, the max value for memory and vcores - // needs to be Integer.MAX_VALUE - int max = resourceValue > Integer.MAX_VALUE ? Integer.MAX_VALUE : - resourceValue.intValue(); Map types = ResourceUtils.getResourceTypes(); if (types != null) { for (Map.Entry entry : types.entrySet()) { tmp.put(entry.getKey(), ResourceInformation.newInstance(entry.getValue())); - tmp.get(entry.getKey()).setValue((long) max); + tmp.get(entry.getKey()).setValue(resourceValue); } } return tmp; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org