Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 9F40A200CBA for ; Mon, 3 Jul 2017 10:54:31 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 9DAF5160C04; Mon, 3 Jul 2017 08:54:31 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id F028B160C03 for ; Mon, 3 Jul 2017 10:54:29 +0200 (CEST) Received: (qmail 95032 invoked by uid 500); 3 Jul 2017 08:54:15 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 92982 invoked by uid 99); 3 Jul 2017 08:54:14 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Jul 2017 08:54:14 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DB6E3F5528; Mon, 3 Jul 2017 08:54:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Mon, 03 Jul 2017 08:55:00 -0000 Message-Id: In-Reply-To: <4838ee3ab1d94bbfa5eff70a35861707@git.apache.org> References: <4838ee3ab1d94bbfa5eff70a35861707@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [50/50] [abbrv] hadoop git commit: YARN-6445. [YARN-3926] Performance improvements in resource profile branch with respect to SLS. Contributed by Varun Vasudev. archived-at: Mon, 03 Jul 2017 08:54:31 -0000 YARN-6445. [YARN-3926] Performance improvements in resource profile branch with respect to SLS. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1acec64d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1acec64d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1acec64d Branch: refs/heads/YARN-3926 Commit: 1acec64d86a7ee79591a937d24980fb796f3089c Parents: c607f6d Author: Sunil G Authored: Tue Apr 25 11:53:11 2017 +0530 Committer: Varun Vasudev Committed: Mon Jul 3 12:43:04 2017 +0530 ---------------------------------------------------------------------- .../hadoop/yarn/api/records/Resource.java | 19 +++++-- .../yarn/api/records/ResourceInformation.java | 51 ++++++++++------- .../hadoop/yarn/util/UnitsConversionUtil.java | 34 ++++++----- .../yarn/conf/TestResourceInformation.java | 4 +- .../yarn/util/TestUnitsConversionUtil.java | 60 ++++++++++---------- .../api/records/impl/pb/ResourcePBImpl.java | 23 ++++---- .../resource/DominantResourceCalculator.java | 54 ++++++++---------- .../hadoop/yarn/util/resource/Resources.java | 18 +++--- .../yarn/util/resource/TestResourceUtils.java | 1 + .../yarn/util/resource/TestResources.java | 12 +++- .../resource/ResourceProfilesManagerImpl.java | 3 +- 11 files changed, 157 insertions(+), 122 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index c349a32..4356986 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -107,12 +107,23 @@ public abstract class Resource implements Comparable { @InterfaceStability.Unstable public static Resource newInstance(Resource resource) { Resource ret = Resource.newInstance(0, 0); - for (Map.Entry entry : resource.getResources() + Resource.copy(resource, ret); + return ret; + } + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static void copy(Resource source, Resource dest) { + for (Map.Entry entry : source.getResources() .entrySet()) { - ret.setResourceInformation(entry.getKey(), - ResourceInformation.newInstance(entry.getValue())); + try { + ResourceInformation.copy(entry.getValue(), + dest.getResourceInformation(entry.getKey())); + } catch (YarnException ye) { + dest.setResourceInformation(entry.getKey(), + ResourceInformation.newInstance(entry.getValue())); + } } - return ret; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 7d74efc..d75b441 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -30,9 +30,9 @@ public class ResourceInformation implements Comparable { private String name; private String units; private ResourceTypes resourceType; - private Long value; - private Long minimumAllocation; - private Long maximumAllocation; + private long value; + private long minimumAllocation; + private long maximumAllocation; private static final String MEMORY_URI = "memory-mb"; private static final String VCORES_URI = "vcores"; @@ -106,7 +106,7 @@ public class ResourceInformation implements Comparable { * * @return the resource value */ - public Long getValue() { + public long getValue() { return value; } @@ -115,7 +115,7 @@ public class ResourceInformation implements Comparable { * * @param rValue the resource value */ - public void setValue(Long rValue) { + public void setValue(long rValue) { this.value = rValue; } @@ -124,7 +124,7 @@ public class ResourceInformation implements Comparable { * * @return the minimum allocation for the resource */ - public Long getMinimumAllocation() { + public long getMinimumAllocation() { return minimumAllocation; } @@ -133,7 +133,7 @@ public class ResourceInformation implements Comparable { * * @param minimumAllocation the minimum allocation for the resource */ - public void setMinimumAllocation(Long minimumAllocation) { + public void setMinimumAllocation(long minimumAllocation) { this.minimumAllocation = minimumAllocation; } @@ -142,7 +142,7 @@ public class ResourceInformation implements Comparable { * * @return the maximum allocation for the resource */ - public Long getMaximumAllocation() { + public long getMaximumAllocation() { return maximumAllocation; } @@ -151,7 +151,7 @@ public class ResourceInformation implements Comparable { * * @param maximumAllocation the maximum allocation for the resource */ - public void setMaximumAllocation(Long maximumAllocation) { + public void setMaximumAllocation(long maximumAllocation) { this.maximumAllocation = maximumAllocation; } @@ -163,18 +163,13 @@ public class ResourceInformation implements Comparable { */ public static ResourceInformation newInstance(ResourceInformation other) { ResourceInformation ret = new ResourceInformation(); - ret.setName(other.getName()); - ret.setResourceType(other.getResourceType()); - ret.setUnits(other.getUnits()); - ret.setValue(other.getValue()); - ret.setMinimumAllocation(other.getMinimumAllocation()); - ret.setMaximumAllocation(other.getMaximumAllocation()); + copy(other, ret); return ret; } public static ResourceInformation newInstance(String name, String units, - Long value, ResourceTypes type, Long minimumAllocation, - Long maximumAllocation) { + long value, ResourceTypes type, long minimumAllocation, + long maximumAllocation) { ResourceInformation ret = new ResourceInformation(); ret.setName(name); ret.setResourceType(type); @@ -186,7 +181,7 @@ public class ResourceInformation implements Comparable { } public static ResourceInformation newInstance(String name, String units, - Long value) { + long value) { return ResourceInformation .newInstance(name, units, value, ResourceTypes.COUNTABLE, 0L, Long.MAX_VALUE); @@ -198,7 +193,7 @@ public class ResourceInformation implements Comparable { Long.MAX_VALUE); } - public static ResourceInformation newInstance(String name, Long value) { + public static ResourceInformation newInstance(String name, long value) { return ResourceInformation .newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L, Long.MAX_VALUE); @@ -208,6 +203,22 @@ public class ResourceInformation implements Comparable { return ResourceInformation.newInstance(name, ""); } + /** + * Copies the content of the source ResourceInformation object to the + * destination object, overwriting all properties of the destination object. + * @param src Source ResourceInformation object + * @param dst Destination ResourceInformation object + */ + + public static void copy(ResourceInformation src, ResourceInformation dst) { + dst.setName(src.getName()); + dst.setResourceType(src.getResourceType()); + dst.setUnits(src.getUnits()); + dst.setValue(src.getValue()); + dst.setMinimumAllocation(src.getMinimumAllocation()); + dst.setMaximumAllocation(src.getMaximumAllocation()); + } + @Override public String toString() { return "name: " + this.name + ", units: " + this.units + ", type: " @@ -244,7 +255,7 @@ public class ResourceInformation implements Comparable { 939769357 + name.hashCode(); // prime * result = 939769357 initially result = prime * result + resourceType.hashCode(); result = prime * result + units.hashCode(); - result = prime * result + value.hashCode(); + result = prime * result + Long.hashCode(value); return result; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java index 47bb3df..c7663de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java @@ -128,23 +128,24 @@ public class UnitsConversionUtil { * @param fromValue the value you wish to convert * @return the value in toUnit */ - public static Long convert(String fromUnit, String toUnit, Long fromValue) { - if (toUnit == null || fromUnit == null || fromValue == null) { + public static long convert(String fromUnit, String toUnit, long fromValue) { + if (toUnit == null || fromUnit == null) { throw new IllegalArgumentException("One or more arguments are null"); } - String overflowMsg = - "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit - + "' will result in an overflow of Long"; + if (fromUnit.equals(toUnit)) { return fromValue; } Converter fc = getConverter(fromUnit); Converter tc = getConverter(toUnit); - Long numerator = fc.numerator * tc.denominator; - Long denominator = fc.denominator * tc.numerator; - Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator; + long numerator = fc.numerator * tc.denominator; + long denominator = fc.denominator * tc.numerator; + long numeratorMultiplierLimit = Long.MAX_VALUE / numerator; if (numerator < denominator) { if (numeratorMultiplierLimit < fromValue) { + String overflowMsg = + "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit + + "' will result in an overflow of Long"; throw new IllegalArgumentException(overflowMsg); } return (fromValue * numerator) / denominator; @@ -152,8 +153,11 @@ public class UnitsConversionUtil { if (numeratorMultiplierLimit > fromValue) { return (numerator * fromValue) / denominator; } - Long tmp = numerator / denominator; + long tmp = numerator / denominator; if ((Long.MAX_VALUE / tmp) < fromValue) { + String overflowMsg = + "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit + + "' will result in an overflow of Long"; throw new IllegalArgumentException(overflowMsg); } return fromValue * tmp; @@ -170,8 +174,8 @@ public class UnitsConversionUtil { * @return +1, 0 or -1 depending on whether the relationship is greater than, * equal to or lesser than */ - public static int compare(String unitA, Long valueA, String unitB, - Long valueB) { + public static int compare(String unitA, long valueA, String unitB, + long valueB) { if (unitA == null || unitB == null || !KNOWN_UNITS.contains(unitA) || !KNOWN_UNITS.contains(unitB)) { throw new IllegalArgumentException("Units cannot be null"); @@ -185,19 +189,19 @@ public class UnitsConversionUtil { Converter unitAC = getConverter(unitA); Converter unitBC = getConverter(unitB); if (unitA.equals(unitB)) { - return valueA.compareTo(valueB); + return Long.valueOf(valueA).compareTo(valueB); } int unitAPos = SORTED_UNITS.indexOf(unitA); int unitBPos = SORTED_UNITS.indexOf(unitB); try { - Long tmpA = valueA; - Long tmpB = valueB; + long tmpA = valueA; + long tmpB = valueB; if (unitAPos < unitBPos) { tmpB = convert(unitB, unitA, valueB); } else { tmpA = convert(unitA, unitB, valueA); } - return tmpA.compareTo(tmpB); + return Long.valueOf(tmpA).compareTo(tmpB); } catch (IllegalArgumentException ie) { BigInteger tmpA = BigInteger.valueOf(valueA); BigInteger tmpB = BigInteger.valueOf(valueB); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java index 28f69c9..2efb990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java @@ -50,7 +50,7 @@ public class TestResourceInformation { @Test public void testValue() { String name = "yarn.io/test"; - Long value = 1l; + long value = 1L; ResourceInformation ri = ResourceInformation.newInstance(name, value); Assert.assertEquals("Resource name incorrect", name, ri.getName()); Assert.assertEquals("Resource value incorrect", value, ri.getValue()); @@ -59,7 +59,7 @@ public class TestResourceInformation { @Test public void testResourceInformation() { String name = "yarn.io/test"; - Long value = 1l; + long value = 1L; String units = "m"; ResourceInformation ri = ResourceInformation.newInstance(name, units, value); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java index 0f999e8..0b512a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java @@ -27,62 +27,62 @@ public class TestUnitsConversionUtil { public void testUnitsConversion() { int value = 5; String fromUnit = ""; - Long test = Long.valueOf(value); + long test = value; Assert.assertEquals("pico test failed", - Long.valueOf(value * 1000l * 1000l * 1000l * 1000l), + value * 1000L * 1000L * 1000L * 1000L, UnitsConversionUtil.convert(fromUnit, "p", test)); Assert.assertEquals("nano test failed", - Long.valueOf(value * 1000l * 1000l * 1000l), + value * 1000L * 1000L * 1000L, UnitsConversionUtil.convert(fromUnit, "n", test)); Assert - .assertEquals("micro test failed", Long.valueOf(value * 1000l * 1000l), + .assertEquals("micro test failed", value * 1000L * 1000L, UnitsConversionUtil.convert(fromUnit, "u", test)); - Assert.assertEquals("milli test failed", Long.valueOf(value * 1000l), + Assert.assertEquals("milli test failed", value * 1000L, UnitsConversionUtil.convert(fromUnit, "m", test)); - test = Long.valueOf(value * 1000l * 1000l * 1000l * 1000l * 1000l); + test = value * 1000L * 1000L * 1000L * 1000L * 1000L; fromUnit = ""; - Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l), + Assert.assertEquals("kilo test failed", test / 1000L, UnitsConversionUtil.convert(fromUnit, "k", test)); Assert - .assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)), + .assertEquals("mega test failed", test / (1000L * 1000L), UnitsConversionUtil.convert(fromUnit, "M", test)); Assert.assertEquals("giga test failed", - Long.valueOf(test / (1000l * 1000l * 1000l)), + test / (1000L * 1000L * 1000L), UnitsConversionUtil.convert(fromUnit, "G", test)); Assert.assertEquals("tera test failed", - Long.valueOf(test / (1000l * 1000l * 1000l * 1000l)), + test / (1000L * 1000L * 1000L * 1000L), UnitsConversionUtil.convert(fromUnit, "T", test)); Assert.assertEquals("peta test failed", - Long.valueOf(test / (1000l * 1000l * 1000l * 1000l * 1000l)), + test / (1000L * 1000L * 1000L * 1000L * 1000L), UnitsConversionUtil.convert(fromUnit, "P", test)); - Assert.assertEquals("nano to pico test failed", Long.valueOf(value * 1000l), - UnitsConversionUtil.convert("n", "p", Long.valueOf(value))); + Assert.assertEquals("nano to pico test failed", value * 1000L, + UnitsConversionUtil.convert("n", "p", value)); - Assert.assertEquals("mega to giga test failed", Long.valueOf(value), - UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l))); + Assert.assertEquals("mega to giga test failed", value, + UnitsConversionUtil.convert("M", "G", value * 1000L)); - Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value), - UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l))); + Assert.assertEquals("Mi to Gi test failed", value, + UnitsConversionUtil.convert("Mi", "Gi", value * 1024L)); - Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024), - UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value))); + Assert.assertEquals("Mi to Ki test failed", value * 1024, + UnitsConversionUtil.convert("Mi", "Ki", value)); - Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024), - UnitsConversionUtil.convert("Ki", "", Long.valueOf(5))); + Assert.assertEquals("Ki to base units test failed", 5 * 1024, + UnitsConversionUtil.convert("Ki", "", 5)); - Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741), - UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024))); + Assert.assertEquals("Mi to k test failed", 1073741, + UnitsConversionUtil.convert("Mi", "k", 1024)); - Assert.assertEquals("M to Mi test failed", Long.valueOf(953), - UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000))); + Assert.assertEquals("M to Mi test failed", 953, + UnitsConversionUtil.convert("M", "Mi", 1000)); } @Test public void testOverflow() { - Long test = Long.valueOf(5 * 1000l * 1000l * 1000l * 1000l * 1000l); + long test = 5 * 1000L * 1000L * 1000L * 1000L * 1000L; try { UnitsConversionUtil.convert("P", "p", test); Assert.fail("this operation should result in an overflow"); @@ -100,9 +100,9 @@ public class TestUnitsConversionUtil { @Test public void testCompare() { String unitA = "P"; - Long valueA = Long.valueOf(1); + long valueA = 1; String unitB = "p"; - Long valueB = Long.valueOf(2); + long valueB = 2; Assert.assertEquals(1, UnitsConversionUtil.compare(unitA, valueA, unitB, valueB)); Assert.assertEquals(-1, @@ -120,7 +120,7 @@ public class TestUnitsConversionUtil { Assert.assertEquals(-1, UnitsConversionUtil.compare(unitB, valueB, unitA, valueA)); Assert.assertEquals(0, - UnitsConversionUtil.compare(unitA, valueA, unitB, 1000l)); + UnitsConversionUtil.compare(unitA, valueA, unitB, 1000L)); unitA = "p"; unitB = "n"; @@ -129,7 +129,7 @@ public class TestUnitsConversionUtil { Assert.assertEquals(1, UnitsConversionUtil.compare(unitB, valueB, unitA, valueA)); Assert.assertEquals(0, - UnitsConversionUtil.compare(unitA, 1000l, unitB, valueA)); + UnitsConversionUtil.compare(unitA, 1000L, unitB, valueA)); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 955ea52..a9abed9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import org.apache.commons.collections.map.UnmodifiableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -46,6 +47,7 @@ public class ResourcePBImpl extends Resource { boolean viaProto = false; private Map resources; + private Map readOnlyResources; // call via ProtoUtils.convertToProtoFormat(Resource) @@ -68,6 +70,7 @@ public class ResourcePBImpl extends Resource { public ResourcePBImpl(ResourceProto proto) { this.proto = proto; viaProto = true; + this.readOnlyResources = null; this.resources = null; initResources(); } @@ -111,10 +114,9 @@ public class ResourcePBImpl extends Resource { @Override public void setMemorySize(long memory) { - setResourceInformation(ResourceInformation.MEMORY_MB.getName(), - ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(), - ResourceInformation.MEMORY_MB.getUnits(), memory)); - + maybeInitBuilder(); + getResourceInformation(ResourceInformation.MEMORY_MB.getName()) + .setValue(memory); } @Override @@ -127,9 +129,9 @@ public class ResourcePBImpl extends Resource { @Override public void setVirtualCores(int vCores) { - setResourceInformation(ResourceInformation.VCORES.getName(), - ResourceInformation.newInstance(ResourceInformation.VCORES.getName(), - ResourceInformation.VCORES.getUnits(), (long) vCores)); + maybeInitBuilder(); + getResourceInformation(ResourceInformation.VCORES.getName()) + .setValue(vCores); } private void initResources() { @@ -143,7 +145,7 @@ public class ResourcePBImpl extends Resource { entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) : ResourceTypes.COUNTABLE; String units = entry.hasUnits() ? entry.getUnits() : ""; - Long value = entry.hasValue() ? entry.getValue() : 0L; + long value = entry.hasValue() ? entry.getValue() : 0L; ResourceInformation ri = ResourceInformation .newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE); if (resources.containsKey(ri.getName())) { @@ -171,7 +173,7 @@ public class ResourcePBImpl extends Resource { } initResources(); if (resources.containsKey(resource)) { - resources.put(resource, resourceInformation); + ResourceInformation.copy(resourceInformation, resources.get(resource)); } } @@ -193,7 +195,7 @@ public class ResourcePBImpl extends Resource { @Override public Map getResources() { initResources(); - return Collections.unmodifiableMap(this.resources); + return readOnlyResources; } @Override @@ -226,6 +228,7 @@ public class ResourcePBImpl extends Resource { resources.put(entry.getKey(), ResourceInformation.newInstance(entry.getValue())); } + readOnlyResources = Collections.unmodifiableMap(resources); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 7db1da4..76bc364 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -154,7 +154,7 @@ public class DominantResourceCalculator extends ResourceCalculator { clusterResource.getResourceInformation(rName); ResourceInformation resourceInformation = resource.getResourceInformation(rName); - Long resourceValue = UnitsConversionUtil + long resourceValue = UnitsConversionUtil .convert(resourceInformation.getUnits(), clusterResourceResourceInformation.getUnits(), resourceInformation.getValue()); @@ -180,11 +180,11 @@ public class DominantResourceCalculator extends ResourceCalculator { available.getResourceInformation(resource); ResourceInformation requiredResource = required.getResourceInformation(resource); - Long requiredResourceValue = UnitsConversionUtil + long requiredResourceValue = UnitsConversionUtil .convert(requiredResource.getUnits(), availableResource.getUnits(), requiredResource.getValue()); if (requiredResourceValue != 0) { - Long tmp = availableResource.getValue() / requiredResourceValue; + long tmp = availableResource.getValue() / requiredResourceValue; min = min < tmp ? min : tmp; } } catch (YarnException ye) { @@ -228,7 +228,7 @@ public class DominantResourceCalculator extends ResourceCalculator { a.getResourceInformation(resource); ResourceInformation bResourceInformation = b.getResourceInformation(resource); - Long bResourceValue = UnitsConversionUtil + long bResourceValue = UnitsConversionUtil .convert(bResourceInformation.getUnits(), aResourceInformation.getUnits(), bResourceInformation.getValue()); @@ -249,14 +249,13 @@ public class DominantResourceCalculator extends ResourceCalculator { } public Resource divideAndCeil(Resource numerator, long denominator) { - Resource ret = Resources.createResource(0, 0); + Resource ret = Resource.newInstance(numerator); for (String resource : resourceNames) { try { - ResourceInformation resourceInformation = ResourceInformation - .newInstance(numerator.getResourceInformation(resource)); + ResourceInformation resourceInformation = + ret.getResourceInformation(resource); resourceInformation.setValue( divideAndCeil(resourceInformation.getValue(), denominator)); - ret.setResourceInformation(resource, resourceInformation); } catch (YarnException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); @@ -276,7 +275,7 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { - Resource ret = Resources.createResource(0, 0); + Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { try { ResourceInformation rResourceInformation = @@ -287,28 +286,26 @@ public class DominantResourceCalculator extends ResourceCalculator { maximumResource.getResourceInformation(resource); ResourceInformation stepFactorResourceInformation = stepFactor.getResourceInformation(resource); - ResourceInformation tmp = - ResourceInformation.newInstance(rResourceInformation); + ResourceInformation tmp = ret.getResourceInformation(resource); - Long rValue = rResourceInformation.getValue(); - Long minimumValue = UnitsConversionUtil + long rValue = rResourceInformation.getValue(); + long minimumValue = UnitsConversionUtil .convert(minimumResourceInformation.getUnits(), rResourceInformation.getUnits(), minimumResourceInformation.getValue()); - Long maximumValue = UnitsConversionUtil + long maximumValue = UnitsConversionUtil .convert(maximumResourceInformation.getUnits(), rResourceInformation.getUnits(), maximumResourceInformation.getValue()); - Long stepFactorValue = UnitsConversionUtil + long stepFactorValue = UnitsConversionUtil .convert(stepFactorResourceInformation.getUnits(), rResourceInformation.getUnits(), stepFactorResourceInformation.getValue()); - Long value = Math.max(rValue, minimumValue); + long value = Math.max(rValue, minimumValue); if (stepFactorValue != 0) { value = roundUp(value, stepFactorValue); } tmp.setValue(Math.min(value, maximumValue)); - ret.setResourceInformation(resource, tmp); } catch (YarnException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); @@ -328,28 +325,27 @@ public class DominantResourceCalculator extends ResourceCalculator { } private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { - Resource ret = Resources.createResource(0, 0); + Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { try { ResourceInformation rResourceInformation = r.getResourceInformation(resource); ResourceInformation stepFactorResourceInformation = stepFactor.getResourceInformation(resource); - ResourceInformation tmp = - ResourceInformation.newInstance(rResourceInformation); - Long rValue = rResourceInformation.getValue(); - Long stepFactorValue = UnitsConversionUtil + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil .convert(stepFactorResourceInformation.getUnits(), rResourceInformation.getUnits(), stepFactorResourceInformation.getValue()); - Long value = rValue; + long value = rValue; if (stepFactorValue != 0) { value = roundUp ? roundUp(rValue, stepFactorValue) : roundDown(rValue, stepFactorValue); } - tmp.setValue(value); - ret.setResourceInformation(resource, tmp); + ResourceInformation + .copy(rResourceInformation, ret.getResourceInformation(resource)); + ret.getResourceInformation(resource).setValue(value); } catch (YarnException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); @@ -372,15 +368,14 @@ public class DominantResourceCalculator extends ResourceCalculator { private Resource multiplyAndNormalize(Resource r, double by, Resource stepFactor, boolean roundUp) { - Resource ret = Resources.createResource(0, 0); + Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { try { ResourceInformation rResourceInformation = r.getResourceInformation(resource); ResourceInformation stepFactorResourceInformation = stepFactor.getResourceInformation(resource); - ResourceInformation tmp = - ResourceInformation.newInstance(rResourceInformation); + ResourceInformation tmp = ret.getResourceInformation(resource); Long rValue = rResourceInformation.getValue(); Long stepFactorValue = UnitsConversionUtil @@ -397,7 +392,6 @@ public class DominantResourceCalculator extends ResourceCalculator { roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by); } tmp.setValue(value); - ret.setResourceInformation(resource, tmp); } catch (YarnException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); @@ -414,7 +408,7 @@ public class DominantResourceCalculator extends ResourceCalculator { smaller.getResourceInformation(resource); ResourceInformation bResourceInformation = bigger.getResourceInformation(resource); - Long sResourceValue = UnitsConversionUtil + long sResourceValue = UnitsConversionUtil .convert(sResourceInformation.getUnits(), bResourceInformation.getUnits(), sResourceInformation.getValue()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index b0ec907..d143e93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -42,7 +42,7 @@ public class Resources { static class FixedValueResource extends Resource { private Map resources; - private Long resourceValue; + private long resourceValue; private String name; /** @@ -50,7 +50,7 @@ public class Resources { * @param rName the name of the resource * @param value the fixed value to be returned for all resource types */ - FixedValueResource(String rName, Long value) { + FixedValueResource(String rName, long value) { this.resourceValue = value; this.name = rName; resources = initResourceMap(); @@ -60,7 +60,7 @@ public class Resources { if(this.resourceValue > Integer.MAX_VALUE) { return Integer.MAX_VALUE; } - return this.resourceValue.intValue(); + return Long.valueOf(this.resourceValue).intValue(); } @Override @@ -203,7 +203,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs); @@ -225,7 +225,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs); @@ -288,7 +288,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = (long) (UnitsConversionUtil + long convertedRhs = (long) (UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()) * by); lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs); @@ -422,7 +422,7 @@ public class Resources { try { ResourceInformation rhsValue = bigger.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); if(lhsValue.getValue() > convertedRhs) { @@ -448,7 +448,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); ResourceInformation outInfo = @@ -469,7 +469,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); ResourceInformation outInfo = http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 6f17659..38554b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -283,6 +283,7 @@ public class TestResourceUtils { Map actual = ResourceUtils.getNodeResourceInformation(conf); Assert.assertEquals(entry.getValue().getResources(), actual); + dest.delete(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index d81dcdb..1555e55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -42,6 +42,16 @@ import static org.junit.Assert.assertTrue; public class TestResources { + static class ExtendedResources extends Resources { + public static Resource unbounded() { + return new FixedValueResource("UNBOUNDED", Long.MAX_VALUE); + } + + public static Resource none() { + return new FixedValueResource("NONE", 0L); + } + } + private static final String EXTRA_RESOURCE_TYPE = "resource2"; private String resourceTypesFile; @@ -87,7 +97,7 @@ public class TestResources { @Test(timeout = 10000) public void testCompareToWithUnboundedResource() { unsetExtraResourceType(); - Resource unboundedClone = Resources.clone(Resources.unbounded()); + Resource unboundedClone = Resources.clone(ExtendedResources.unbounded()); assertTrue(unboundedClone .compareTo(createResource(Long.MAX_VALUE, Integer.MAX_VALUE)) == 0); assertTrue(unboundedClone.compareTo(createResource(Long.MAX_VALUE, 0)) > 0); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acec64d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java index 8839bf9..7987ded 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java @@ -131,7 +131,8 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { continue; } if (resourceName.equals(VCORES)) { - resource.setVirtualCores(resourceValue.getValue().intValue()); + resource + .setVirtualCores(Long.valueOf(resourceValue.getValue()).intValue()); continue; } if (resourceTypes.containsKey(resourceName)) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org