From common-commits-return-82242-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Wed May 2 22:40:35 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id B3CFF180675 for ; Wed, 2 May 2018 22:40:33 +0200 (CEST) Received: (qmail 33959 invoked by uid 500); 2 May 2018 20:40:32 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 33947 invoked by uid 99); 2 May 2018 20:40:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 May 2018 20:40:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3C8DCE38B8; Wed, 2 May 2018 20:40:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shv@apache.org To: common-commits@hadoop.apache.org Date: Wed, 02 May 2018 20:40:33 -0000 Message-Id: <3cbca9c058b246818eceeb08b52f1ad1@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [02/51] [abbrv] hadoop git commit: YARN-6788. [YARN-3926] Improve performance of resource profile branch (Contributed by Sunil Govindan via Daniel Templeton) YARN-6788. [YARN-3926] Improve performance of resource profile branch (Contributed by Sunil Govindan via Daniel Templeton) (cherry picked from commit 3aeaafecb823ef6c175ea5d0d9cb726faacaf32d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58e264ea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58e264ea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58e264ea Branch: refs/heads/YARN-8200 Commit: 58e264eadcc543264f3d2780e8f66a9adc03f3a1 Parents: 4498b26 Author: Daniel Templeton Authored: Fri Aug 4 08:42:34 2017 -0700 Committer: Konstantin V Shvachko Committed: Wed May 2 13:19:57 2018 -0700 ---------------------------------------------------------------------- .../dev-support/findbugs-exclude.xml | 18 + .../hadoop/yarn/api/records/Resource.java | 234 ++++---- .../yarn/api/records/ResourceInformation.java | 13 +- .../yarn/api/records/impl/BaseResource.java | 133 +++++ .../yarn/api/records/impl/package-info.java | 22 + .../hadoop/yarn/util/UnitsConversionUtil.java | 8 +- .../yarn/util/resource/ResourceUtils.java | 534 +++++++++++++++++++ .../hadoop/yarn/util/resource/package-info.java | 22 + .../yarn/api/records/impl/pb/ProtoUtils.java | 5 +- .../api/records/impl/pb/ResourcePBImpl.java | 110 ++-- .../resource/DominantResourceCalculator.java | 67 ++- .../yarn/util/resource/ResourceUtils.java | 488 ----------------- .../hadoop/yarn/util/resource/Resources.java | 194 ++++--- .../yarn/util/resource/TestResourceUtils.java | 14 +- .../yarn/util/resource/TestResources.java | 7 +- .../rmapp/attempt/RMAppAttemptMetrics.java | 11 +- .../scheduler/SchedulerApplicationAttempt.java | 9 +- .../webapp/dao/SchedulerInfo.java | 3 +- .../server/resourcemanager/TestAppManager.java | 1 + 19 files changed, 1031 insertions(+), 862 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index c129203..e086fbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -615,4 +615,22 @@ + + + + + + + + + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index fb31745..bbd4c87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.api.records; +import java.util.Arrays; + import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -25,13 +27,10 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.records.impl.BaseResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.Records; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; /** *

Resource models a set of computer resources in the @@ -60,97 +59,49 @@ import java.util.Map; @Stable public abstract class Resource implements Comparable { - private static Resource tmpResource = Records.newRecord(Resource.class); - - private static class SimpleResource extends Resource { - private long memory; - private long vcores; - private Map resourceInformationMap; - - SimpleResource(long memory, long vcores) { - this.memory = memory; - this.vcores = vcores; - - } - @Override - public int getMemory() { - return castToIntSafely(memory); - } - @Override - public void setMemory(int memory) { - this.memory = memory; - } - @Override - public long getMemorySize() { - return memory; - } - @Override - public void setMemorySize(long memory) { - this.memory = memory; - } - @Override - public int getVirtualCores() { - return castToIntSafely(vcores); - } - @Override - public void setVirtualCores(int vcores) { - this.vcores = vcores; - } - @Override - public Map getResources() { - if (resourceInformationMap == null) { - resourceInformationMap = new HashMap<>(); - resourceInformationMap.put(ResourceInformation.MEMORY_MB.getName(), - ResourceInformation.newInstance(ResourceInformation.MEMORY_MB)); - resourceInformationMap.put(ResourceInformation.VCORES.getName(), - ResourceInformation.newInstance(ResourceInformation.VCORES)); - } - resourceInformationMap.get(ResourceInformation.MEMORY_MB.getName()) - .setValue(this.memory); - resourceInformationMap.get(ResourceInformation.VCORES.getName()) - .setValue(this.vcores); - return Collections.unmodifiableMap(resourceInformationMap); - } - } + protected static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); + protected static final String VCORES = ResourceInformation.VCORES.getName(); @Public @Stable public static Resource newInstance(int memory, int vCores) { - if (tmpResource.getResources().size() > 2) { + if (ResourceUtils.getResourceTypesArray().length > 2) { Resource ret = Records.newRecord(Resource.class); ret.setMemorySize(memory); ret.setVirtualCores(vCores); return ret; } - return new SimpleResource(memory, vCores); + return new BaseResource(memory, vCores); } @Public @Stable public static Resource newInstance(long memory, int vCores) { - if (tmpResource.getResources().size() > 2) { + if (ResourceUtils.getResourceTypesArray().length > 2) { Resource ret = Records.newRecord(Resource.class); ret.setMemorySize(memory); ret.setVirtualCores(vCores); return ret; } - return new SimpleResource(memory, vCores); + return new BaseResource(memory, vCores); } @InterfaceAudience.Private @InterfaceStability.Unstable public static Resource newInstance(Resource resource) { - Resource ret = Resource.newInstance(0, 0); - Resource.copy(resource, ret); + Resource ret = Resource.newInstance(resource.getMemorySize(), + resource.getVirtualCores()); + if (ResourceUtils.getResourceTypesArray().length > 2) { + Resource.copy(resource, ret); + } return ret; } @InterfaceAudience.Private @InterfaceStability.Unstable public static void copy(Resource source, Resource dest) { - for (Map.Entry entry : source.getResources() - .entrySet()) { - dest.setResourceInformation(entry.getKey(), entry.getValue()); + for (ResourceInformation entry : source.getResources()) { + dest.setResourceInformation(entry.getName(), entry); } } @@ -251,25 +202,26 @@ public abstract class Resource implements Comparable { */ @Public @Evolving - public abstract Map getResources(); + public abstract ResourceInformation[] getResources(); /** * Get ResourceInformation for a specified resource. * * @param resource name of the resource * @return the ResourceInformation object for the resource - * @throws YarnException if the resource can't be found + * @throws ResourceNotFoundException if the resource can't be found */ @Public @Evolving public ResourceInformation getResourceInformation(String resource) - throws YarnException { - if (getResources().containsKey(resource)) { - return getResources().get(resource); + throws ResourceNotFoundException { + Integer index = ResourceUtils.getResourceTypeIndex().get(resource); + ResourceInformation[] resources = getResources(); + if (index != null) { + return resources[index]; } - throw new YarnException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + throw new ResourceNotFoundException("Unknown resource '" + resource + + "'. Known resources are " + Arrays.toString(resources)); } /** @@ -278,17 +230,13 @@ public abstract class Resource implements Comparable { * * @param resource name of the resource * @return the value for the resource - * @throws YarnException if the resource can't be found + * @throws ResourceNotFoundException if the resource can't be found */ @Public @Evolving - public Long getResourceValue(String resource) throws YarnException { - if (getResources().containsKey(resource)) { - return getResources().get(resource).getValue(); - } - throw new YarnException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + public long getResourceValue(String resource) + throws ResourceNotFoundException { + return getResourceInformation(resource).getValue(); } /** @@ -301,23 +249,18 @@ public abstract class Resource implements Comparable { @Public @Evolving public void setResourceInformation(String resource, - ResourceInformation resourceInformation) throws ResourceNotFoundException { - if (resource.equals(ResourceInformation.MEMORY_MB.getName())) { + ResourceInformation resourceInformation) + throws ResourceNotFoundException { + if (resource.equals(MEMORY)) { this.setMemorySize(resourceInformation.getValue()); return; } - if (resource.equals(ResourceInformation.VCORES.getName())) { + if (resource.equals(VCORES)) { this.setVirtualCores((int) resourceInformation.getValue()); return; } - if (getResources().containsKey(resource)) { - ResourceInformation - .copy(resourceInformation, getResources().get(resource)); - return; - } - throw new ResourceNotFoundException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + ResourceInformation storedResourceInfo = getResourceInformation(resource); + ResourceInformation.copy(resourceInformation, storedResourceInfo); } /** @@ -332,21 +275,17 @@ public abstract class Resource implements Comparable { @Evolving public void setResourceValue(String resource, Long value) throws ResourceNotFoundException { - if (resource.equals(ResourceInformation.MEMORY_MB.getName())) { + if (resource.equals(MEMORY)) { this.setMemorySize(value); return; } - if (resource.equals(ResourceInformation.VCORES.getName())) { + if (resource.equals(VCORES)) { this.setVirtualCores(value.intValue()); return; } - if (getResources().containsKey(resource)) { - getResources().get(resource).setValue(value); - return; - } - throw new ResourceNotFoundException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + + ResourceInformation storedResourceInfo = getResourceInformation(resource); + storedResourceInfo.setValue(value); } @Override @@ -356,13 +295,10 @@ public abstract class Resource implements Comparable { int result = (int) (939769357 + getMemorySize()); // prime * result = 939769357 initially result = prime * result + getVirtualCores(); - for (Map.Entry entry : getResources() - .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) - || entry.getKey().equals(ResourceInformation.VCORES.getName())) { - continue; + for (ResourceInformation entry : getResources()) { + if (!entry.getName().equals(MEMORY) && !entry.getName().equals(VCORES)) { + result = prime * result + entry.hashCode(); } - result = prime * result + entry.getValue().hashCode(); } return result; } @@ -379,11 +315,26 @@ public abstract class Resource implements Comparable { return false; } Resource other = (Resource) obj; - if (getMemorySize() != other.getMemorySize() || getVirtualCores() != other - .getVirtualCores()) { + if (getMemorySize() != other.getMemorySize() + || getVirtualCores() != other.getVirtualCores()) { + return false; + } + + ResourceInformation[] myVectors = getResources(); + ResourceInformation[] otherVectors = other.getResources(); + + if (myVectors.length != otherVectors.length) { return false; } - return this.getResources().equals(other.getResources()); + + for (int i = 0; i < myVectors.length; i++) { + ResourceInformation a = myVectors[i]; + ResourceInformation b = otherVectors[i]; + if ((a != b) && ((a == null) || !a.equals(b))) { + return false; + } + } + return true; } @Override @@ -391,21 +342,20 @@ public abstract class Resource implements Comparable { StringBuilder sb = new StringBuilder(); sb.append(" entry : getResources() - .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) - && entry.getValue().getUnits() + for (ResourceInformation entry : getResources()) { + if (entry.getName().equals(MEMORY) + && entry.getUnits() .equals(ResourceInformation.MEMORY_MB.getUnits())) { continue; } - if (entry.getKey().equals(ResourceInformation.VCORES.getName()) - && entry.getValue().getUnits() + if (entry.getName().equals(VCORES) + && entry.getUnits() .equals(ResourceInformation.VCORES.getUnits())) { continue; } - sb.append(", ").append(entry.getKey()).append(": ") - .append(entry.getValue().getValue()) - .append(entry.getValue().getUnits()); + sb.append(", ").append(entry.getName()).append(": ") + .append(entry.getValue()) + .append(entry.getUnits()); } sb.append(">"); return sb.toString(); @@ -413,28 +363,30 @@ public abstract class Resource implements Comparable { @Override public int compareTo(Resource other) { - Map thisResources, otherResources; - thisResources = this.getResources(); - otherResources = other.getResources(); - long diff = thisResources.size() - otherResources.size(); + ResourceInformation[] thisResources = this.getResources(); + ResourceInformation[] otherResources = other.getResources(); + + // compare memory and vcores first(in that order) to preserve + // existing behaviour + long diff = this.getMemorySize() - other.getMemorySize(); if (diff == 0) { - // compare memory and vcores first(in that order) to preserve - // existing behaviour - if (thisResources.keySet().equals(otherResources.keySet())) { - diff = this.getMemorySize() - other.getMemorySize(); - if (diff == 0) { - diff = this.getVirtualCores() - other.getVirtualCores(); - } - if (diff == 0) { - for (Map.Entry entry : thisResources - .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) - || entry.getKey() - .equals(ResourceInformation.VCORES.getName())) { - continue; - } - diff = - entry.getValue().compareTo(otherResources.get(entry.getKey())); + diff = this.getVirtualCores() - other.getVirtualCores(); + } + if (diff == 0) { + diff = thisResources.length - otherResources.length; + if (diff == 0) { + int maxLength = ResourceUtils.getResourceTypesArray().length; + for (int i = 0; i < maxLength; i++) { + // For memory and vcores, we can skip the loop as it's already + // compared. + if (i < 2) { + continue; + } + + ResourceInformation entry = thisResources[i]; + ResourceInformation otherEntry = otherResources[i]; + if (entry.getName().equals(otherEntry.getName())) { + diff = entry.compareTo(otherEntry); if (diff != 0) { break; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 7c0f055..4717d82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -242,10 +242,15 @@ public class ResourceInformation implements Comparable { return false; } ResourceInformation r = (ResourceInformation) obj; - int cmp = - UnitsConversionUtil.compare(this.units, this.value, r.units, r.value); - return this.name.equals(r.getName()) && this.resourceType - .equals(r.getResourceType()) && (cmp == 0); + if (!this.name.equals(r.getName()) + || !this.resourceType.equals(r.getResourceType())) { + return false; + } + if (this.units.equals(r.units)) { + return this.value == r.value; + } + return (UnitsConversionUtil.compare(this.units, this.value, r.units, + r.value) == 0); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java new file mode 100644 index 0000000..83db542 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; + +import java.util.Arrays; + +/** + *

+ * BaseResource extends Resource to handle base resources such + * as memory and CPU. + * TODO: We have a long term plan to use AbstractResource when additional + * resource types are to be handled as well. + *

+ * + *

+ * Currently it models both memory and CPU. + *

+ * + *

+ * The unit for memory is megabytes. CPU is modeled with virtual cores (vcores), + * a unit for expressing parallelism. A node's capacity should be configured + * with virtual cores equal to its number of physical cores. A container should + * be requested with the number of cores it can saturate, i.e. the average + * number of threads it expects to have runnable at a time. + *

+ * + *

+ * Virtual cores take integer values and thus currently CPU-scheduling is very + * coarse. A complementary axis for CPU requests that represents processing + * power will likely be added in the future to enable finer-grained resource + * configuration. + *

+ * + * @see Resource + */ +@Public +@Unstable +public class BaseResource extends Resource { + + private ResourceInformation memoryResInfo; + private ResourceInformation vcoresResInfo; + protected ResourceInformation[] resources = null; + protected ResourceInformation[] readOnlyResources = null; + + protected enum MandatoryResources { + MEMORY(0), VCORES(1); + + private final int id; + + MandatoryResources(int id) { + this.id = id; + } + + public int getId() { + return this.id; + } + } + + public BaseResource() { + // Base constructor. + } + + public BaseResource(long memory, long vcores) { + this.memoryResInfo = ResourceInformation.newInstance(MEMORY, + ResourceInformation.MEMORY_MB.getUnits(), memory); + this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores); + + resources = new ResourceInformation[MandatoryResources.values().length]; + readOnlyResources = new ResourceInformation[MandatoryResources + .values().length]; + resources[MandatoryResources.MEMORY.id] = memoryResInfo; + resources[MandatoryResources.VCORES.id] = vcoresResInfo; + readOnlyResources = Arrays.copyOf(resources, resources.length); + } + + @Override + @SuppressWarnings("deprecation") + public int getMemory() { + return (int) memoryResInfo.getValue(); + } + + @Override + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + this.memoryResInfo.setValue(memory); + } + + @Override + public long getMemorySize() { + return memoryResInfo.getValue(); + } + + @Override + public void setMemorySize(long memory) { + this.memoryResInfo.setValue(memory); + } + + @Override + public int getVirtualCores() { + return (int) vcoresResInfo.getValue(); + } + + @Override + public void setVirtualCores(int vcores) { + this.vcoresResInfo.setValue(vcores); + } + + @Override + public ResourceInformation[] getResources() { + return readOnlyResources; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java new file mode 100644 index 0000000..d04e339 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.api.records.impl; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java index c7663de..7b737bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java @@ -186,11 +186,11 @@ public class UnitsConversionUtil { if (!KNOWN_UNITS.contains(unitB)) { throw new IllegalArgumentException("Unknown unit '" + unitB + "'"); } - Converter unitAC = getConverter(unitA); - Converter unitBC = getConverter(unitB); if (unitA.equals(unitB)) { - return Long.valueOf(valueA).compareTo(valueB); + return Long.compare(valueA, valueB); } + Converter unitAC = getConverter(unitA); + Converter unitBC = getConverter(unitB); int unitAPos = SORTED_UNITS.indexOf(unitA); int unitBPos = SORTED_UNITS.indexOf(unitB); try { @@ -201,7 +201,7 @@ public class UnitsConversionUtil { } else { tmpA = convert(unitA, unitB, valueA); } - return Long.valueOf(tmpA).compareTo(tmpB); + return Long.compare(tmpA, tmpB); } catch (IllegalArgumentException ie) { BigInteger tmpA = BigInteger.valueOf(valueA); BigInteger tmpB = BigInteger.valueOf(valueB); http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java new file mode 100644 index 0000000..49feafa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -0,0 +1,534 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util.resource; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Helper class to read the resource-types to be supported by the system. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class ResourceUtils { + + public static final String UNITS = ".units"; + public static final String TYPE = ".type"; + public static final String MINIMUM_ALLOCATION = ".minimum-allocation"; + public static final String MAXIMUM_ALLOCATION = ".maximum-allocation"; + + private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); + private static final String VCORES = ResourceInformation.VCORES.getName(); + + private static final Set DISALLOWED_NAMES = new HashSet<>(); + static { + DISALLOWED_NAMES.add("memory"); + DISALLOWED_NAMES.add(MEMORY); + DISALLOWED_NAMES.add(VCORES); + } + + private static volatile boolean initializedResources = false; + private static final Map RESOURCE_NAME_TO_INDEX = + new ConcurrentHashMap(); + private static volatile Map resourceTypes; + private static volatile String[] resourceNamesArray; + private static volatile ResourceInformation[] resourceTypesArray; + private static volatile boolean initializedNodeResources = false; + private static volatile Map readOnlyNodeResources; + + static final Log LOG = LogFactory.getLog(ResourceUtils.class); + + private ResourceUtils() { + } + + private static void checkMandatatoryResources( + Map resourceInformationMap) + throws YarnRuntimeException { + if (resourceInformationMap.containsKey(MEMORY)) { + ResourceInformation memInfo = resourceInformationMap.get(MEMORY); + String memUnits = ResourceInformation.MEMORY_MB.getUnits(); + ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType(); + if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType() + .equals(memType)) { + throw new YarnRuntimeException( + "Attempt to re-define mandatory resource 'memory-mb'. It can only" + + " be of type 'COUNTABLE' and have units 'Mi'."); + } + } + + if (resourceInformationMap.containsKey(VCORES)) { + ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES); + String vcoreUnits = ResourceInformation.VCORES.getUnits(); + ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType(); + if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo + .getResourceType().equals(vcoreType)) { + throw new YarnRuntimeException( + "Attempt to re-define mandatory resource 'vcores'. It can only be" + + " of type 'COUNTABLE' and have units ''(no units)."); + } + } + } + + private static void addManadtoryResources( + Map res) { + ResourceInformation ri; + if (!res.containsKey(MEMORY)) { + LOG.info("Adding resource type - name = " + MEMORY + ", units = " + + ResourceInformation.MEMORY_MB.getUnits() + ", type = " + + ResourceTypes.COUNTABLE); + ri = ResourceInformation + .newInstance(MEMORY, + ResourceInformation.MEMORY_MB.getUnits()); + res.put(MEMORY, ri); + } + if (!res.containsKey(VCORES)) { + LOG.info("Adding resource type - name = " + VCORES + ", units = , type = " + + ResourceTypes.COUNTABLE); + ri = + ResourceInformation.newInstance(VCORES); + res.put(VCORES, ri); + } + } + + private static void setMinimumAllocationForMandatoryResources( + Map res, Configuration conf) { + String[][] resourceTypesKeys = { + {ResourceInformation.MEMORY_MB.getName(), + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + String.valueOf( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB), + ResourceInformation.MEMORY_MB.getName()}, + {ResourceInformation.VCORES.getName(), + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + String.valueOf( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES), + ResourceInformation.VCORES.getName()}}; + for (String[] arr : resourceTypesKeys) { + String resourceTypesKey = + YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION; + long minimumResourceTypes = conf.getLong(resourceTypesKey, -1); + long minimumConf = conf.getLong(arr[1], -1); + long minimum; + if (minimumResourceTypes != -1) { + minimum = minimumResourceTypes; + if (minimumConf != -1) { + LOG.warn("Using minimum allocation for memory specified in " + + "resource-types config file with key " + + minimumResourceTypes + ", ignoring minimum specified using " + + arr[1]); + } + } else { + minimum = conf.getLong(arr[1], Long.parseLong(arr[2])); + } + ResourceInformation ri = res.get(arr[3]); + ri.setMinimumAllocation(minimum); + } + } + + private static void setMaximumAllocationForMandatoryResources( + Map res, Configuration conf) { + String[][] resourceTypesKeys = { + {ResourceInformation.MEMORY_MB.getName(), + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + String.valueOf( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB), + ResourceInformation.MEMORY_MB.getName()}, + {ResourceInformation.VCORES.getName(), + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + String.valueOf( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES), + ResourceInformation.VCORES.getName()}}; + for (String[] arr : resourceTypesKeys) { + String resourceTypesKey = + YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION; + long maximumResourceTypes = conf.getLong(resourceTypesKey, -1); + long maximumConf = conf.getLong(arr[1], -1); + long maximum; + if (maximumResourceTypes != -1) { + maximum = maximumResourceTypes; + if (maximumConf != -1) { + LOG.warn("Using maximum allocation for memory specified in " + + "resource-types config file with key " + + maximumResourceTypes + ", ignoring maximum specified using " + + arr[1]); + } + } else { + maximum = conf.getLong(arr[1], Long.parseLong(arr[2])); + } + ResourceInformation ri = res.get(arr[3]); + ri.setMaximumAllocation(maximum); + } + } + + @VisibleForTesting + static void initializeResourcesMap(Configuration conf, + Map resourceInformationMap) { + + String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES); + + if (resourceNames != null && resourceNames.length != 0) { + for (String resourceName : resourceNames) { + String resourceUnits = conf.get( + YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, ""); + String resourceTypeName = conf.get( + YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE, + ResourceTypes.COUNTABLE.toString()); + Long minimumAllocation = conf.getLong( + YarnConfiguration.RESOURCE_TYPES + "." + resourceName + + MINIMUM_ALLOCATION, 0L); + Long maximumAllocation = conf.getLong( + YarnConfiguration.RESOURCE_TYPES + "." + resourceName + + MAXIMUM_ALLOCATION, Long.MAX_VALUE); + if (resourceName == null || resourceName.isEmpty() + || resourceUnits == null || resourceTypeName == null) { + throw new YarnRuntimeException( + "Incomplete configuration for resource type '" + resourceName + + "'. One of name, units or type is configured incorrectly."); + } + if (DISALLOWED_NAMES.contains(resourceName)) { + throw new YarnRuntimeException( + "Resource type cannot be named '" + resourceName + + "'. That name is disallowed."); + } + ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName); + LOG.info("Adding resource type - name = " + resourceName + ", units = " + + resourceUnits + ", type = " + resourceTypeName); + if (resourceInformationMap.containsKey(resourceName)) { + throw new YarnRuntimeException( + "Error in config, key '" + resourceName + "' specified twice"); + } + resourceInformationMap.put(resourceName, ResourceInformation + .newInstance(resourceName, resourceUnits, 0L, resourceType, + minimumAllocation, maximumAllocation)); + } + } + checkMandatatoryResources(resourceInformationMap); + addManadtoryResources(resourceInformationMap); + setMinimumAllocationForMandatoryResources(resourceInformationMap, conf); + setMaximumAllocationForMandatoryResources(resourceInformationMap, conf); + resourceTypes = Collections.unmodifiableMap(resourceInformationMap); + updateKnownResources(); + updateResourceTypeIndex(); + } + + private static void updateKnownResources() { + // Update resource names. + resourceNamesArray = new String[resourceTypes.size()]; + resourceTypesArray = new ResourceInformation[resourceTypes.size()]; + + int index = 2; + for (ResourceInformation resInfo : resourceTypes.values()) { + if (resInfo.getName().equals(MEMORY)) { + resourceTypesArray[0] = ResourceInformation + .newInstance(resourceTypes.get(MEMORY)); + resourceNamesArray[0] = MEMORY; + } else if (resInfo.getName().equals(VCORES)) { + resourceTypesArray[1] = ResourceInformation + .newInstance(resourceTypes.get(VCORES)); + resourceNamesArray[1] = VCORES; + } else { + resourceTypesArray[index] = ResourceInformation.newInstance(resInfo); + resourceNamesArray[index] = resInfo.getName(); + index++; + } + } + } + + private static void updateResourceTypeIndex() { + RESOURCE_NAME_TO_INDEX.clear(); + + for (int index = 0; index < resourceTypesArray.length; index++) { + ResourceInformation resInfo = resourceTypesArray[index]; + RESOURCE_NAME_TO_INDEX.put(resInfo.getName(), index); + } + } + + /** + * Get associate index of resource types such memory, cpu etc. + * This could help to access each resource types in a resource faster. + * @return Index map for all Resource Types. + */ + public static Map getResourceTypeIndex() { + return RESOURCE_NAME_TO_INDEX; + } + + /** + * Get the resource types to be supported by the system. + * @return A map of the resource name to a ResouceInformation object + * which contains details such as the unit. + */ + public static Map getResourceTypes() { + return getResourceTypes(null, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + } + + /** + * Get resource names array, this is mostly for performance perspective. Never + * modify returned array. + * + * @return resourceNamesArray + */ + public static String[] getResourceNamesArray() { + getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + return resourceNamesArray; + } + + public static ResourceInformation[] getResourceTypesArray() { + getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + return resourceTypesArray; + } + + private static Map getResourceTypes( + Configuration conf) { + return getResourceTypes(conf, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + } + + private static Map getResourceTypes( + Configuration conf, String resourceFile) { + if (!initializedResources) { + synchronized (ResourceUtils.class) { + if (!initializedResources) { + Map resources = new HashMap<>(); + if (conf == null) { + conf = new YarnConfiguration(); + } + try { + addResourcesFileToConf(resourceFile, conf); + LOG.debug("Found " + resourceFile + ", adding to configuration"); + initializeResourcesMap(conf, resources); + initializedResources = true; + } catch (FileNotFoundException fe) { + LOG.info("Unable to find '" + resourceFile + + "'. Falling back to memory and vcores as resources", fe); + initializeResourcesMap(conf, resources); + initializedResources = true; + } + } + } + } + return resourceTypes; + } + + private static InputStream getConfInputStream(String resourceFile, + Configuration conf) throws IOException, YarnException { + + ConfigurationProvider provider = + ConfigurationProviderFactory.getConfigurationProvider(conf); + try { + provider.init(conf); + } catch (Exception e) { + throw new IOException(e); + } + + InputStream ris = provider.getConfigurationInputStream(conf, resourceFile); + if (ris == null) { + if (conf.getResource(resourceFile) == null) { + throw new FileNotFoundException("Unable to find " + resourceFile); + } + throw new IOException( + "Unable to open resource types file '" + resourceFile + + "'. Using provider " + provider); + } + return ris; + } + + private static void addResourcesFileToConf(String resourceFile, + Configuration conf) throws FileNotFoundException { + try { + InputStream ris = getConfInputStream(resourceFile, conf); + LOG.debug("Found " + resourceFile + ", adding to configuration"); + conf.addResource(ris); + } catch (FileNotFoundException fe) { + throw fe; + } catch (IOException ie) { + LOG.fatal("Exception trying to read resource types configuration '" + + resourceFile + "'.", ie); + throw new YarnRuntimeException(ie); + } catch (YarnException ye) { + LOG.fatal("YARN Exception trying to read resource types configuration '" + + resourceFile + "'.", ye); + throw new YarnRuntimeException(ye); + } + } + + @VisibleForTesting + synchronized static void resetResourceTypes() { + initializedResources = false; + } + + @VisibleForTesting + public static void resetResourceTypes(Configuration conf) { + synchronized (ResourceUtils.class) { + initializedResources = false; + } + getResourceTypes(conf); + } + + public static String getUnits(String resourceValue) { + String units; + for (int i = 0; i < resourceValue.length(); i++) { + if (Character.isAlphabetic(resourceValue.charAt(i))) { + units = resourceValue.substring(i); + if (StringUtils.isAlpha(units)) { + return units; + } + } + } + return ""; + } + + /** + * Function to get the resources for a node. This function will look at the + * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to + * determine the node resources. + * + * @param conf configuration file + * @return a map to resource name to the ResourceInformation object. The map + * is guaranteed to have entries for memory and vcores + */ + public static Map getNodeResourceInformation( + Configuration conf) { + if (!initializedNodeResources) { + synchronized (ResourceUtils.class) { + if (!initializedNodeResources) { + Map nodeResources = initializeNodeResourceInformation( + conf); + addManadtoryResources(nodeResources); + checkMandatatoryResources(nodeResources); + setMinimumAllocationForMandatoryResources(nodeResources, conf); + setMaximumAllocationForMandatoryResources(nodeResources, conf); + readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); + initializedNodeResources = true; + } + } + } + return readOnlyNodeResources; + } + + private static Map + initializeNodeResourceInformation(Configuration conf) { + Map nodeResources = new HashMap<>(); + try { + addResourcesFileToConf( + YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf); + for (Map.Entry entry : conf) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) { + addResourceInformation(key, value, nodeResources); + } + } + } catch (FileNotFoundException fe) { + LOG.info("Couldn't find node resources file"); + } + return nodeResources; + } + + private static void addResourceInformation(String prop, String value, + Map nodeResources) { + String[] parts = prop.split("\\."); + LOG.info("Found resource entry " + prop); + if (parts.length == 4) { + String resourceType = parts[3]; + if (!nodeResources.containsKey(resourceType)) { + nodeResources + .put(resourceType, ResourceInformation.newInstance(resourceType)); + } + String units = getUnits(value); + Long resourceValue = + Long.valueOf(value.substring(0, value.length() - units.length())); + nodeResources.get(resourceType).setValue(resourceValue); + nodeResources.get(resourceType).setUnits(units); + LOG.debug("Setting value for resource type " + resourceType + " to " + + resourceValue + " with units " + units); + } + } + + @VisibleForTesting + synchronized public static void resetNodeResources() { + initializedNodeResources = false; + } + + public static Resource getResourceTypesMinimumAllocation() { + Resource ret = Resource.newInstance(0, 0); + for (ResourceInformation entry : resourceTypesArray) { + String name = entry.getName(); + if (name.equals(ResourceInformation.MEMORY_MB.getName())) { + ret.setMemorySize(entry.getMinimumAllocation()); + } else if (name.equals(ResourceInformation.VCORES.getName())) { + Long tmp = entry.getMinimumAllocation(); + if (tmp > Integer.MAX_VALUE) { + tmp = (long) Integer.MAX_VALUE; + } + ret.setVirtualCores(tmp.intValue()); + } else { + ret.setResourceValue(name, entry.getMinimumAllocation()); + } + } + return ret; + } + + /** + * Get a Resource object with for the maximum allocation possible. + * @return a Resource object with the maximum allocation for the scheduler + */ + public static Resource getResourceTypesMaximumAllocation() { + Resource ret = Resource.newInstance(0, 0); + for (ResourceInformation entry : resourceTypesArray) { + String name = entry.getName(); + if (name.equals(ResourceInformation.MEMORY_MB.getName())) { + ret.setMemorySize(entry.getMaximumAllocation()); + } else if (name.equals(ResourceInformation.VCORES.getName())) { + Long tmp = entry.getMaximumAllocation(); + if (tmp > Integer.MAX_VALUE) { + tmp = (long) Integer.MAX_VALUE; + } + ret.setVirtualCores(tmp.intValue()); + continue; + } else { + ret.setResourceValue(name, entry.getMaximumAllocation()); + } + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java new file mode 100644 index 0000000..01c7eb0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.util.resource; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index e8b0594..8176068 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -472,9 +472,8 @@ public class ProtoUtils { List pList) { Resource tmp = Resource.newInstance(0, 0); Map ret = new HashMap<>(); - for (Map.Entry entry : tmp.getResources() - .entrySet()) { - ret.put(entry.getKey(), 0L); + for (ResourceInformation entry : tmp.getResources()) { + ret.put(entry.getName(), 0L); } if (pList != null) { for (YarnProtos.StringLongMapProto p : pList) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 6e51efa..00be77a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.impl.BaseResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; @@ -33,14 +34,13 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.UnitsConversionUtil; -import java.util.HashMap; +import java.util.Arrays; import java.util.Map; -import java.util.Collections; @Private @Unstable -public class ResourcePBImpl extends Resource { +public class ResourcePBImpl extends BaseResource { private static final Log LOG = LogFactory.getLog(ResourcePBImpl.class); @@ -48,10 +48,6 @@ public class ResourcePBImpl extends Resource { ResourceProto.Builder builder = null; boolean viaProto = false; - private Map resources; - private Map readOnlyResources; - - // call via ProtoUtils.convertToProtoFormat(Resource) static ResourceProto getProto(Resource r) { final ResourcePBImpl pb; @@ -72,8 +68,6 @@ public class ResourcePBImpl extends Resource { public ResourcePBImpl(ResourceProto proto) { this.proto = proto; viaProto = true; - this.readOnlyResources = null; - this.resources = null; initResources(); } @@ -101,11 +95,13 @@ public class ResourcePBImpl extends Resource { public long getMemorySize() { // memory should always be present initResources(); - ResourceInformation ri = - this.getResourceInformation(ResourceInformation.MEMORY_MB.getName()); - return UnitsConversionUtil - .convert(ri.getUnits(), ResourceInformation.MEMORY_MB.getUnits(), - ri.getValue()); + ResourceInformation ri = resources[MandatoryResources.MEMORY.getId()]; + + if (ri.getUnits().isEmpty()) { + return ri.getValue(); + } + return UnitsConversionUtil.convert(ri.getUnits(), + ResourceInformation.MEMORY_MB.getUnits(), ri.getValue()); } @Override @@ -117,23 +113,20 @@ public class ResourcePBImpl extends Resource { @Override public void setMemorySize(long memory) { maybeInitBuilder(); - getResourceInformation(ResourceInformation.MEMORY_MB.getName()) - .setValue(memory); + getResourceInformation(MEMORY).setValue(memory); } @Override public int getVirtualCores() { // vcores should always be present initResources(); - return this.getResourceValue(ResourceInformation.VCORES.getName()) - .intValue(); + return (int) resources[MandatoryResources.VCORES.getId()].getValue(); } @Override public void setVirtualCores(int vCores) { maybeInitBuilder(); - getResourceInformation(ResourceInformation.VCORES.getName()) - .setValue(vCores); + getResourceInformation(VCORES).setValue(vCores); } private void initResources() { @@ -142,6 +135,7 @@ public class ResourcePBImpl extends Resource { } ResourceProtoOrBuilder p = viaProto ? proto : builder; initResourcesMap(); + Map indexMap = ResourceUtils.getResourceTypeIndex(); for (ResourceInformationProto entry : p.getResourceValueMapList()) { ResourceTypes type = entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) : @@ -150,14 +144,16 @@ public class ResourcePBImpl extends Resource { long value = entry.hasValue() ? entry.getValue() : 0L; ResourceInformation ri = ResourceInformation .newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE); - if (resources.containsKey(ri.getName())) { - resources.get(ri.getName()).setResourceType(ri.getResourceType()); - resources.get(ri.getName()).setUnits(ri.getUnits()); - resources.get(ri.getName()).setValue(value); - } else { + Integer index = indexMap.get(entry.getKey()); + if (index == null) { LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping"); + } else { + resources[index].setResourceType(ri.getResourceType()); + resources[index].setUnits(ri.getUnits()); + resources[index].setValue(value); } } + readOnlyResources = Arrays.copyOf(resources, resources.length); this.setMemorySize(p.getMemory()); this.setVirtualCores(p.getVirtualCores()); } @@ -173,79 +169,67 @@ public class ResourcePBImpl extends Resource { if (!resource.equals(resourceInformation.getName())) { resourceInformation.setName(resource); } - initResources(); - if (resources.containsKey(resource)) { - ResourceInformation.copy(resourceInformation, resources.get(resource)); - } + ResourceInformation storedResourceInfo = getResourceInformation(resource); + ResourceInformation.copy(resourceInformation, storedResourceInfo); } @Override public void setResourceValue(String resource, Long value) throws ResourceNotFoundException { maybeInitBuilder(); - initResources(); if (resource == null) { throw new IllegalArgumentException("resource type object cannot be null"); } - if (resources == null || (!resources.containsKey(resource))) { - throw new ResourceNotFoundException( - "Resource " + resource + " not found"); - } - resources.get(resource).setValue(value); + getResourceInformation(resource).setValue(value); } @Override - public Map getResources() { + public ResourceInformation[] getResources() { initResources(); - return readOnlyResources; + return super.getResources(); } @Override - public ResourceInformation getResourceInformation(String resource) { + public ResourceInformation getResourceInformation(String resource) + throws ResourceNotFoundException { initResources(); - if (this.resources.containsKey(resource)) { - return this.resources.get(resource); - } - throw new ResourceNotFoundException("Could not find entry for " + resource); + return super.getResourceInformation(resource); } @Override - public Long getResourceValue(String resource) { + public long getResourceValue(String resource) + throws ResourceNotFoundException { initResources(); - if (this.resources.containsKey(resource)) { - return this.resources.get(resource).getValue(); - } - throw new ResourceNotFoundException("Could not find entry for " + resource); + return super.getResourceValue(resource); } private void initResourcesMap() { if (resources == null) { - resources = new HashMap<>(); - Map types = ResourceUtils.getResourceTypes(); + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); if (types == null) { throw new YarnRuntimeException( "Got null return value from ResourceUtils.getResourceTypes()"); } - for (Map.Entry entry : types.entrySet()) { - resources.put(entry.getKey(), - ResourceInformation.newInstance(entry.getValue())); + + resources = new ResourceInformation[types.length]; + readOnlyResources = new ResourceInformation[types.length]; + for (ResourceInformation entry : types) { + int index = ResourceUtils.getResourceTypeIndex().get(entry.getName()); + resources[index] = ResourceInformation.newInstance(entry); } - readOnlyResources = Collections.unmodifiableMap(resources); } } synchronized private void mergeLocalToBuilder() { builder.clearResourceValueMap(); - if (resources != null && !resources.isEmpty()) { - for (Map.Entry entry : - resources.entrySet()) { - ResourceInformationProto.Builder e = - ResourceInformationProto.newBuilder(); - e.setKey(entry.getKey()); - e.setUnits(entry.getValue().getUnits()); - e.setType( - ProtoUtils.converToProtoFormat(entry.getValue().getResourceType())); - e.setValue(entry.getValue().getValue()); + if(resources != null && resources.length != 0) { + for (ResourceInformation resInfo : resources) { + ResourceInformationProto.Builder e = ResourceInformationProto + .newBuilder(); + e.setKey(resInfo.getName()); + e.setUnits(resInfo.getUnits()); + e.setType(ProtoUtils.converToProtoFormat(resInfo.getResourceType())); + e.setValue(resInfo.getValue()); builder.addResourceValueMap(e); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e264ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 79bb03d..ea5c8a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -23,10 +23,9 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; -import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.util.UnitsConversionUtil; -import java.util.Set; /** * A {@link ResourceCalculator} which uses the concept of @@ -56,10 +55,10 @@ public class DominantResourceCalculator extends ResourceCalculator { LogFactory.getLog(DominantResourceCalculator.class); - private Set resourceNames; + private String[] resourceNames; public DominantResourceCalculator() { - resourceNames = ResourceUtils.getResourceTypes().keySet(); + resourceNames = ResourceUtils.getResourceNamesArray(); } /** @@ -88,7 +87,7 @@ public class DominantResourceCalculator extends ResourceCalculator { } else if (diff <= -1) { rhsGreater = true; } - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { throw new IllegalArgumentException( "Error getting resource information for " + rName, ye); } @@ -163,7 +162,7 @@ public class DominantResourceCalculator extends ResourceCalculator { .getValue(); min = min < tmp ? min : tmp; max = max > tmp ? max : tmp; - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); } @@ -187,7 +186,7 @@ public class DominantResourceCalculator extends ResourceCalculator { long tmp = availableResource.getValue() / requiredResourceValue; min = min < tmp ? min : tmp; } - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); } @@ -206,14 +205,9 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public boolean isInvalidDivisor(Resource r) { - for (String resource : resourceNames) { - try { - if (r.getResourceValue(resource).equals(0L)) { - return true; - } - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource value for " + resource, ye); + for (ResourceInformation res : r.getResources()) { + if (res.getValue() == 0L) { + return true; } } return false; @@ -235,7 +229,7 @@ public class DominantResourceCalculator extends ResourceCalculator { float tmp = (float) aResourceInformation.getValue() / (float) bResourceValue; ratio = ratio > tmp ? ratio : tmp; - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); } @@ -256,7 +250,7 @@ public class DominantResourceCalculator extends ResourceCalculator { ret.getResourceInformation(resource); resourceInformation.setValue( divideAndCeil(resourceInformation.getValue(), denominator)); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); } @@ -307,7 +301,7 @@ public class DominantResourceCalculator extends ResourceCalculator { } tmp.setValue(Math.min(value, maximumValue)); ret.setResourceInformation(resource, tmp); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); } @@ -347,7 +341,7 @@ public class DominantResourceCalculator extends ResourceCalculator { ResourceInformation .copy(rResourceInformation, ret.getResourceInformation(resource)); ret.getResourceInformation(resource).setValue(value); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); } @@ -372,28 +366,29 @@ public class DominantResourceCalculator extends ResourceCalculator { Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { try { - ResourceInformation rResourceInformation = - r.getResourceInformation(resource); - ResourceInformation stepFactorResourceInformation = - stepFactor.getResourceInformation(resource); + ResourceInformation rResourceInformation = r + .getResourceInformation(resource); + ResourceInformation stepFactorResourceInformation = stepFactor + .getResourceInformation(resource); ResourceInformation tmp = ret.getResourceInformation(resource); - Long rValue = rResourceInformation.getValue(); - Long stepFactorValue = UnitsConversionUtil - .convert(stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); - Long value; + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil.convert( + stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + long value; if (stepFactorValue != 0) { - value = roundUp ? - roundUp((long) Math.ceil(rValue * by), stepFactorValue) : - roundDown((long) (rValue * by), stepFactorValue); + value = roundUp + ? roundUp((long) Math.ceil(rValue * by), stepFactorValue) + : roundDown((long) (rValue * by), stepFactorValue); } else { - value = - roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by); + value = roundUp + ? (long) Math.ceil(rValue * by) + : (long) (rValue * by); } tmp.setValue(value); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); } @@ -416,7 +411,7 @@ public class DominantResourceCalculator extends ResourceCalculator { if(sResourceValue > bResourceInformation.getValue()) { return false; } - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { return false; } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org