Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 3DBD9200CF8 for ; Thu, 31 Aug 2017 00:58:52 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 3C6E016A24E; Wed, 30 Aug 2017 22:58:52 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3E43C16A23D for ; Thu, 31 Aug 2017 00:58:51 +0200 (CEST) Received: (qmail 59053 invoked by uid 500); 30 Aug 2017 22:58:43 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 51552 invoked by uid 99); 30 Aug 2017 22:58:38 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Aug 2017 22:58:38 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2C6C0F55C5; Wed, 30 Aug 2017 22:58:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Date: Wed, 30 Aug 2017 22:59:11 -0000 Message-Id: <09433e4fa9ed4689becefe0502b60116@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [36/50] [abbrv] hadoop git commit: YARN-7030. [YARN-3926] Performance optimizations in Resource and ResourceUtils class. Contributed by Wangda Tan. archived-at: Wed, 30 Aug 2017 22:58:52 -0000 YARN-7030. [YARN-3926] Performance optimizations in Resource and ResourceUtils class. Contributed by Wangda Tan. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b6327e7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b6327e7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b6327e7 Branch: refs/heads/YARN-3926 Commit: 7b6327e7bcdb3e9f23a1558bb942c0f5e4f76f54 Parents: 84bebc8 Author: Sunil G Authored: Thu Aug 17 21:14:51 2017 +0530 Committer: Wangda Tan Committed: Wed Aug 30 15:50:35 2017 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/api/records/Resource.java | 6 ++--- .../yarn/api/records/impl/BaseResource.java | 10 +++++--- .../yarn/util/resource/ResourceUtils.java | 25 ++++++++++++++++---- .../hadoop/yarn/util/resource/Resources.java | 16 ++++++------- 4 files changed, 39 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b6327e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 1e9f213..04579c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -65,7 +65,7 @@ public abstract class Resource implements Comparable { @Public @Stable public static Resource newInstance(int memory, int vCores) { - if (ResourceUtils.getResourceTypesArray().length > 2) { + if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { Resource ret = Records.newRecord(Resource.class); ret.setMemorySize(memory); ret.setVirtualCores(vCores); @@ -77,7 +77,7 @@ public abstract class Resource implements Comparable { @Public @Stable public static Resource newInstance(long memory, int vCores) { - if (ResourceUtils.getResourceTypesArray().length > 2) { + if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { Resource ret = Records.newRecord(Resource.class); ret.setMemorySize(memory); ret.setVirtualCores(vCores); @@ -91,7 +91,7 @@ public abstract class Resource implements Comparable { public static Resource newInstance(Resource resource) { Resource ret = Resource.newInstance(resource.getMemorySize(), resource.getVirtualCores()); - if (ResourceUtils.getResourceTypesArray().length > 2) { + if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { Resource.copy(resource, ret); } return ret; http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b6327e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java index 83db542..b5cc4d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java @@ -63,6 +63,11 @@ public class BaseResource extends Resource { protected ResourceInformation[] resources = null; protected ResourceInformation[] readOnlyResources = null; + // Number of mandatory resources, this is added to avoid invoke + // MandatoryResources.values().length, since values() internally will + // copy array, etc. + private static final int NUM_MANDATORY_RESOURCES = 2; + protected enum MandatoryResources { MEMORY(0), VCORES(1); @@ -86,9 +91,8 @@ public class BaseResource extends Resource { ResourceInformation.MEMORY_MB.getUnits(), memory); this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores); - resources = new ResourceInformation[MandatoryResources.values().length]; - readOnlyResources = new ResourceInformation[MandatoryResources - .values().length]; + resources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; + readOnlyResources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; resources[MandatoryResources.MEMORY.id] = memoryResInfo; resources[MandatoryResources.VCORES.id] = vcoresResInfo; readOnlyResources = Arrays.copyOf(resources, resources.length); http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b6327e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 49feafa..06a5414 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -74,6 +74,7 @@ public class ResourceUtils { private static volatile ResourceInformation[] resourceTypesArray; private static volatile boolean initializedNodeResources = false; private static volatile Map readOnlyNodeResources; + private static volatile int numKnownResourceTypes = -1; static final Log LOG = LogFactory.getLog(ResourceUtils.class); @@ -308,23 +309,33 @@ public class ResourceUtils { * @return resourceNamesArray */ public static String[] getResourceNamesArray() { - getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + initializeResourceTypesIfNeeded(null, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); return resourceNamesArray; } public static ResourceInformation[] getResourceTypesArray() { - getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + initializeResourceTypesIfNeeded(null, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); return resourceTypesArray; } + public static int getNumberOfKnownResourceTypes() { + if (numKnownResourceTypes < 0) { + initializeResourceTypesIfNeeded(null, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + } + return numKnownResourceTypes; + } + private static Map getResourceTypes( Configuration conf) { return getResourceTypes(conf, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); } - private static Map getResourceTypes( - Configuration conf, String resourceFile) { + private static void initializeResourceTypesIfNeeded(Configuration conf, + String resourceFile) { if (!initializedResources) { synchronized (ResourceUtils.class) { if (!initializedResources) { @@ -346,6 +357,12 @@ public class ResourceUtils { } } } + numKnownResourceTypes = resourceTypes.size(); + } + + private static Map getResourceTypes( + Configuration conf, String resourceFile) { + initializeResourceTypesIfNeeded(conf, resourceFile); return resourceTypes; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b6327e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 702c1c8..54ef036 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -173,7 +173,7 @@ public class Resources { } public static Resource addTo(Resource lhs, Resource rhs) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -197,7 +197,7 @@ public class Resources { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -243,7 +243,7 @@ public class Resources { } public static Resource multiplyTo(Resource lhs, double by) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); @@ -266,7 +266,7 @@ public class Resources { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -299,7 +299,7 @@ public class Resources { public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); @@ -408,7 +408,7 @@ public class Resources { } public static boolean fitsIn(Resource smaller, Resource bigger) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = bigger.getResourceInformation(i); @@ -436,7 +436,7 @@ public class Resources { public static Resource componentwiseMin(Resource lhs, Resource rhs) { Resource ret = createResource(0); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -460,7 +460,7 @@ public class Resources { public static Resource componentwiseMax(Resource lhs, Resource rhs) { Resource ret = createResource(0); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org