hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [41/45] hadoop git commit: YARN-7030. [YARN-3926] Performance optimizations in Resource and ResourceUtils class. Contributed by Wangda Tan.
Date Tue, 22 Aug 2017 00:18:11 GMT
YARN-7030. [YARN-3926] Performance optimizations in Resource and ResourceUtils class. Contributed
by Wangda Tan.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d6ca4b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d6ca4b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d6ca4b7

Branch: refs/heads/YARN-3926
Commit: 9d6ca4b7588cf8955cbc5f9d2cc43090d1fccb6b
Parents: a77d6d4
Author: Sunil G <sunilg@apache.org>
Authored: Thu Aug 17 21:14:51 2017 +0530
Committer: Wangda Tan <wangda@apache.org>
Committed: Mon Aug 21 16:56:02 2017 -0700

----------------------------------------------------------------------
 .../hadoop/yarn/api/records/Resource.java       |  6 ++---
 .../yarn/api/records/impl/BaseResource.java     | 10 +++++---
 .../yarn/util/resource/ResourceUtils.java       | 25 ++++++++++++++++----
 .../hadoop/yarn/util/resource/Resources.java    | 16 ++++++-------
 4 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d6ca4b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 1e9f213..04579c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -65,7 +65,7 @@ public abstract class Resource implements Comparable<Resource> {
   @Public
   @Stable
   public static Resource newInstance(int memory, int vCores) {
-    if (ResourceUtils.getResourceTypesArray().length > 2) {
+    if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
       Resource ret = Records.newRecord(Resource.class);
       ret.setMemorySize(memory);
       ret.setVirtualCores(vCores);
@@ -77,7 +77,7 @@ public abstract class Resource implements Comparable<Resource> {
   @Public
   @Stable
   public static Resource newInstance(long memory, int vCores) {
-    if (ResourceUtils.getResourceTypesArray().length > 2) {
+    if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
       Resource ret = Records.newRecord(Resource.class);
       ret.setMemorySize(memory);
       ret.setVirtualCores(vCores);
@@ -91,7 +91,7 @@ public abstract class Resource implements Comparable<Resource> {
   public static Resource newInstance(Resource resource) {
     Resource ret = Resource.newInstance(resource.getMemorySize(),
         resource.getVirtualCores());
-    if (ResourceUtils.getResourceTypesArray().length > 2) {
+    if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
       Resource.copy(resource, ret);
     }
     return ret;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d6ca4b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
index 83db542..b5cc4d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
@@ -63,6 +63,11 @@ public class BaseResource extends Resource {
   protected ResourceInformation[] resources = null;
   protected ResourceInformation[] readOnlyResources = null;
 
+  // Number of mandatory resources, this is added to avoid invoke
+  // MandatoryResources.values().length, since values() internally will
+  // copy array, etc.
+  private static final int NUM_MANDATORY_RESOURCES = 2;
+
   protected enum MandatoryResources {
     MEMORY(0), VCORES(1);
 
@@ -86,9 +91,8 @@ public class BaseResource extends Resource {
         ResourceInformation.MEMORY_MB.getUnits(), memory);
     this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores);
 
-    resources = new ResourceInformation[MandatoryResources.values().length];
-    readOnlyResources = new ResourceInformation[MandatoryResources
-        .values().length];
+    resources = new ResourceInformation[NUM_MANDATORY_RESOURCES];
+    readOnlyResources = new ResourceInformation[NUM_MANDATORY_RESOURCES];
     resources[MandatoryResources.MEMORY.id] = memoryResInfo;
     resources[MandatoryResources.VCORES.id] = vcoresResInfo;
     readOnlyResources = Arrays.copyOf(resources, resources.length);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d6ca4b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 49feafa..06a5414 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -74,6 +74,7 @@ public class ResourceUtils {
   private static volatile ResourceInformation[] resourceTypesArray;
   private static volatile boolean initializedNodeResources = false;
   private static volatile Map<String, ResourceInformation> readOnlyNodeResources;
+  private static volatile int numKnownResourceTypes = -1;
 
   static final Log LOG = LogFactory.getLog(ResourceUtils.class);
 
@@ -308,23 +309,33 @@ public class ResourceUtils {
    * @return resourceNamesArray
    */
   public static String[] getResourceNamesArray() {
-    getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+    initializeResourceTypesIfNeeded(null,
+        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
     return resourceNamesArray;
   }
 
   public static ResourceInformation[] getResourceTypesArray() {
-    getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+    initializeResourceTypesIfNeeded(null,
+        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
     return resourceTypesArray;
   }
 
+  public static int getNumberOfKnownResourceTypes() {
+    if (numKnownResourceTypes < 0) {
+      initializeResourceTypesIfNeeded(null,
+          YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+    }
+    return numKnownResourceTypes;
+  }
+
   private static Map<String, ResourceInformation> getResourceTypes(
       Configuration conf) {
     return getResourceTypes(conf,
         YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
   }
 
-  private static Map<String, ResourceInformation> getResourceTypes(
-      Configuration conf, String resourceFile) {
+  private static void initializeResourceTypesIfNeeded(Configuration conf,
+      String resourceFile) {
     if (!initializedResources) {
       synchronized (ResourceUtils.class) {
         if (!initializedResources) {
@@ -346,6 +357,12 @@ public class ResourceUtils {
         }
       }
     }
+    numKnownResourceTypes = resourceTypes.size();
+  }
+
+  private static Map<String, ResourceInformation> getResourceTypes(
+      Configuration conf, String resourceFile) {
+    initializeResourceTypesIfNeeded(conf, resourceFile);
     return resourceTypes;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d6ca4b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 702c1c8..54ef036 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -173,7 +173,7 @@ public class Resources {
   }
 
   public static Resource addTo(Resource lhs, Resource rhs) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -197,7 +197,7 @@ public class Resources {
   }
 
   public static Resource subtractFrom(Resource lhs, Resource rhs) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -243,7 +243,7 @@ public class Resources {
   }
 
   public static Resource multiplyTo(Resource lhs, double by) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation lhsValue = lhs.getResourceInformation(i);
@@ -266,7 +266,7 @@ public class Resources {
    */
   public static Resource multiplyAndAddTo(
       Resource lhs, Resource rhs, double by) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -299,7 +299,7 @@ public class Resources {
   
   public static Resource multiplyAndRoundDown(Resource lhs, double by) {
     Resource out = clone(lhs);
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation lhsValue = lhs.getResourceInformation(i);
@@ -408,7 +408,7 @@ public class Resources {
   }
   
   public static boolean fitsIn(Resource smaller, Resource bigger) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = bigger.getResourceInformation(i);
@@ -436,7 +436,7 @@ public class Resources {
   
   public static Resource componentwiseMin(Resource lhs, Resource rhs) {
     Resource ret = createResource(0);
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -460,7 +460,7 @@ public class Resources {
   
   public static Resource componentwiseMax(Resource lhs, Resource rhs) {
     Resource ret = createResource(0);
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message