hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sun...@apache.org
Subject hadoop git commit: YARN-7410. Cleanup FixedValueResource to avoid dependency to ResourceUtils. Contributed by Wangda Tan.
Date Fri, 03 Nov 2017 01:55:50 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk ad0fff2b4 -> 1700adc6f


YARN-7410. Cleanup FixedValueResource to avoid dependency to ResourceUtils. Contributed by
Wangda Tan.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1700adc6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1700adc6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1700adc6

Branch: refs/heads/trunk
Commit: 1700adc6f7e0ec9f855f554cd0d470a8f04e9c51
Parents: ad0fff2
Author: Sunil G <sunilg@apache.org>
Authored: Fri Nov 3 07:25:29 2017 +0530
Committer: Sunil G <sunilg@apache.org>
Committed: Fri Nov 3 07:25:29 2017 +0530

----------------------------------------------------------------------
 .../hadoop/yarn/api/records/Resource.java       |  2 +-
 .../yarn/util/resource/ResourceUtils.java       |  1 +
 .../hadoop/yarn/util/resource/Resources.java    | 59 ++++++++++++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1700adc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 6bdde18..be292ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -364,7 +364,7 @@ public abstract class Resource implements Comparable<Resource> {
     }
   }
 
-  private void throwExceptionWhenArrayOutOfBound(int index) {
+  protected void throwExceptionWhenArrayOutOfBound(int index) {
     String exceptionMsg = String.format(
         "Trying to access ResourceInformation for given index=%d. "
             + "Acceptable index range is [0,%d), please check double check "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1700adc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 8f75909..c9cc27b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -267,6 +267,7 @@ public class ResourceUtils {
     updateKnownResources();
     updateResourceTypeIndex();
     initializedResources = true;
+    numKnownResourceTypes = resourceTypes.size();
   }
 
   private static void updateKnownResources() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1700adc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 3690946..068e7f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -118,6 +118,65 @@ public class Resources {
       throw new RuntimeException(name + " cannot be modified!");
     }
 
+    /*
+     *  FixedValueResource cannot be updated when any resource types refresh
+     *  by using approach introduced by YARN-7307 and do operations like
+     *  Resources.compare(resource_x, Resources.none()) will throw exceptions.
+     *
+     *  That's why we do reinitialize resource maps for following methods.
+     */
+
+    @Override
+    public ResourceInformation getResourceInformation(int index)
+        throws ResourceNotFoundException {
+      ResourceInformation ri = null;
+      try {
+        ri = super.getResourceInformation(index);
+      } catch (ResourceNotFoundException e) {
+        // Retry once to reinitialize resource information.
+        initResourceMap();
+        try {
+          return super.getResourceInformation(index);
+        } catch (ResourceNotFoundException ee) {
+          throwExceptionWhenArrayOutOfBound(index);
+        }
+      }
+      return ri;
+    }
+
+    @Override
+    public ResourceInformation getResourceInformation(String resource)
+        throws ResourceNotFoundException {
+      ResourceInformation ri;
+      try {
+        ri = super.getResourceInformation(resource);
+      } catch (ResourceNotFoundException e) {
+        // Retry once to reinitialize resource information.
+        initResourceMap();
+        try {
+          return super.getResourceInformation(resource);
+        } catch (ResourceNotFoundException ee) {
+          throw ee;
+        }
+      }
+      return ri;
+    }
+
+    @Override
+    public ResourceInformation[] getResources() {
+      if (resources.length != ResourceUtils.getNumberOfKnownResourceTypes()) {
+        // Retry once to reinitialize resource information.
+        initResourceMap();
+        if (resources.length != ResourceUtils.getNumberOfKnownResourceTypes()) {
+          throw new ResourceNotFoundException("Failed to reinitialize "
+              + "FixedValueResource to get number of resource types same "
+              + "as configured");
+        }
+      }
+
+      return resources;
+    }
+
     private void initResourceMap() {
       ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
       if (types != null) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message