hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject hadoop git commit: YARN-7410. Cleanup FixedValueResource to avoid dependency to ResourceUtils. (wangda)
Date Mon, 06 Nov 2017 23:27:01 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 170b6a48c -> df27824b8


YARN-7410. Cleanup FixedValueResource to avoid dependency to ResourceUtils. (wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df27824b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df27824b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df27824b

Branch: refs/heads/branch-3.0
Commit: df27824b80b8eca03d02837efe2da31f089e67ec
Parents: 170b6a4
Author: Wangda Tan <wangda@apache.org>
Authored: Mon Nov 6 15:26:49 2017 -0800
Committer: Wangda Tan <wangda@apache.org>
Committed: Mon Nov 6 15:26:49 2017 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/api/records/Resource.java       |  2 +-
 .../yarn/util/resource/ResourceUtils.java       |  1 +
 .../hadoop/yarn/util/resource/Resources.java    | 59 ++++++++++++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/df27824b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 1fabae8..14131cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -364,7 +364,7 @@ public abstract class Resource implements Comparable<Resource> {
     }
   }
 
-  private void throwExceptionWhenArrayOutOfBound(int index) {
+  protected void throwExceptionWhenArrayOutOfBound(int index) {
     String exceptionMsg = String.format(
         "Trying to access ResourceInformation for given index=%d. "
             + "Acceptable index range is [0,%d), please check double check "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df27824b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 674f9de..703f325 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -266,6 +266,7 @@ public class ResourceUtils {
     updateKnownResources();
     updateResourceTypeIndex();
     initializedResources = true;
+    numKnownResourceTypes = resourceTypes.size();
   }
 
   private static void updateKnownResources() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df27824b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 793aebf..e044ea3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -125,6 +125,65 @@ public class Resources {
       throw new RuntimeException(name + " cannot be modified!");
     }
 
+    /*
+     *  FixedValueResource cannot be updated when any resource types refresh
+     *  by using approach introduced by YARN-7307 and do operations like
+     *  Resources.compare(resource_x, Resources.none()) will throw exceptions.
+     *
+     *  That's why we do reinitialize resource maps for following methods.
+     */
+
+    @Override
+    public ResourceInformation getResourceInformation(int index)
+        throws ResourceNotFoundException {
+      ResourceInformation ri = null;
+      try {
+        ri = super.getResourceInformation(index);
+      } catch (ResourceNotFoundException e) {
+        // Retry once to reinitialize resource information.
+        initResourceMap();
+        try {
+          return super.getResourceInformation(index);
+        } catch (ResourceNotFoundException ee) {
+          throwExceptionWhenArrayOutOfBound(index);
+        }
+      }
+      return ri;
+    }
+
+    @Override
+    public ResourceInformation getResourceInformation(String resource)
+        throws ResourceNotFoundException {
+      ResourceInformation ri;
+      try {
+        ri = super.getResourceInformation(resource);
+      } catch (ResourceNotFoundException e) {
+        // Retry once to reinitialize resource information.
+        initResourceMap();
+        try {
+          return super.getResourceInformation(resource);
+        } catch (ResourceNotFoundException ee) {
+          throw ee;
+        }
+      }
+      return ri;
+    }
+
+    @Override
+    public ResourceInformation[] getResources() {
+      if (resources.length != ResourceUtils.getNumberOfKnownResourceTypes()) {
+        // Retry once to reinitialize resource information.
+        initResourceMap();
+        if (resources.length != ResourceUtils.getNumberOfKnownResourceTypes()) {
+          throw new ResourceNotFoundException("Failed to reinitialize "
+              + "FixedValueResource to get number of resource types same "
+              + "as configured");
+        }
+      }
+
+      return resources;
+    }
+
     private void initResourceMap() {
       ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
       if (types != null) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message