hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bra...@apache.org
Subject [30/49] hadoop git commit: YARN-8987. Usability improvements node-attributes CLI. Contributed by Bibin A Chundatt.
Date Tue, 13 Nov 2018 07:50:57 GMT
YARN-8987. Usability improvements node-attributes CLI. Contributed by  Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7411095
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7411095
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7411095

Branch: refs/heads/HDFS-13891
Commit: c741109522d2913b87638957c64b94dee6b51029
Parents: 9c32b50
Author: Weiwei Yang <wwei@apache.org>
Authored: Mon Nov 12 18:18:23 2018 +0800
Committer: Weiwei Yang <wwei@apache.org>
Committed: Mon Nov 12 18:18:23 2018 +0800

----------------------------------------------------------------------
 .../yarn/client/cli/NodeAttributesCLI.java      |  3 +++
 .../server/resourcemanager/AdminService.java    | 22 +++++++++++++++
 .../resourcemanager/TestRMAdminService.java     | 28 ++++++++++++++++++++
 3 files changed, 53 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7411095/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
index 13d5e24..d525087 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
@@ -205,6 +205,9 @@ public class NodeAttributesCLI extends Configured implements Tool {
       // print admin command detail
       printUsage(true, handler);
       return exitCode;
+    } catch (YarnException e) {
+      errOut.println(e.toString());
+      return exitCode;
     } catch (Exception e) {
       errOut.println(e.toString());
       printUsage(true, handler);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7411095/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 601917a..880741a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -993,6 +993,7 @@ public class AdminService extends CompositeService implements
         nodeAttributesManager.addNodeAttributes(nodeAttributeMapping);
         break;
       case REMOVE:
+        validateAttributesExists(nodesToAttributes);
         nodeAttributesManager.removeNodeAttributes(nodeAttributeMapping);
         break;
       case REPLACE:
@@ -1013,6 +1014,27 @@ public class AdminService extends CompositeService implements
         .newRecordInstance(NodesToAttributesMappingResponse.class);
   }
 
+  private void validateAttributesExists(
+      List<NodeToAttributes> nodesToAttributes) throws IOException {
+    NodeAttributesManager nodeAttributesManager =
+        rm.getRMContext().getNodeAttributesManager();
+    for (NodeToAttributes nodeToAttrs : nodesToAttributes) {
+      String hostname = nodeToAttrs.getNode();
+      if (hostname == null) {
+        continue;
+      }
+      Set<NodeAttribute> attrs =
+          nodeAttributesManager.getAttributesForNode(hostname).keySet();
+      List<NodeAttribute> attributes = nodeToAttrs.getNodeAttributes();
+      for (NodeAttribute nodeAttr : attributes) {
+        if (!attrs.contains(nodeAttr)) {
+          throw new IOException("Node attribute [" + nodeAttr.getAttributeKey()
+              + "] doesn't exist on node " + nodeToAttrs.getNode());
+        }
+      }
+    }
+  }
+
   /**
    * @param nodesToAttributesMapping input to be validated
    * @param failOnUnknownNodes indicates to fail if the nodes are not available.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7411095/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 475dd8c..01c89d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -1585,6 +1585,16 @@ public class TestRMAdminService {
     Mockito.verify(spiedAttributesManager, Mockito.times(1))
         .addNodeAttributes(Mockito.anyMap());
 
+    // add attributes for later removals
+    request =
+        NodesToAttributesMappingRequest
+            .newInstance(AttributeMappingOperationType.ADD,
+                ImmutableList.of(NodeToAttributes.newInstance("host4",
+                    ImmutableList.of(NodeAttribute.newInstance(
+                        NodeAttribute.PREFIX_CENTRALIZED, "x",
+                        NodeAttributeType.STRING, "dfasdf")))),
+                true);
+    rm.adminService.mapAttributesToNodes(request);
     request =
         NodesToAttributesMappingRequest
             .newInstance(AttributeMappingOperationType.REMOVE,
@@ -1601,6 +1611,24 @@ public class TestRMAdminService {
     Mockito.verify(spiedAttributesManager, Mockito.times(1))
         .removeNodeAttributes(Mockito.anyMap());
 
+    // Assert node to attributes mappings are empty.
+    Assert.assertTrue("Attributes of host4 should be empty",
+        rm.getRMContext().getNodeAttributesManager()
+            .getAttributesForNode("host4").isEmpty());
+    // remove non existing attributes.
+    request = NodesToAttributesMappingRequest
+        .newInstance(AttributeMappingOperationType.REMOVE, ImmutableList
+            .of(NodeToAttributes.newInstance("host4", ImmutableList
+                .of(NodeAttribute
+                    .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "x",
+                        NodeAttributeType.STRING, "dfasdf")))), true);
+    try {
+      rm.adminService.mapAttributesToNodes(request);
+      fail("Should have failed for non exists attribute");
+    } catch (Exception ex) {
+      assertTrue("Exception expected if attributes does not exist", true);
+    }
+
     request =
         NodesToAttributesMappingRequest
             .newInstance(AttributeMappingOperationType.ADD,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message