hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [32/50] [abbrv] hadoop git commit: YARN-7041. Nodemanager NPE running jobs with security off. Contributed by Varun Saxena.
Date Thu, 31 Aug 2017 16:41:11 GMT
YARN-7041. Nodemanager NPE running jobs with security off. Contributed by Varun Saxena.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e276c75e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e276c75e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e276c75e

Branch: refs/heads/HADOOP-13345
Commit: e276c75ec17634fc3b521fdb15b6ac141b001274
Parents: 32188d3
Author: Rohith Sharma K S <rohithsharmaks@apache.org>
Authored: Sat Aug 19 11:37:17 2017 +0530
Committer: Varun Saxena <varunsaxena@apache.org>
Committed: Wed Aug 30 11:29:54 2017 +0530

----------------------------------------------------------------------
 .../records/impl/pb/CollectorInfoPBImpl.java    |  8 +++--
 .../impl/pb/NodeHeartbeatRequestPBImpl.java     | 22 +++++++++----
 .../impl/pb/NodeHeartbeatResponsePBImpl.java    | 23 ++++++++-----
 .../records/impl/pb/AppCollectorDataPBImpl.java |  8 +++--
 .../hadoop/yarn/TestYarnServerApiClasses.java   | 34 +++++++++++++++++---
 5 files changed, 71 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e276c75e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java
index bb54133..5835d1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java
@@ -114,9 +114,13 @@ public class CollectorInfoPBImpl extends CollectorInfo {
   @Override
   public Token getCollectorToken() {
     CollectorInfoProtoOrBuilder p = viaProto ? proto : builder;
-    if (this.collectorToken == null && p.hasCollectorToken()) {
-      this.collectorToken = convertFromProtoFormat(p.getCollectorToken());
+    if (this.collectorToken != null) {
+      return this.collectorToken;
+    }
+    if (!p.hasCollectorToken()) {
+      return null;
     }
+    this.collectorToken = convertFromProtoFormat(p.getCollectorToken());
     return this.collectorToken;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e276c75e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
index c07a6eb..1ffd223 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
@@ -168,12 +168,17 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest
{
     for (Map.Entry<ApplicationId, AppCollectorData> entry :
         registeringCollectors.entrySet()) {
       AppCollectorData data = entry.getValue();
-      builder.addRegisteringCollectors(AppCollectorDataProto.newBuilder()
-          .setAppId(convertToProtoFormat(entry.getKey()))
-          .setAppCollectorAddr(data.getCollectorAddr())
-          .setAppCollectorToken(convertToProtoFormat(data.getCollectorToken()))
-          .setRmIdentifier(data.getRMIdentifier())
-          .setVersion(data.getVersion()));
+      AppCollectorDataProto.Builder appCollectorDataBuilder =
+          AppCollectorDataProto.newBuilder()
+              .setAppId(convertToProtoFormat(entry.getKey()))
+              .setAppCollectorAddr(data.getCollectorAddr())
+              .setRmIdentifier(data.getRMIdentifier())
+              .setVersion(data.getVersion());
+      if (data.getCollectorToken() != null) {
+        appCollectorDataBuilder.setAppCollectorToken(
+            convertToProtoFormat(data.getCollectorToken()));
+      }
+      builder.addRegisteringCollectors(appCollectorDataBuilder);
     }
   }
 
@@ -274,7 +279,10 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest
{
       this.registeringCollectors = new HashMap<>();
       for (AppCollectorDataProto c : list) {
         ApplicationId appId = convertFromProtoFormat(c.getAppId());
-        Token collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
+        Token collectorToken = null;
+        if (c.hasAppCollectorToken()){
+          collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
+        }
         AppCollectorData data = AppCollectorData.newInstance(appId,
             c.getAppCollectorAddr(), c.getRmIdentifier(), c.getVersion(),
             collectorToken);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e276c75e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index a1e6a21..11f5f61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -154,13 +154,17 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse
{
     for (Map.Entry<ApplicationId, AppCollectorData> entry
         : appCollectorsMap.entrySet()) {
       AppCollectorData data = entry.getValue();
-      builder.addAppCollectors(AppCollectorDataProto.newBuilder()
-          .setAppId(convertToProtoFormat(entry.getKey()))
-          .setAppCollectorAddr(data.getCollectorAddr())
-          .setAppCollectorToken(
-              convertToProtoFormat(entry.getValue().getCollectorToken()))
-          .setRmIdentifier(data.getRMIdentifier())
-          .setVersion(data.getVersion()));
+      AppCollectorDataProto.Builder appCollectorDataBuilder =
+          AppCollectorDataProto.newBuilder()
+              .setAppId(convertToProtoFormat(entry.getKey()))
+              .setAppCollectorAddr(data.getCollectorAddr())
+              .setRmIdentifier(data.getRMIdentifier())
+              .setVersion(data.getVersion());
+      if (data.getCollectorToken() != null) {
+        appCollectorDataBuilder.setAppCollectorToken(
+            convertToProtoFormat(data.getCollectorToken()));
+      }
+      builder.addAppCollectors(appCollectorDataBuilder);
     }
   }
 
@@ -604,7 +608,10 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse
{
       this.appCollectorsMap = new HashMap<>();
       for (AppCollectorDataProto c : list) {
         ApplicationId appId = convertFromProtoFormat(c.getAppId());
-        Token collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
+        Token collectorToken = null;
+        if (c.hasAppCollectorToken()){
+          collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
+        }
         AppCollectorData data = AppCollectorData.newInstance(appId,
             c.getAppCollectorAddr(), c.getRmIdentifier(), c.getVersion(),
             collectorToken);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e276c75e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java
index 7144f51..c08e9ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java
@@ -163,9 +163,13 @@ public class AppCollectorDataPBImpl extends AppCollectorData {
   @Override
   public Token getCollectorToken() {
     AppCollectorDataProtoOrBuilder p = viaProto ? proto : builder;
-    if (this.collectorToken == null && p.hasAppCollectorToken()) {
-      this.collectorToken = new TokenPBImpl(p.getAppCollectorToken());
+    if (this.collectorToken != null) {
+      return this.collectorToken;
+    }
+    if (!p.hasAppCollectorToken()) {
+      return null;
     }
+    this.collectorToken = new TokenPBImpl(p.getAppCollectorToken());
     return this.collectorToken;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e276c75e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
index a1890dd..8b1d0bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
@@ -111,7 +111,7 @@ public class TestYarnServerApiClasses {
     original.setLastKnownNMTokenMasterKey(getMasterKey());
     original.setNodeStatus(getNodeStatus());
     original.setNodeLabels(getValidNodeLabels());
-    Map<ApplicationId, AppCollectorData> collectors = getCollectors();
+    Map<ApplicationId, AppCollectorData> collectors = getCollectors(false);
     original.setRegisteringCollectors(collectors);
     NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl(
         original.getProto());
@@ -130,6 +130,16 @@ public class TestYarnServerApiClasses {
     Assert.assertEquals(0, copy.getNodeLabels().size());
   }
 
+  @Test
+  public void testNodeHBRequestPBImplWithNullCollectorToken() {
+    NodeHeartbeatRequestPBImpl original = new NodeHeartbeatRequestPBImpl();
+    Map<ApplicationId, AppCollectorData> collectors = getCollectors(true);
+    original.setRegisteringCollectors(collectors);
+    NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl(
+        original.getProto());
+    assertEquals(collectors, copy.getRegisteringCollectors());
+  }
+
   /**
    * Test NodeHeartbeatRequestPBImpl.
    */
@@ -155,7 +165,7 @@ public class TestYarnServerApiClasses {
     original.setNextHeartBeatInterval(1000);
     original.setNodeAction(NodeAction.NORMAL);
     original.setResponseId(100);
-    Map<ApplicationId, AppCollectorData> collectors = getCollectors();
+    Map<ApplicationId, AppCollectorData> collectors = getCollectors(false);
     original.setAppCollectors(collectors);
 
     NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl(
@@ -180,6 +190,16 @@ public class TestYarnServerApiClasses {
   }
 
   @Test
+  public void testNodeHBResponsePBImplWithNullCollectorToken() {
+    NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl();
+    Map<ApplicationId, AppCollectorData> collectors = getCollectors(true);
+    original.setAppCollectors(collectors);
+    NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl(
+        original.getProto());
+    assertEquals(collectors, copy.getAppCollectors());
+  }
+
+  @Test
   public void testNodeHeartbeatResponsePBImplWithDecreasedContainers() {
     NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl();
     original.addAllContainersToUpdate(
@@ -349,11 +369,15 @@ public class TestYarnServerApiClasses {
     return nodeLabels;
   }
 
-  private Map<ApplicationId, AppCollectorData> getCollectors() {
+  private Map<ApplicationId, AppCollectorData> getCollectors(
+      boolean hasNullCollectorToken) {
     ApplicationId appID = ApplicationId.newInstance(1L, 1);
     String collectorAddr = "localhost:0";
-    AppCollectorData data = AppCollectorData.newInstance(appID, collectorAddr,
-        Token.newInstance(new byte[0], "kind", new byte[0], "s"));
+    AppCollectorData data = AppCollectorData.newInstance(appID, collectorAddr);
+    if (!hasNullCollectorToken) {
+      data.setCollectorToken(
+          Token.newInstance(new byte[0], "kind", new byte[0], "s"));
+    }
     Map<ApplicationId, AppCollectorData> collectorMap =
         new HashMap<>();
     collectorMap.put(appID, data);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message