hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rohithsharm...@apache.org
Subject [hadoop] 04/06: YARN-6149. Allow port range to be specified while starting NM Timeline collector manager. Contributed by Abhishek Modi.
Date Thu, 05 Sep 2019 07:19:19 GMT
This is an automated email from the ASF dual-hosted git repository.

rohithsharmaks pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 5345508fa33782bcfeb0cbb378ec210fa2a45708
Author: Rohith Sharma K S <rohithsharmaks@apache.org>
AuthorDate: Fri Jan 4 14:07:02 2019 +0530

    YARN-6149. Allow port range to be specified while starting NM Timeline collector manager.
Contributed by Abhishek Modi.
---
 .../org/apache/hadoop/yarn/conf/YarnConfiguration.java    |  4 ++++
 .../collector/NodeTimelineCollectorManager.java           | 15 +++++++++++++--
 .../collector/TestNMTimelineCollectorManager.java         |  6 ++++--
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 04a7003..20b96c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3809,6 +3809,10 @@ public class YarnConfiguration extends Configuration {
       TIMELINE_SERVICE_COLLECTOR_PREFIX + "bind-host";
 
   @Private
+  public static final String TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES =
+      TIMELINE_SERVICE_COLLECTOR_PREFIX + "bind-port-ranges";
+
+  @Private
   public static final String TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS =
       TIMELINE_SERVICE_COLLECTOR_PREFIX + "webapp.address";
   public static final String DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
index 696f4a3..0371d49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
@@ -58,6 +58,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+
 /**
  * Class on the NodeManager side that manages adding and removing collectors and
  * their lifecycle. Also instantiates the per-node collector webapp.
@@ -280,14 +281,21 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager
{
     String bindAddress = null;
     String host =
         conf.getTrimmed(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_HOST);
+    Configuration.IntegerRanges portRanges = conf.getRange(
+        YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES, "");
+    int startPort = 0;
+    if (portRanges != null && !portRanges.isEmpty()) {
+      startPort = portRanges.getRangeStart();
+    }
     if (host == null || host.isEmpty()) {
       // if collector bind-host is not set, fall back to
       // timeline-service.bind-host to maintain compatibility
       bindAddress =
           conf.get(YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST,
-              YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
+              YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST)
+              + ":" + startPort;
     } else {
-      bindAddress = host + ":0";
+      bindAddress = host + ":" + startPort;
     }
 
     try {
@@ -297,6 +305,9 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager
{
           .addEndpoint(URI.create(
               (YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
                   bindAddress));
+      if (portRanges != null && !portRanges.isEmpty()) {
+        builder.setPortRanges(portRanges);
+      }
       if (YarnConfiguration.useHttps(conf)) {
         builder = WebAppUtils.loadSslConfiguration(builder, conf);
       }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
index af9acce..0f2af21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
@@ -60,6 +60,8 @@ public class TestNMTimelineCollectorManager {
     Configuration conf = new YarnConfiguration();
     conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
         FileSystemTimelineWriterImpl.class, TimelineWriter.class);
+    conf.set(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES,
+        "30000-30100");
     collectorManager.init(conf);
     collectorManager.start();
   }
@@ -83,7 +85,8 @@ public class TestNMTimelineCollectorManager {
     String[] parts = address.split(":");
     assertEquals(2, parts.length);
     assertNotNull(parts[0]);
-    assertTrue(Integer.valueOf(parts[1]) > 0);
+    assertTrue(Integer.valueOf(parts[1]) >= 30000 &&
+        Integer.valueOf(parts[1]) <= 30100);
   }
 
   @Test(timeout=60000)
@@ -153,7 +156,6 @@ public class TestNMTimelineCollectorManager {
   private NodeTimelineCollectorManager createCollectorManager() {
     final NodeTimelineCollectorManager cm =
         spy(new NodeTimelineCollectorManager());
-    doReturn(new Configuration()).when(cm).getConfig();
     CollectorNodemanagerProtocol nmCollectorService =
         mock(CollectorNodemanagerProtocol.class);
     GetTimelineCollectorContextResponse response =


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message