ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-20469. Add a new property to yarn-site.xml yarn.log.server.web-service.url (Madhuvanthi Radhakrishnan via smohanty)
Date Thu, 16 Mar 2017 02:26:08 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 8268b842e -> 9e30137c6


AMBARI-20469. Add a new property to yarn-site.xml yarn.log.server.web-service.url (Madhuvanthi
Radhakrishnan via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9e30137c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9e30137c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9e30137c

Branch: refs/heads/branch-2.5
Commit: 9e30137c6840c13fac0798313a7f8536243d5abd
Parents: 8268b84
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Wed Mar 15 17:42:13 2017 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Wed Mar 15 17:42:13 2017 -0700

----------------------------------------------------------------------
 .../BlueprintConfigurationProcessor.java        |  1 +
 .../services/YARN/configuration/yarn-site.xml   | 20 +++++++++
 .../stacks/HDP/2.6/services/stack_advisor.py    | 44 ++++++++++++++++----
 3 files changed, 58 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9e30137c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index e604598..9faa2ce 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -2489,6 +2489,7 @@ public class BlueprintConfigurationProcessor {
     yarnSiteMap.put("yarn.timeline-service.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER"));
     yarnSiteMap.put("yarn.timeline-service.webapp.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER"));
     yarnSiteMap.put("yarn.timeline-service.webapp.https.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER"));
+    yarnSiteMap.put("yarn.log.server.web-service.url", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER"));
 
     // HIVE_SERVER
     hiveSiteMap.put("hive.server2.authentication.ldap.url", new SingleHostTopologyUpdater("HIVE_SERVER2"));

http://git-wip-us.apache.org/repos/asf/ambari/blob/9e30137c/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
index 4b97148..0de6f00 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
@@ -63,4 +63,24 @@
     </description>
     <on-ambari-upgrade add="false"/>
   </property>
+  <property>
+    <name>yarn.log.server.web-service.url</name>
+    <value>http://localhost:8188/ws/v1/applicationhistory</value>
+    <description>Log Server Web Service URL.</description>
+    <depends-on>
+      <property>
+        <type>yarn-site</type>
+        <name>yarn.http.policy</name>
+      </property>
+      <property>
+        <type>yarn-site</type>
+        <name>yarn.resourcemanager.webapp.address</name>
+      </property>
+      <property>
+        <type>yarn-site</type>
+        <name>yarn.resourcemanager.webapp.https.address</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="false"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9e30137c/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 0968d8b..53ff007 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -167,6 +167,19 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
     else:
       ranger_yarn_plugin_enabled = False
 
+    #yarn timeline service url depends on http policy and takes the host name of the yarn
webapp.
+    if "yarn-site" in services["configurations"] and \
+                    "yarn.timeline-service.webapp.https.address" in services["configurations"]["yarn-site"]["properties"]
and \
+                    "yarn.http.policy" in services["configurations"]["yarn-site"]["properties"]
and \
+                    "yarn.log.server.web-service.url" in services["configurations"]["yarn-site"]["properties"]:
+        if services["configurations"]["yarn-site"]["properties"]["yarn.http.policy"] == 'HTTP_ONLY':
+            webapp_address = services["configurations"]["yarn-site"]["properties"]["yarn.timeline-service.webapp.address"]
+            webservice_url = "http://"+webapp_address+"/ws/v1/applicationhistory"
+        else:
+            webapp_address = services["configurations"]["yarn-site"]["properties"]["yarn.timeline-service.webapp.https.address"]
+            webservice_url = "https://"+webapp_address+"/ws/v1/applicationhistory"
+        putYarnSiteProperty('yarn.log.server.web-service.url',webservice_url )
+
     if ranger_yarn_plugin_enabled and 'ranger-yarn-plugin-properties' in services['configurations']
and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-yarn-plugin-properties']['properties']:
       Logger.info("Setting Yarn Repo user for Ranger.")
       putRangerYarnPluginProperty = self.putProperty(configurations, "ranger-yarn-plugin-properties",
services)
@@ -234,7 +247,8 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
           "DRUID": {"druid-env": self.validateDruidEnvConfigurations,
                     "druid-historical": self.validateDruidHistoricalConfigurations,
                     "druid-broker": self.validateDruidBrokerConfigurations},
-          "RANGER": {"ranger-ugsync-site": self.validateRangerUsersyncConfigurations}
+          "RANGER": {"ranger-ugsync-site": self.validateRangerUsersyncConfigurations},
+          "YARN" : {"yarn-site": self.validateYarnSiteConfigurations}
       }
       self.mergeValidators(parentValidators, childValidators)
       return parentValidators
@@ -259,6 +273,22 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
                    })
       return self.toConfigurationValidationProblems(validationItems, "druid-env")
 
+  def validateYarnSiteConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
+      validationItems = []
+      siteProperties = services["configurations"]["yarn-site"]["properties"]
+      if services["configurations"]["yarn-site"]["properties"]["yarn.http.policy"] == 'HTTP_ONLY':
+         webapp_address = services["configurations"]["yarn-site"]["properties"]["yarn.timeline-service.webapp.address"]
+         propertyValue = "http://"+webapp_address+"/ws/v1/applicationhistory"
+      else:
+         webapp_address = services["configurations"]["yarn-site"]["properties"]["yarn.timeline-service.webapp.https.address"]
+         propertyValue = "https://"+webapp_address+"/ws/v1/applicationhistory"
+      Logger.info("validateYarnSiteConfigurations: recommended value for webservice url"+services["configurations"]["yarn-site"]["properties"]["yarn.log.server.web-service.url"])
+      if services["configurations"]["yarn-site"]["properties"]["yarn.log.server.web-service.url"]
!= propertyValue:
+         validationItems = [
+              {"config-name": "yarn.log.server.web-service.url",
+               "item": self.getWarnItem("Value should be %s" % propertyValue)}]
+      return self.toConfigurationValidationProblems(validationItems, "yarn-site")
+
   def validateDruidHistoricalConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
       validationItems = [
           {"config-name": "druid.processing.numThreads",
@@ -268,12 +298,12 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
       return self.toConfigurationValidationProblems(validationItems, "druid-historical")
 
   def validateDruidBrokerConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
-      validationItems = [
-          {"config-name": "druid.processing.numThreads",
-           "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
-                                                         "druid.processing.numThreads")}
-      ]
-      return self.toConfigurationValidationProblems(validationItems, "druid-broker")
+        validationItems = [
+            {"config-name": "druid.processing.numThreads",
+             "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
+                                                           "druid.processing.numThreads")}
+        ]
+        return self.toConfigurationValidationProblems(validationItems, "druid-broker")
 
   def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP26StackAdvisor, self).recommendTezConfigurations(configurations, clusterData,
services, hosts)


Mime
View raw message