ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ja...@apache.org
Subject ambari git commit: AMBARI-14911: Populate hawq_site properties when YARN is installed after HAWQ. (adenissov via jaoki)
Date Fri, 05 Feb 2016 23:48:07 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk f57f9b291 -> 31309e28b


AMBARI-14911: Populate hawq_site properties when YARN is installed after HAWQ. (adenissov
via jaoki)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/31309e28
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/31309e28
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/31309e28

Branch: refs/heads/trunk
Commit: 31309e28b470f45f4a9c44ad7e0393dd31eb2dd6
Parents: f57f9b2
Author: Jun Aoki <jaoki@apache.org>
Authored: Fri Feb 5 15:47:56 2016 -0800
Committer: Jun Aoki <jaoki@apache.org>
Committed: Fri Feb 5 15:47:56 2016 -0800

----------------------------------------------------------------------
 .../HAWQ/2.0.0/configuration/hawq-site.xml      | 12 +++++
 .../stacks/HDP/2.3/services/stack_advisor.py    | 37 +++++++++++---
 .../stacks/2.3/common/test_stack_advisor.py     | 51 +++++++++++++++++++-
 3 files changed, 93 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/31309e28/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
index f034749..f195215 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
@@ -153,6 +153,12 @@
     <description>
       The address of YARN resource manager server.
     </description>
+    <depends-on>
+      <property>
+        <type>yarn-site</type>
+        <name>yarn.resourcemanager.address</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>
@@ -161,6 +167,12 @@
     <description>
       The address of YARN scheduler server.
     </description>
+    <depends-on>
+      <property>
+        <type>yarn-site</type>
+        <name>yarn.resourcemanager.scheduler.address</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/31309e28/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index dae3e8a..b354378 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -678,19 +678,32 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
 
 
   def recommendHAWQConfigurations(self, configurations, clusterData, services, hosts):
+    if "hawq-site" not in services["configurations"]:
+      return
+    hawq_site = services["configurations"]["hawq-site"]["properties"]
     putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services)
-    if self.isHawqMasterComponentOnAmbariServer(services):
-      if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]:
-        putHawqSiteProperty('hawq_master_address_port', '')
-    # calculate optimal number of virtual segments
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item["StackServiceComponents"] for sublist in componentsListList for
item in sublist]
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+    # remove master port when master is colocated with Ambari server
+    if self.isHawqMasterComponentOnAmbariServer(services) and "hawq_master_address_port"
in hawq_site:
+        putHawqSiteProperty('hawq_master_address_port', '')
+
+    # calculate optimal number of virtual segments
     numSegments = len(self.__getHosts(componentsList, "HAWQSEGMENT"))
     # update default if segments are deployed
-    if numSegments and "hawq-site" in services["configurations"] and "default_segment_num"
in services["configurations"]["hawq-site"]["properties"]:
+    if numSegments and "default_segment_num" in hawq_site:
       factor = 6 if numSegments < 50 else 4
       putHawqSiteProperty('default_segment_num', numSegments * factor)
-          
+
+    # update YARN RM urls with the values from yarn-site if YARN is installed
+    if "YARN" in servicesList and "yarn-site" in services["configurations"]:
+      yarn_site = services["configurations"]["yarn-site"]["properties"]
+      for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items():
+        if hs_prop in hawq_site and ys_prop in yarn_site:
+          putHawqSiteProperty(hs_prop, yarn_site[ys_prop])
+
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
     childValidators = {
@@ -946,8 +959,20 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     display_name = 'HAWQ Segment temp directory'
     self.validateIfRootDir (properties, validationItems, prop_name, display_name)
 
+    # 3. Check YARN RM address properties
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    if "YARN" in servicesList and "yarn-site" in configurations:
+      yarn_site = getSiteProperties(configurations, "yarn-site")
+      for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items():
+        if hs_prop in hawq_site and ys_prop in yarn_site and hawq_site[hs_prop] != yarn_site[ys_prop]:
+          message = "Expected value: {0} (this property should have the same value as the
property {1} in yarn-site)".format(yarn_site[ys_prop], ys_prop)
+          validationItems.append({"config-name": hs_prop, "item": self.getWarnItem(message)})
+
     return self.toConfigurationValidationProblems(validationItems, "hawq-site")
   
   
   def isComponentUsingCardinalityForLayout(self, componentName):
     return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']
+
+  def getHAWQYARNPropertyMapping(self):
+    return { "hawq_rm_yarn_address": "yarn.resourcemanager.address", "hawq_rm_yarn_scheduler_address":
"yarn.resourcemanager.scheduler.address" }

http://git-wip-us.apache.org/repos/asf/ambari/blob/31309e28/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 965ce98..72e7718 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1323,8 +1323,14 @@ class TestHDP23StackAdvisor(TestCase):
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item for sublist in componentsListList for item in sublist]
     hawqSegmentComponent = [component["StackServiceComponents"] for component in componentsList
if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
-    services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24"}}
 
+    # setup default configuration values
+    services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24",
+                                                              "hawq_rm_yarn_address": "localhost:8032",
+                                                              "hawq_rm_yarn_scheduler_address":
"localhost:8030"}}
+    services["configurations"]["yarn-site"] = {"properties": {"yarn.resourcemanager.address":
"host1:8050",
+                                                              "yarn.resourcemanager.scheduler.address":
"host1:8030"}}
+    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
     configurations = {}
     clusterData = {}
 
@@ -1333,6 +1339,10 @@ class TestHDP23StackAdvisor(TestCase):
     self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
     self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(3
* 6))
 
+    # check derived properties
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_address"],
"host1:8050")
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_scheduler_address"],
"host1:8030")
+
     # Test 2 - with 49 segments
     hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(49)]
     self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
@@ -1345,6 +1355,7 @@ class TestHDP23StackAdvisor(TestCase):
 
     # Test 4 - with no segments
     configurations = {}
+    services["configurations"]["hawq-site"] = {"properties":{'hawq-site': {'properties':
{}}}}
     hawqSegmentComponent["hostnames"] = []
     self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
     self.assertEquals(configurations, {'hawq-site': {'properties': {}}})
@@ -1616,3 +1627,41 @@ class TestHDP23StackAdvisor(TestCase):
     services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "false"
     problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
     self.assertEqual(len(problems), 0)
+
+  def test_validateHAWQConfigurations(self):
+    services = self.load_json("services-hawq-3-hosts.json")
+    # setup default configuration values
+    configurations = services["configurations"]
+    configurations["hawq-site"] = {"properties": {"hawq_rm_yarn_address": "localhost:8032",
+                                                  "hawq_rm_yarn_scheduler_address": "localhost:8030"}}
+    configurations["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050",
+                                                  "yarn.resourcemanager.scheduler.address":
"host1:8030"}}
+    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
+    properties = configurations["hawq-site"]["properties"]
+    defaults = {}
+    hosts = {}
+
+    expected_warnings = {
+      'hawq_rm_yarn_address': {
+        'config-type': 'hawq-site',
+        'message': 'Expected value: host1:8050 (this property should have the same value
as the property yarn.resourcemanager.address in yarn-site)',
+        'type': 'configuration',
+        'config-name': 'hawq_rm_yarn_address',
+        'level': 'WARN'
+      },
+      'hawq_rm_yarn_scheduler_address': {
+        'config-type': 'hawq-site',
+        'message': 'Expected value: host1:8030 (this property should have the same value
as the property yarn.resourcemanager.scheduler.address in yarn-site)',
+        'type': 'configuration',
+        'config-name': 'hawq_rm_yarn_scheduler_address',
+        'level': 'WARN'
+      }
+    }
+
+    problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, configurations,
services, hosts)
+    problems_dict = {}
+    for problem in problems:
+      problems_dict[problem['config-name']] = problem
+    self.assertEqual(len(problems), 2)
+    self.assertEqual(problems_dict, expected_warnings)
+


Mime
View raw message