ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From odiache...@apache.org
Subject ambari git commit: AMBARI-14622. Stack Advisor should display warning when PXF component not collocated with NAMENODE and DATANODE(mithmatt via odiachenko).
Date Tue, 12 Jan 2016 22:46:58 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 1696aec0c -> 53aa58158


AMBARI-14622. Stack Advisor should display warning when PXF component not collocated with
NAMENODE and DATANODE(mithmatt via odiachenko).


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/53aa5815
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/53aa5815
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/53aa5815

Branch: refs/heads/trunk
Commit: 53aa581586eb66b78c5d3d6a3748015642b4d9e2
Parents: 1696aec
Author: Oleksandr Diachenko <odiachenko@pivotal.io>
Authored: Tue Jan 12 14:46:52 2016 -0800
Committer: Oleksandr Diachenko <odiachenko@pivotal.io>
Committed: Tue Jan 12 14:46:52 2016 -0800

----------------------------------------------------------------------
 .../stacks/HDP/2.3/services/stack_advisor.py    |  45 ++++++--
 .../stacks/2.3/common/test_stack_advisor.py     | 115 ++++++++++++++++++-
 2 files changed, 147 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/53aa5815/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index b37b95a..ed241ff 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -62,12 +62,12 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
       hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
       hostsCount = len(hostsList)
 
-      hawqMasterHosts = [component["hostnames"] for component in componentsList if component["component_name"]
== "HAWQMASTER"][0]
-      hawqStandbyHosts = [component["hostnames"] for component in componentsList if component["component_name"]
== "HAWQSTANDBY"][0]
-      hawqSegmentHosts = [component["hostnames"] for component in componentsList if component["component_name"]
== "HAWQSEGMENT"][0]
-      datanodeHosts = [component["hostnames"] for component in componentsList if component["component_name"]
== "DATANODE"][0]
+      hawqMasterHosts = self.__getHosts(componentsList, "HAWQMASTER")
+      hawqStandbyHosts = self.__getHosts(componentsList, "HAWQSTANDBY")
+      hawqSegmentHosts = self.__getHosts(componentsList, "HAWQSEGMENT")
+      datanodeHosts = self.__getHosts(componentsList, "DATANODE")
 
-      # Generate WARNING if any HAWQSEGMENT is not colocated with a DATANODE
+      # Generate WARNING if any HAWQSEGMENT is not co-located with a DATANODE
       mismatchHosts = set(hawqSegmentHosts).symmetric_difference(set(datanodeHosts))
       if len(mismatchHosts) > 0:
         hostsString = ', '.join(mismatchHosts)
@@ -91,16 +91,40 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
                   "to a value different from the port number used by Ambari Server database."
         childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
 
+    if "PXF" in servicesList:
+      pxfHosts = self.__getHosts(componentsList, "PXF")
+      namenodeHosts = self.__getHosts(componentsList, "NAMENODE")
+      datanodeHosts = self.__getHosts(componentsList, "DATANODE")
+
+      # Generate WARNING if any PXF is not co-located with a NAMENODE
+      mismatchHosts = sorted(set(namenodeHosts).difference(set(pxfHosts)))
+      if len(mismatchHosts) > 0:
+        hostsString = ', '.join(mismatchHosts)
+        message = "PXF is not co-located with NAMENODE on the following {0} host(s): {1}".format(len(mismatchHosts),
hostsString)
+        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'PXF' } )
+
+      # Generate WARNING if any PXF is not co-located with a DATANODE
+      mismatchHosts = sorted(set(datanodeHosts).difference(set(pxfHosts)))
+      if len(mismatchHosts) > 0:
+        hostsString = ', '.join(mismatchHosts)
+        message = "PXF is not co-located with DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts),
hostsString)
+        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'PXF' } )
+
+      # Generate WARNING if any PXF is not co-located with a NAMENODE or a DATANODE
+      mismatchHosts = sorted(set(pxfHosts).difference(set(namenodeHosts).union(set(datanodeHosts))))
+      if len(mismatchHosts) > 0:
+        hostsString = ', '.join(mismatchHosts)
+        message = "PXF is not co-located with NAMENODE or DATANODE on the following {0} host(s):
{1}".format(len(mismatchHosts), hostsString)
+        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'PXF' } )
+
     if "SPARK" in servicesList:
       if "SPARK_THRIFTSERVER" in servicesList:
         if not "HIVE_SERVER" in servicesList:
           message = "SPARK_THRIFTSERVER requires HIVE services to be selected."
           childItems.append( {"type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'SPARK_THRIFTSERVER'} )
 
-      hmsHosts = [component["hostnames"] for component in componentsList
-                  if component["component_name"] == "HIVE_METASTORE"][0] if "HIVE" in servicesList
else []
-      sparkTsHosts = [component["hostnames"] for component in componentsList
-                      if component["component_name"] == "SPARK_THRIFTSERVER"][0] if "SPARK"
in servicesList else []
+      hmsHosts = self.__getHosts(componentsList, "HIVE_METASTORE") if "HIVE" in servicesList
else []
+      sparkTsHosts = self.__getHosts(componentsList, "SPARK_THRIFTSERVER") if "SPARK" in
servicesList else []
 
       # if Spark Thrift Server is deployed but no Hive Server is deployed
       if len(sparkTsHosts) > 0 and len(hmsHosts) == 0:
@@ -110,6 +134,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     parentItems.extend(childItems)
     return parentItems
 
+  def __getHosts(self, componentsList, componentName):
+    return [component["hostnames"] for component in componentsList if component["component_name"]
== componentName][0]
+
   def getNotPreferableOnServerComponents(self):
     parentComponents = super(HDP23StackAdvisor, self).getNotPreferableOnServerComponents()
     parentComponents.extend(['HAWQMASTER', 'HAWQSTANDBY'])

http://git-wip-us.apache.org/repos/asf/ambari/blob/53aa5815/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 7a8d8ce..7e59a6f 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -182,6 +182,10 @@ class TestHDP23StackAdvisor(TestCase):
       return 'c6401.ambari.apache.org' if value is None else value
 
 
+  def __getHosts(self, componentsList, componentName):
+    return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"]
== componentName][0]
+
+
   def test_validations_hawqsegment(self):
     """ Test validation warning for HAWQ segment not colocated with DATANODE """
 
@@ -190,11 +194,9 @@ class TestHDP23StackAdvisor(TestCase):
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item for sublist in componentsListList for item in sublist]
 
-    hawqsegmentComponent = [component["StackServiceComponents"] for component in componentsList
-                            if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
+    hawqsegmentComponent = self.__getHosts(componentsList, "HAWQSEGMENT")
     hawqsegmentComponent["hostnames"] = ['c6401.ambari.apache.org']
-    datanodeComponent = [component["StackServiceComponents"] for component in componentsList
-                         if component["StackServiceComponents"]["component_name"] == "DATANODE"][0]
+    datanodeComponent = self.__getHosts(componentsList, "DATANODE")
     datanodeComponent["hostnames"] = ['c6402.ambari.apache.org']
 
     validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
@@ -206,6 +208,111 @@ class TestHDP23StackAdvisor(TestCase):
     self.assertEquals(len(validations), 0)
 
 
+  def test_getComponentLayoutValidations_pxf_not_co_located_with_nn(self):
+    """ Test warning is generated when PXF is not co-located with NAMENODE """
+
+    services = self.load_json("services-hawq-pxf-hdfs.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    nameNodeComponent = self.__getHosts(componentsList, "NAMENODE")
+    dataNodeComponent = self.__getHosts(componentsList, "DATANODE")
+    pxfComponent = self.__getHosts(componentsList, "PXF")
+    nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
+    dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+    pxfComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services,
hosts) if validation["component-name"] == "PXF"]
+    self.assertEquals(len(validations), 1)
+    expected = {
+        "type": 'host-component',
+        "level": 'WARN',
+        "message": 'PXF is not co-located with NAMENODE on the following 1 host(s): c6401.ambari.apache.org',
+        "component-name": 'PXF'
+      }
+    self.assertEquals(validations[0], expected)
+
+
+  def test_getComponentLayoutValidations_pxf_not_co_located_with_dn(self):
+    """ Test warning is generated when PXF is not co-located with NAMENODE or DATANODE """
+
+    services = self.load_json("services-hawq-pxf-hdfs.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    nameNodeComponent = self.__getHosts(componentsList, "NAMENODE")
+    dataNodeComponent = self.__getHosts(componentsList, "DATANODE")
+    pxfComponent = self.__getHosts(componentsList, "PXF")
+    nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
+    dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+    pxfComponent["hostnames"] = ["c6401.ambari.apache.org"]
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services,
hosts) if validation["component-name"] == "PXF"]
+    self.assertEquals(len(validations), 1)
+    expected = {
+        "type": 'host-component',
+        "level": 'WARN',
+        "message": 'PXF is not co-located with DATANODE on the following 2 host(s): c6402.ambari.apache.org,
c6403.ambari.apache.org',
+        "component-name": 'PXF'
+      }
+    self.assertEquals(validations[0], expected)
+
+
+  def test_getComponentLayoutValidations_pxf_not_co_located_with_nn_or_dn(self):
+    """ Test warning is generated when PXF is not co-located with NAMENODE or DATANODE """
+
+    services = self.load_json("services-hawq-pxf-hdfs.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    nameNodeComponent = self.__getHosts(componentsList, "NAMENODE")
+    dataNodeComponent = self.__getHosts(componentsList, "DATANODE")
+    pxfComponent = self.__getHosts(componentsList, "PXF")
+    nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
+    dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org"]
+    pxfComponent["hostnames"] = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services,
hosts) if validation["component-name"] == "PXF"]
+    self.assertEquals(len(validations), 1)
+    expected = {
+        "type": 'host-component',
+        "level": 'WARN',
+        "message": 'PXF is not co-located with NAMENODE or DATANODE on the following 1 host(s):
c6403.ambari.apache.org',
+        "component-name": 'PXF'
+      }
+    self.assertEquals(validations[0], expected)
+
+
+  def test_getComponentLayoutValidations_pxf_co_located_with_nn_and_dn(self):
+    """ Test NO warning is generated when PXF is co-located with NAMENODE and DATANODE """
+
+    services = self.load_json("services-hawq-pxf-hdfs.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    nameNodeComponent = self.__getHosts(componentsList, "NAMENODE")
+    dataNodeComponent = self.__getHosts(componentsList, "DATANODE")
+    pxfComponent = self.__getHosts(componentsList, "PXF")
+    nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
+    dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+    pxfComponent["hostnames"] = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services,
hosts) if validation["component-name"] == "PXF"]
+    self.assertEquals(len(validations), 0)
+
+
   @patch('socket.getfqdn', side_effect=fqdn_mock_result)
   def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock):
     """ Test layout validations for HAWQ components on a 3-node cluster """


Mime
View raw message