Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F39F418B7A for ; Tue, 12 Jan 2016 22:46:58 +0000 (UTC) Received: (qmail 59352 invoked by uid 500); 12 Jan 2016 22:46:58 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 59321 invoked by uid 500); 12 Jan 2016 22:46:58 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 59312 invoked by uid 99); 12 Jan 2016 22:46:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Jan 2016 22:46:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 92C80E2C5E; Tue, 12 Jan 2016 22:46:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: odiachenko@apache.org To: commits@ambari.apache.org Message-Id: <5ebd83c5c3704ed8a5190051406dac75@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-14622. Stack Advisor should display warning when PXF component not collocated with NAMENODE and DATANODE(mithmatt via odiachenko). Date: Tue, 12 Jan 2016 22:46:58 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/trunk 1696aec0c -> 53aa58158 AMBARI-14622. Stack Advisor should display warning when PXF component not collocated with NAMENODE and DATANODE(mithmatt via odiachenko). Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/53aa5815 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/53aa5815 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/53aa5815 Branch: refs/heads/trunk Commit: 53aa581586eb66b78c5d3d6a3748015642b4d9e2 Parents: 1696aec Author: Oleksandr Diachenko Authored: Tue Jan 12 14:46:52 2016 -0800 Committer: Oleksandr Diachenko Committed: Tue Jan 12 14:46:52 2016 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.3/services/stack_advisor.py | 45 ++++++-- .../stacks/2.3/common/test_stack_advisor.py | 115 ++++++++++++++++++- 2 files changed, 147 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/53aa5815/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index b37b95a..ed241ff 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -62,12 +62,12 @@ class HDP23StackAdvisor(HDP22StackAdvisor): hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] hostsCount = len(hostsList) - hawqMasterHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQMASTER"][0] - hawqStandbyHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQSTANDBY"][0] - hawqSegmentHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQSEGMENT"][0] - datanodeHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "DATANODE"][0] + hawqMasterHosts = self.__getHosts(componentsList, "HAWQMASTER") + hawqStandbyHosts = self.__getHosts(componentsList, "HAWQSTANDBY") + hawqSegmentHosts = self.__getHosts(componentsList, "HAWQSEGMENT") + datanodeHosts = self.__getHosts(componentsList, "DATANODE") - # Generate WARNING if any HAWQSEGMENT is not colocated with a DATANODE + # Generate WARNING if any HAWQSEGMENT is not co-located with a DATANODE mismatchHosts = set(hawqSegmentHosts).symmetric_difference(set(datanodeHosts)) if len(mismatchHosts) > 0: hostsString = ', '.join(mismatchHosts) @@ -91,16 +91,40 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "to a value different from the port number used by Ambari Server database." childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } ) + if "PXF" in servicesList: + pxfHosts = self.__getHosts(componentsList, "PXF") + namenodeHosts = self.__getHosts(componentsList, "NAMENODE") + datanodeHosts = self.__getHosts(componentsList, "DATANODE") + + # Generate WARNING if any PXF is not co-located with a NAMENODE + mismatchHosts = sorted(set(namenodeHosts).difference(set(pxfHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "PXF is not co-located with NAMENODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) + + # Generate WARNING if any PXF is not co-located with a DATANODE + mismatchHosts = sorted(set(datanodeHosts).difference(set(pxfHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "PXF is not co-located with DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) + + # Generate WARNING if any PXF is not co-located with a NAMENODE or a DATANODE + mismatchHosts = sorted(set(pxfHosts).difference(set(namenodeHosts).union(set(datanodeHosts)))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "PXF is not co-located with NAMENODE or DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) + if "SPARK" in servicesList: if "SPARK_THRIFTSERVER" in servicesList: if not "HIVE_SERVER" in servicesList: message = "SPARK_THRIFTSERVER requires HIVE services to be selected." childItems.append( {"type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER'} ) - hmsHosts = [component["hostnames"] for component in componentsList - if component["component_name"] == "HIVE_METASTORE"][0] if "HIVE" in servicesList else [] - sparkTsHosts = [component["hostnames"] for component in componentsList - if component["component_name"] == "SPARK_THRIFTSERVER"][0] if "SPARK" in servicesList else [] + hmsHosts = self.__getHosts(componentsList, "HIVE_METASTORE") if "HIVE" in servicesList else [] + sparkTsHosts = self.__getHosts(componentsList, "SPARK_THRIFTSERVER") if "SPARK" in servicesList else [] # if Spark Thrift Server is deployed but no Hive Server is deployed if len(sparkTsHosts) > 0 and len(hmsHosts) == 0: @@ -110,6 +134,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor): parentItems.extend(childItems) return parentItems + def __getHosts(self, componentsList, componentName): + return [component["hostnames"] for component in componentsList if component["component_name"] == componentName][0] + def getNotPreferableOnServerComponents(self): parentComponents = super(HDP23StackAdvisor, self).getNotPreferableOnServerComponents() parentComponents.extend(['HAWQMASTER', 'HAWQSTANDBY']) http://git-wip-us.apache.org/repos/asf/ambari/blob/53aa5815/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 7a8d8ce..7e59a6f 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -182,6 +182,10 @@ class TestHDP23StackAdvisor(TestCase): return 'c6401.ambari.apache.org' if value is None else value + def __getHosts(self, componentsList, componentName): + return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == componentName][0] + + def test_validations_hawqsegment(self): """ Test validation warning for HAWQ segment not colocated with DATANODE """ @@ -190,11 +194,9 @@ class TestHDP23StackAdvisor(TestCase): componentsListList = [service["components"] for service in services["services"]] componentsList = [item for sublist in componentsListList for item in sublist] - hawqsegmentComponent = [component["StackServiceComponents"] for component in componentsList - if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0] + hawqsegmentComponent = self.__getHosts(componentsList, "HAWQSEGMENT") hawqsegmentComponent["hostnames"] = ['c6401.ambari.apache.org'] - datanodeComponent = [component["StackServiceComponents"] for component in componentsList - if component["StackServiceComponents"]["component_name"] == "DATANODE"][0] + datanodeComponent = self.__getHosts(componentsList, "DATANODE") datanodeComponent["hostnames"] = ['c6402.ambari.apache.org'] validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) @@ -206,6 +208,111 @@ class TestHDP23StackAdvisor(TestCase): self.assertEquals(len(validations), 0) + def test_getComponentLayoutValidations_pxf_not_co_located_with_nn(self): + """ Test warning is generated when PXF is not co-located with NAMENODE """ + + services = self.load_json("services-hawq-pxf-hdfs.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeComponent = self.__getHosts(componentsList, "NAMENODE") + dataNodeComponent = self.__getHosts(componentsList, "DATANODE") + pxfComponent = self.__getHosts(componentsList, "PXF") + nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"] + dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + pxfComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] + self.assertEquals(len(validations), 1) + expected = { + "type": 'host-component', + "level": 'WARN', + "message": 'PXF is not co-located with NAMENODE on the following 1 host(s): c6401.ambari.apache.org', + "component-name": 'PXF' + } + self.assertEquals(validations[0], expected) + + + def test_getComponentLayoutValidations_pxf_not_co_located_with_dn(self): + """ Test warning is generated when PXF is not co-located with NAMENODE or DATANODE """ + + services = self.load_json("services-hawq-pxf-hdfs.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeComponent = self.__getHosts(componentsList, "NAMENODE") + dataNodeComponent = self.__getHosts(componentsList, "DATANODE") + pxfComponent = self.__getHosts(componentsList, "PXF") + nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"] + dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + pxfComponent["hostnames"] = ["c6401.ambari.apache.org"] + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] + self.assertEquals(len(validations), 1) + expected = { + "type": 'host-component', + "level": 'WARN', + "message": 'PXF is not co-located with DATANODE on the following 2 host(s): c6402.ambari.apache.org, c6403.ambari.apache.org', + "component-name": 'PXF' + } + self.assertEquals(validations[0], expected) + + + def test_getComponentLayoutValidations_pxf_not_co_located_with_nn_or_dn(self): + """ Test warning is generated when PXF is not co-located with NAMENODE or DATANODE """ + + services = self.load_json("services-hawq-pxf-hdfs.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeComponent = self.__getHosts(componentsList, "NAMENODE") + dataNodeComponent = self.__getHosts(componentsList, "DATANODE") + pxfComponent = self.__getHosts(componentsList, "PXF") + nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"] + dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org"] + pxfComponent["hostnames"] = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"] + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] + self.assertEquals(len(validations), 1) + expected = { + "type": 'host-component', + "level": 'WARN', + "message": 'PXF is not co-located with NAMENODE or DATANODE on the following 1 host(s): c6403.ambari.apache.org', + "component-name": 'PXF' + } + self.assertEquals(validations[0], expected) + + + def test_getComponentLayoutValidations_pxf_co_located_with_nn_and_dn(self): + """ Test NO warning is generated when PXF is co-located with NAMENODE and DATANODE """ + + services = self.load_json("services-hawq-pxf-hdfs.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeComponent = self.__getHosts(componentsList, "NAMENODE") + dataNodeComponent = self.__getHosts(componentsList, "DATANODE") + pxfComponent = self.__getHosts(componentsList, "PXF") + nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"] + dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + pxfComponent["hostnames"] = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"] + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] + self.assertEquals(len(validations), 0) + + @patch('socket.getfqdn', side_effect=fqdn_mock_result) def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock): """ Test layout validations for HAWQ components on a 3-node cluster """