ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mithm...@apache.org
Subject [16/17] ambari git commit: AMBARI-16647: Move service advisor tests for HAWQ and PXF (Lav Jain via mithmatt)
Date Wed, 06 Jul 2016 06:42:04 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py
new file mode 100644
index 0000000..780370d
--- /dev/null
+++ b/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py
@@ -0,0 +1,1051 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os, imp, json
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+
+
+class TestHAWQ200ServiceAdvisor(TestCase):
+
+  testDirectory = os.path.dirname(os.path.abspath(__file__))
+  serviceAdvisorPath = '../../../../main/resources/common-services/HAWQ/2.0.0/service_advisor.py'
+  hawq200ServiceAdvisorPath = os.path.join(testDirectory, serviceAdvisorPath)
+  with open(hawq200ServiceAdvisorPath, 'rb') as fp:
+    service_advisor_impl = imp.load_module('service_advisor_impl', fp, hawq200ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+
+  def setUp(self):
+    serviceAdvisorClass = getattr(self.service_advisor_impl, 'HAWQ200ServiceAdvisor')
+    self.serviceAdvisor = serviceAdvisorClass()
+
+  def fqdn_mock_result(value=None):
+      return 'c6401.ambari.apache.org' if value is None else value
+
+  def load_json(self, filename):
+    file = os.path.join(self.testDirectory, "../configs", filename)
+    with open(file, 'rb') as f:
+      data = json.load(f)
+    return data
+
+  def prepareHosts(self, hostsNames):
+    hosts = { "items": [] }
+    for hostName in hostsNames:
+      nextHost = {"Hosts":{"host_name" : hostName}}
+      hosts["items"].append(nextHost)
+    return hosts
+
+  def getHosts(self, componentsList, componentName):
+    hosts = [component["StackServiceComponents"] for component in componentsList
+             if component["StackServiceComponents"]["component_name"] == componentName]
+    return hosts[0] if len(hosts) > 0 else []
+
+  def getHostsFromRecommendations(self, recommendations, componentName):
+    hostGroups = [hostgroup["name"] for hostgroup in recommendations["blueprint"]["host_groups"] if
+                  {"name": componentName} in hostgroup["components"]]
+    return set([host["fqdn"] for hostgroup in recommendations["blueprint_cluster_binding"]["host_groups"] if
+                hostgroup["name"] in hostGroups for host in hostgroup["hosts"]])
+
+  def getComponentsListFromServices(self, services):
+    componentsListList = [service["components"] for service in services["services"]]
+    return [item for sublist in componentsListList for item in sublist]
+
+  def getComponentsFromServices(self, services):
+    componentsList = self.getComponentsListFromServices(services)
+    return [component["StackServiceComponents"]["component_name"] for component in componentsList]
+
+  def getComponentsFromRecommendations(self, recommendations):
+    componentsListList = [hostgroup["components"] for hostgroup in
+                          recommendations["recommendations"]["blueprint"]["host_groups"]]
+    return [item["name"] for sublist in componentsListList for item in sublist]
+
+  def insertHAWQServiceAdvisorInfo(self, services):
+    for service in services["services"]:
+      if service["StackServices"]["service_name"] == 'HAWQ':
+        service["StackServices"]["advisor_name"] = "HAWQ200ServiceAdvisor"
+        service["StackServices"]["advisor_path"] = self.hawq200ServiceAdvisorPath
+
+  @patch("socket.getfqdn")
+  def test_getHostsForMasterComponent(self, getfqdn_mock):
+    getfqdn_mock.return_value = "c6401.ambari.apache.org"
+
+    services = {
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "HAWQ"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "HAWQMASTER",
+                "hostnames": [
+                  "c6403.ambari.apache.org"
+                ]
+              }
+            },
+            {
+              "StackServiceComponents": {
+                "component_name": "HAWQSTANDBY",
+                "hostnames": [
+                ]
+              }
+            }
+          ]
+        }
+      ]
+    }
+
+    hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org", "c6404.ambari.apache.org"]
+
+    component = {
+      "StackServiceComponents": {
+        "component_name": "HAWQSTANDBY"
+      }
+    }
+
+    # Case 1:
+    # Ambari Server is placed on c6401.ambari.apache.org
+    # HAWQMASTER is placed on c6403.ambari.apache.org
+    # There are 4 available hosts in the cluster
+    # Recommend HAWQSTANDBY on next available host, c6402.ambari.apache.org
+    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
+    self.assertEquals(standbyHosts, ["c6402.ambari.apache.org"])
+
+    # Case 2:
+    # Ambari Server is placed on c6401.ambari.apache.org
+    # HAWQMASTER is placed on c6402.ambari.apache.org
+    # There are 4 available hosts in the cluster
+    # Recommend HAWQSTANDBY on next available host, c6403.ambari.apache.org
+    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org"]
+    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
+    self.assertEquals(standbyHosts, ["c6403.ambari.apache.org"])
+
+    # Case 3:
+    # Ambari Server is placed on c6401.ambari.apache.org
+    # HAWQMASTER is placed on c6402.ambari.apache.org
+    # There are 2 available hosts in the cluster
+    # Recommend HAWQSTANDBY on a host which does not have HAWQMASTER, c6401.ambari.apache.org
+    hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
+    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
+    self.assertEquals(standbyHosts, ["c6401.ambari.apache.org"])
+
+    # Case 4:
+    # Ambari Server is placed on c6401.ambari.apache.org
+    # HAWQMASTER is placed on c6401.ambari.apache.org
+    # There is 1 available host in the cluster
+    # Do not recommend HAWQSTANDBY on a single node cluster
+    hostsList = ["c6401.ambari.apache.org"]
+    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6401.ambari.apache.org"]
+    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
+    self.assertEquals(standbyHosts, [])
+
+    # Case 5:
+    # Ambari Server is placed on c6401.ambari.apache.org
+    # HAWQMASTER is placed on c6402.ambari.apache.org
+    # HAWQSTANDBY is placed on c6401.ambari.apache.org
+    # There are 3 available host in the cluster
+    # Do not change HAWQSTANDBY host according to recommendation since HAWQSTANDBY has already been assigned a host
+    hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org"]
+    services["services"][0]["components"][1]["StackServiceComponents"]["hostnames"] = ["c6401.ambari.apache.org"]
+    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
+    self.assertEquals(standbyHosts, ["c6401.ambari.apache.org"])
+
+
+  def test_getServiceConfigurationRecommendations(self):
+
+    configurations = {
+      "hawq-sysctl-env": {
+        "properties": {
+          "vm.overcommit_memory": 1,
+          "vm.overcommit_ratio": 50
+        }
+      },
+      "hawq-site": {
+        "properties": {
+          "hawq_rm_memory_limit_perseg": "65535MB",
+          "hawq_rm_nvcore_limit_perseg": "16",
+          "hawq_global_rm_type": "yarn"
+        }
+      }
+    }
+
+    services = {
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "HAWQ",
+            "service_version": "2.0",
+            "stack_name": "HDP",
+            "stack_version": "2.3"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "HAWQMASTER",
+                "hostnames": [
+                  "c6401.ambari.apache.org"
+                ]
+              }
+            },
+            {
+              "StackServiceComponents": {
+                "component_name": "HAWQSEGMENT",
+                "hostnames": [
+                  "c6402.ambari.apache.org",
+                  "c6404.ambari.apache.org",
+                ]
+              }
+            }
+          ]
+        }
+      ],
+      "configurations": configurations
+    }
+
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "host_name": "c6401.ambari.apache.org",
+            "cpu_count" : 2,
+            "total_mem": 33554432
+          }
+        },
+        {
+          "Hosts": {
+            "host_name": "c6402.ambari.apache.org",
+            "cpu_count" : 4,
+            "total_mem": 33554433
+          }
+        },
+        {
+          "Hosts": {
+            "host_name": "c6403.ambari.apache.org",
+            "cpu_count" : 1,
+            "total_mem": 33554434
+          }
+        },
+        {
+          "Hosts": {
+            "host_name": "c6404.ambari.apache.org",
+            "cpu_count" : 2,
+            "total_mem": 33554435
+          }
+        }
+      ]
+    }
+
+    ## Test if hawq_rm_nvcore_limit_perseg is set correctly
+
+    # Case 1:
+    # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2
+    # hawq_global_rm_type: yarn
+    # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2")
+
+    ## Test if vm.overcommit_memory is set correctly
+
+    # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432)
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2")
+
+    # Case 2: One machine has total_mem below 32GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 33554431
+    services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB"
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1")
+
+    ## Test if hawq_rm_memory_limit_perseg is set correctly
+
+    # Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB as vm.overcommit_memory = 1 and in MB
+    hosts["items"][0]["Hosts"]["total_mem"] = 2097152
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "1536MB")
+
+    # Case 2: Minimum host memory is ~ 16 GB, recommended val must be .75% of 16GB as vm.overcommit_memory = 1 and in GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 16777216
+    hosts["items"][1]["Hosts"]["total_mem"] = 26777216
+    hosts["items"][3]["Hosts"]["total_mem"] = 36777216
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "12GB")
+
+    # Case 2: Minimum host memory is ~ 64 GB, recommended val must be .75% of 32GB as vm.overcommit_memory = 2 and in GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 67108864
+    hosts["items"][1]["Hosts"]["total_mem"] = 77108864
+    hosts["items"][3]["Hosts"]["total_mem"] = 87108864
+    services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB"
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "24GB")
+
+    # Case 4: Minimum host memory is ~ 512 GB, recommended val must be .85% of 256GB as vm.overcommit_memory = 2 and in GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 536870912
+    hosts["items"][1]["Hosts"]["total_mem"] = 636870912
+    hosts["items"][3]["Hosts"]["total_mem"] = 736870912
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "218GB")
+
+    # Case 5: Minimum host memory is ~ 1024 GB, recommended val must be .95% of 512GB as vm.overcommit_memory = 2 and in GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 1073741824
+    hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
+    hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "436GB")
+
+    # Case 6: Minimum host memory is ~ 1024 GB, vm.overcommit_ratio = 75, vm.overcommit_memory = 2
+    # recommended val must be .95% of (1024*75)/100 and in GB
+    hosts["items"][0]["Hosts"]["total_mem"] = 1073741824
+    hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
+    hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
+    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 75
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB")
+
+    ## Test if the properties are set to visible / invisible based on the value of hawq_global_rm_type
+
+    # Case 1: When hawq_global_rm_type is yarn
+    services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "yarn"
+    properties_visible_status = {"hawq_rm_memory_limit_perseg": "false",
+                  "hawq_rm_nvcore_limit_perseg": "false",
+                  "hawq_rm_yarn_app_name":"true",
+                  "hawq_rm_yarn_queue_name": "true",
+                  "hawq_rm_yarn_scheduler_address": "true",
+                  "hawq_rm_yarn_address": "true"}
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    for property, status in properties_visible_status.iteritems():
+      self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status)
+
+    # Case 2: When hawq_global_rm_type is none
+    services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "none"
+    properties_visible_status = {"hawq_rm_memory_limit_perseg": "true",
+                                 "hawq_rm_nvcore_limit_perseg": "true",
+                                 "hawq_rm_yarn_app_name": "false",
+                                 "hawq_rm_yarn_queue_name": "false",
+                                 "hawq_rm_yarn_scheduler_address": "false",
+                                 "hawq_rm_yarn_address": "false"}
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    for property, status in properties_visible_status.iteritems():
+      self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status)
+
+    ## Test if vm.overcommit_ratio is set to visible / invisible based on the value of vm.overcommit_memory
+
+    # Case 1: vm.overcommit_ratio should be invisible when overcommit_memory is set as 0
+    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 0
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "false")
+
+    # Case 2: vm.overcommit_ratio should be invisible when overcommit_memory is set as 1
+    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 1
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "false")
+
+    # Case 3: vm.overcommit_ratio should be visible when overcommit_memory is set as 2
+    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
+    self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "true")
+
+  def test_createComponentLayoutRecommendations_hawq_3_Hosts(self):
+    """ Test that HAWQSTANDBY is recommended on a 3-node cluster """
+
+    services = self.load_json("services-hawq-3-hosts.json")
+    componentNames = self.getComponentsFromServices(services)
+    self.assertTrue('HAWQSTANDBY' in componentNames)
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    self.insertHAWQServiceAdvisorInfo(services)
+    recommendations = self.serviceAdvisor.recommendComponentLayout(services, hosts)
+    recommendedComponents = self.getComponentsFromRecommendations(recommendations)
+    self.assertTrue('HAWQMASTER' in recommendedComponents)
+    self.assertTrue('HAWQSTANDBY' in recommendedComponents)
+    self.assertTrue('HAWQSEGMENT' in recommendedComponents)
+
+    # make sure master components are not collocated
+    componentsListList = [hostgroup["components"] for hostgroup in
+                          recommendations["recommendations"]["blueprint"]["host_groups"]]
+    for sublist in componentsListList:
+      hostComponents = [item["name"] for item in sublist]
+      self.assertFalse({'HAWQMASTER', 'HAWQSTANDBY'}.issubset(hostComponents))
+
+  def test_createComponentLayoutRecommendations_hawq_1_Host(self):
+
+    services = self.load_json("services-hawq-3-hosts.json")
+    componentNames = self.getComponentsFromServices(services)
+    self.assertTrue('HAWQSTANDBY' in componentNames)
+
+    hosts = self.load_json("hosts-1-host.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 1)
+
+    self.insertHAWQServiceAdvisorInfo(services)
+    recommendations = self.serviceAdvisor.recommendComponentLayout(services, hosts)
+    recommendedComponents = self.getComponentsFromRecommendations(recommendations)
+    self.assertTrue('HAWQMASTER' in recommendedComponents)
+    self.assertFalse('HAWQSTANDBY' in recommendedComponents)
+    self.assertTrue('HAWQSEGMENT' in recommendedComponents)
+
+  def test_createComponentLayoutRecommendations_no_hawq_3_Hosts(self):
+    """ Test no failures when there are no HAWQ components """
+
+    services = self.load_json("services-nohawq-3-hosts.json")
+    componentNames = self.getComponentsFromServices(services)
+    self.assertFalse('HAWQMASTER' in componentNames)
+    self.assertFalse('HAWQSTANDBY' in componentNames)
+    self.assertFalse('HAWQSEGMENT' in componentNames)
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    self.insertHAWQServiceAdvisorInfo(services)
+    recommendations = self.serviceAdvisor.recommendComponentLayout(services, hosts)
+    recommendedComponents = self.getComponentsFromRecommendations(recommendations)
+    self.assertFalse('HAWQMASTER' in recommendedComponents)
+    self.assertFalse('HAWQSTANDBY' in recommendedComponents)
+    self.assertFalse('HAWQSEGMENT' in recommendedComponents)
+
+  def test_createComponentLayoutRecommendations_hawqsegment_cluster_install(self):
+    """ Test that HAWQSEGMENT gets recommended correctly during Cluster Install Wizard, when HAWQ is selected for installation """
+
+    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
+    services =  {
+                  "services" : [
+                    {
+                      "StackServices" : {
+                        "service_name" : "HDFS"
+                      },
+                      "components" : [
+                        {
+                          "StackServiceComponents" : {
+                            "cardinality" : "1+",
+                            "component_category" : "SLAVE",
+                            "component_name" : "DATANODE",
+                            "hostnames" : []
+                          }
+                        }
+                      ]
+                    },
+                    {
+                      "StackServices" : {
+                        "service_name" : "HAWQ"
+                      },
+                      "components" : [
+                        {
+                          "StackServiceComponents" : {
+                            "cardinality" : "1+",
+                            "component_category" : "SLAVE",
+                            "component_name" : "HAWQSEGMENT",
+                            "hostnames" : []
+                          }
+                        }
+                      ]
+                    }
+                  ]
+                }
+
+    hawqSegmentHosts = {"c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"}
+    self.insertHAWQServiceAdvisorInfo(services)
+    recommendations = self.serviceAdvisor.createComponentLayoutRecommendations(services, hosts)
+    hostNames = self.getHostsFromRecommendations(recommendations, "HAWQSEGMENT")
+    self.assertEquals(set(hostNames), hawqSegmentHosts)
+
+  def test_createComponentLayoutRecommendations_hawqsegment_add_service_wizard_to_be_installed(self):
+    """ Test that HAWQSEGMENT gets recommended correctly during Add Service Wizard, when HAWQ is selected for installation """
+
+    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
+    services =  {
+                  "services" : [
+                    {
+                      "StackServices" : {
+                        "service_name" : "HDFS"
+                      },
+                      "components" : [
+                        {
+                          "StackServiceComponents" : {
+                            "cardinality" : "1+",
+                            "component_category" : "SLAVE",
+                            "component_name" : "DATANODE",
+                            "hostnames" : ["c6401.ambari.apache.org", "c6403.ambari.apache.org"]
+                          }
+                        }
+                      ]
+                    },
+                    {
+                      "StackServices" : {
+                        "service_name" : "HAWQ"
+                      },
+                      "components" : [
+                        {
+                          "StackServiceComponents" : {
+                            "cardinality" : "1+",
+                            "component_category" : "SLAVE",
+                            "component_name" : "HAWQSEGMENT",
+                            "hostnames" : []
+                          }
+                        }
+                      ]
+                    }
+                  ]
+                }
+
+    hawqSegmentHosts = {"c6401.ambari.apache.org", "c6403.ambari.apache.org"}
+    self.insertHAWQServiceAdvisorInfo(services)
+    recommendations = self.serviceAdvisor.createComponentLayoutRecommendations(services, hosts)
+    hostNames = self.getHostsFromRecommendations(recommendations, "HAWQSEGMENT")
+    self.assertEquals(set(hostNames), hawqSegmentHosts)
+
+  def test_createComponentLayoutRecommendations_hawqsegment_add_service_wizard_already_installed(self):
+    """ Test that HAWQSEGMENT does not get recommended during Add Service Wizard, when HAWQ has already been installed """
+
+    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
+    services =  {
+                  "services" : [
+                    {
+                      "StackServices" : {
+                        "service_name" : "HDFS"
+                      },
+                      "components" : [
+                        {
+                          "StackServiceComponents" : {
+                            "cardinality" : "1+",
+                            "component_category" : "SLAVE",
+                            "component_name" : "DATANODE",
+                            "hostnames" : ["c6401.ambari.apache.org", "c6403.ambari.apache.org"]
+                          }
+                        }
+                      ]
+                    },
+                    {
+                      "StackServices" : {
+                        "service_name" : "HAWQ"
+                      },
+                      "components" : [
+                        {
+                          "StackServiceComponents" : {
+                            "cardinality" : "1+",
+                            "component_category" : "SLAVE",
+                            "component_name" : "HAWQSEGMENT",
+                            "hostnames" : ["c6402.ambari.apache.org"]
+                          }
+                        }
+                      ]
+                    },
+                    {
+                      "StackServices" : {
+                        "service_name" : "PXF"
+                      },
+                      "components" : [
+                        {
+                          "StackServiceComponents" : {
+                            "cardinality" : "1+",
+                            "component_category" : "SLAVE",
+                            "component_name" : "PXF",
+                            "hostnames" : []
+                          }
+                        }
+                      ]
+                    }
+                  ]
+                }
+
+    hawqSegmentHosts = {"c6402.ambari.apache.org"}
+    self.insertHAWQServiceAdvisorInfo(services)
+    recommendations = self.serviceAdvisor.createComponentLayoutRecommendations(services, hosts)
+    hostNames = self.getHostsFromRecommendations(recommendations, "HAWQSEGMENT")
+    self.assertEquals(set(hostNames), hawqSegmentHosts)
+
+  def test_getComponentLayoutValidations_hawqsegment_not_co_located_with_datanode(self):
+    """ Test validation warning for HAWQ segment not colocated with DATANODE """
+
+    services = self.load_json("services-normal-hawq-3-hosts.json")
+    hosts = self.load_json("hosts-3-hosts.json")
+    componentsList = self.getComponentsListFromServices(services)
+
+    hawqsegmentComponent = self.getHosts(componentsList, "HAWQSEGMENT")
+    hawqsegmentComponent["hostnames"] = ['c6401.ambari.apache.org']
+    datanodeComponent = self.getHosts(componentsList, "DATANODE")
+    datanodeComponent["hostnames"] = ['c6402.ambari.apache.org']
+
+    self.insertHAWQServiceAdvisorInfo(services)
+    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
+    expected = {
+      'type': 'host-component',
+      'level': 'WARN',
+      'component-name': 'HAWQSEGMENT',
+      'message': 'HAWQ Segment must be installed on all DataNodes. The following 2 host(s) do not satisfy the colocation recommendation: c6401.ambari.apache.org, c6402.ambari.apache.org',
+    }
+    self.assertEquals(validations[0], expected)
+
+    datanodeComponent["hostnames"] = ['c6401.ambari.apache.org']
+    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 0)
+
+
+  @patch('socket.getfqdn', side_effect=fqdn_mock_result)
+  def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock):
+    """ Test layout validations for HAWQ components on a 3-node cluster """
+
+    # case-1: normal placement, no warnings
+    services = self.load_json("services-normal-hawq-3-hosts.json")
+    componentsList = self.getComponentsListFromServices(services)
+    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")["hostnames"]
+    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")["hostnames"]
+    self.assertEquals(len(hawqMasterHosts), 1)
+    self.assertEquals(len(hawqStandbyHosts), 1)
+    self.assertNotEquals(hawqMasterHosts[0], hawqStandbyHosts[0])
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    self.insertHAWQServiceAdvisorInfo(services)
+    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 0)
+
+    # case-2: HAWQ masters are collocated
+    services = self.load_json("services-master_standby_colo-3-hosts.json")
+    componentsList = self.getComponentsListFromServices(services)
+    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")["hostnames"]
+    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")["hostnames"]
+    self.assertEquals(len(hawqMasterHosts), 1)
+    self.assertEquals(len(hawqStandbyHosts), 1)
+    self.assertEquals(hawqMasterHosts[0], hawqStandbyHosts[0])
+
+    self.insertHAWQServiceAdvisorInfo(services)
+    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 1)
+    expected = {
+      'component-name': 'HAWQSTANDBY',
+      'message': 'HAWQ Master and HAWQ Standby Master cannot be deployed on the same host.',
+      'type': 'host-component',
+      'host': 'c6403.ambari.apache.org',
+      'level': 'ERROR'
+    }
+    self.assertEquals(validations[0], expected)
+
+    # case-3: HAWQ Master and Ambari Server are collocated
+    services = self.load_json("services-master_ambari_colo-3-hosts.json")
+    componentsList = self.getComponentsListFromServices(services)
+    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")["hostnames"]
+    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")["hostnames"]
+    self.assertEquals(len(hawqMasterHosts), 1)
+    self.assertEquals(len(hawqStandbyHosts), 1)
+    self.assertNotEquals(hawqMasterHosts[0], hawqStandbyHosts[0])
+    self.assertEquals(hawqMasterHosts[0], "c6401.ambari.apache.org")
+
+    self.insertHAWQServiceAdvisorInfo(services)
+    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 1)
+    expected = {
+      'component-name': 'HAWQMASTER',
+      'message': 'The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. '  +
+                 'If you are using port 5432 for Postgres, you must either deploy the HAWQ Master on a different host ' +
+                 'or configure a different port for the HAWQ Masters in the HAWQ Configuration page.',
+      'type': 'host-component',
+      'host': 'c6401.ambari.apache.org',
+      'level': 'WARN'
+    }
+    self.assertEquals(validations[0], expected)
+
+    # case-4: HAWQ Standby and Ambari Server are collocated
+    services = self.load_json("services-standby_ambari_colo-3-hosts.json")
+    componentsList = self.getComponentsListFromServices(services)
+    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")["hostnames"]
+    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")["hostnames"]
+    self.assertEquals(len(hawqMasterHosts), 1)
+    self.assertEquals(len(hawqStandbyHosts), 1)
+    self.assertNotEquals(hawqMasterHosts[0], hawqStandbyHosts[0])
+    self.assertEquals(hawqStandbyHosts[0], "c6401.ambari.apache.org")
+
+    self.insertHAWQServiceAdvisorInfo(services)
+    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 1)
+    expected = {
+      'component-name': 'HAWQSTANDBY',
+      'message': 'The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. '  +
+                 'If you are using port 5432 for Postgres, you must either deploy the HAWQ Standby Master on a different host ' +
+                 'or configure a different port for the HAWQ Masters in the HAWQ Configuration page.',
+      'type': 'host-component',
+      'host': 'c6401.ambari.apache.org',
+      'level': 'WARN'
+    }
+    self.assertEquals(validations[0], expected)
+
+
+  @patch('socket.getfqdn', side_effect=fqdn_mock_result)
+  def test_getComponentLayoutValidations_nohawq_3_Hosts(self, socket_mock):
+    """ Test no failures when there are no HAWQ components on a 3-node cluster """
+
+    # normal placement, no warnings
+    services = self.load_json("services-normal-nohawq-3-hosts.json")
+    componentsList = self.getComponentsListFromServices(services)
+    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")
+    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")
+    self.assertEquals(len(hawqMasterHosts), 0)
+    self.assertEquals(len(hawqStandbyHosts), 0)
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    self.insertHAWQServiceAdvisorInfo(services)
+    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 0)
+
+  def test_recommendHAWQConfigurations(self):
+
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "host_name": "c6401.ambari.apache.org",
+            "cpu_count" : 2,
+            "total_mem": 33554432
+          }
+        },
+        {
+          "Hosts": {
+            "host_name": "c6402.ambari.apache.org",
+            "cpu_count" : 4,
+            "total_mem": 33554433
+          }
+        },
+        {
+          "Hosts": {
+            "host_name": "c6403.ambari.apache.org",
+            "cpu_count" : 1,
+            "total_mem": 33554434
+          }
+        },
+        {
+          "Hosts": {
+            "host_name": "c6404.ambari.apache.org",
+            "cpu_count" : 2,
+            "total_mem": 33554435
+          }
+        }
+      ]
+    }
+
+    # original cluster data with 3 segments
+    services = self.load_json("services-normal-hawq-3-hosts.json")
+    componentsList = self.getComponentsListFromServices(services)
+    hawqSegmentComponent = self.getHosts(componentsList, "HAWQSEGMENT")
+
+    # setup default configuration values
+    services["configurations"]["hawq-site"] = {"properties": {"default_hash_table_bucket_number": "24",
+                                                              "hawq_rm_nvseg_perquery_limit": "512",
+                                                              "hawq_rm_yarn_address": "localhost:8032",
+                                                              "hawq_rm_yarn_scheduler_address": "localhost:8030",
+                                                              "hawq_global_rm_type":  "none"}}
+
+    services["configurations"]["hdfs-client"] = {"properties": {"output.replace-datanode-on-failure": "true"}}
+    services["configurations"]["hawq-sysctl-env"] = {"properties": {}}
+
+    services["configurations"]["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050",
+                                                              "yarn.resourcemanager.scheduler.address": "host1:8030"}}
+    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
+    configurations = {}
+    clusterData = {}
+    self.insertHAWQServiceAdvisorInfo(services)
+
+    # Test 1 - with 3 segments
+    self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3)
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(3 * 6))
+    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false")
+
+    # check derived properties
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_address"], "host1:8050")
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_scheduler_address"], "host1:8030")
+
+    # Test 2 - with 100 segments
+    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(100)]
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(100 * 5))
+    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true")
+
+    # Test 3 - with 512 segments
+    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(512)]
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512")
+    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true")
+
+    # Test 4 - with 513 segments
+    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(513)]
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512")
+    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true")
+
+    # Test 5 - with no segments
+    configurations = {}
+    services["configurations"]["hawq-site"] = {"properties":{"hawq_global_rm_type": "none"}}
+    hawqSegmentComponent["hostnames"] = []
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false")
+    self.assertTrue("default_hash_table_bucket_number" not in configurations["hawq-site"])
+
+
+  def test_validateHAWQSiteConfigurations(self):
+    services = self.load_json("services-hawq-3-hosts.json")
+    # setup default configuration values
+    # Test hawq_rm_yarn_address and hawq_rm_scheduler_address are set correctly
+    configurations = services["configurations"]
+    configurations["hawq-site"] = {"properties": {"hawq_rm_yarn_address": "localhost:8032",
+                                                  "hawq_rm_yarn_scheduler_address": "localhost:8030"}}
+    configurations["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050",
+                                                  "yarn.resourcemanager.scheduler.address": "host1:8030"}}
+    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
+    properties = configurations["hawq-site"]["properties"]
+    defaults = {}
+    hosts = {}
+
+    expected_warnings = {
+      'hawq_rm_yarn_address': {
+        'config-type': 'hawq-site',
+        'message': 'Expected value: host1:8050 (this property should have the same value as the property yarn.resourcemanager.address in yarn-site)',
+        'type': 'configuration',
+        'config-name': 'hawq_rm_yarn_address',
+        'level': 'WARN'
+      },
+      'hawq_rm_yarn_scheduler_address': {
+        'config-type': 'hawq-site',
+        'message': 'Expected value: host1:8030 (this property should have the same value as the property yarn.resourcemanager.scheduler.address in yarn-site)',
+        'type': 'configuration',
+        'config-name': 'hawq_rm_yarn_scheduler_address',
+        'level': 'WARN'
+      }
+    }
+
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
+    problems_dict = {}
+    for problem in problems:
+      problems_dict[problem['config-name']] = problem
+    self.assertEqual(len(problems), 2)
+    self.assertEqual(problems_dict, expected_warnings)
+
+    # Test hawq_master_directory multiple directories validation
+    configurations["hawq-site"] = {"properties": {"hawq_master_directory": "/data/hawq/master",
+                                                  "hawq_segment_directory": "/data/hawq/segment"}}
+    properties = configurations["hawq-site"]["properties"]
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
+    problems_dict = {}
+    self.assertEqual(len(problems), 0)
+    expected_warnings = {}
+    self.assertEqual(problems_dict, expected_warnings)
+
+    configurations["hawq-site"] = {"properties": {"hawq_master_directory": "/data/hawq/master1,/data/hawq/master2",
+                                                  "hawq_segment_directory": "/data/hawq/segment1 /data/hawq/segment2"}}
+    properties = configurations["hawq-site"]["properties"]
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
+    problems_dict = {}
+    for problem in problems:
+      problems_dict[problem['config-name']] = problem
+    self.assertEqual(len(problems), 2)
+    expected_warnings = {
+      'hawq_master_directory': {
+        'config-type': 'hawq-site',
+        'message': 'Multiple directories for HAWQ Master directory are not allowed.',
+        'type': 'configuration',
+        'config-name': 'hawq_master_directory',
+        'level': 'ERROR'
+      },
+      'hawq_segment_directory': {
+        'config-type': 'hawq-site',
+        'message': 'Multiple directories for HAWQ Segment directory are not allowed.',
+        'type': 'configuration',
+        'config-name': 'hawq_segment_directory',
+        'level': 'ERROR'
+      }
+    }
+    self.assertEqual(problems_dict, expected_warnings)
+
+    # Test hawq_global_rm_type validation
+    services = {
+                 "services" : [
+                   {
+                     "StackServices" : {
+                     "service_name" : "HAWQ"
+                     },
+                     "components": []
+                   } ],
+                 "configurations":
+                   {
+                     "hawq-site": {
+                       "properties": {
+                         "hawq_global_rm_type": "yarn"
+                       }
+                     }
+                   }
+                }
+    properties = services["configurations"]["hawq-site"]["properties"]
+
+    # case 1: hawq_global_rm_type is set as yarn, but YARN service is not installed. Validation error expected.
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, services["configurations"], services, hosts)
+    self.assertEqual(len(problems), 1)
+    expected = {
+      "config-type": "hawq-site",
+      "message": "hawq_global_rm_type must be set to none if YARN service is not installed",
+      "type": "configuration",
+      "config-name": "hawq_global_rm_type",
+      "level": "ERROR"
+    }
+    self.assertEqual(problems[0], expected)
+
+    # case 2: hawq_global_rm_type is set as yarn, and YARN service is installed. No validation errors expected.
+    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
+
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, services["configurations"], services, hosts)
+    self.assertEqual(len(problems), 0)
+
+    # Test HAWQ Master port conflict with Ambari Server Postgres port
+
+    # case 1: HAWQ Master is placed on Ambari Server and HAWQ Master port is same as Ambari Server Postgres Port
+    self.serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=True)
+    configurations = {
+      "hawq-site": {
+        "properties":
+          {"hawq_master_address_port": "5432"}
+      }
+    }
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 1)
+    expected = {
+      "config-name": "hawq_master_address_port",
+      "config-type": "hawq-site",
+      "level": "WARN",
+      "message": "The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. "
+                 "If you are using port 5432 for Postgres, you must either deploy the HAWQ Masters on a different host "
+                 "or configure a different port for the HAWQ Masters in the HAWQ Configuration page.",
+      "type": "configuration"}
+    self.assertEqual(problems[0], expected)
+
+    # case 2: HAWQ Master is placed on Ambari Server and HAWQ Master port is different from  Ambari Server Postgres Port
+    self.serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=True)
+    configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432"
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 0)
+
+    # case 3: HAWQ Master is not placed on Ambari Server and HAWQ Master port is same as  Ambari Server Postgres Port
+    self.serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False)
+    configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "5432"
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 0)
+
+    # case 4: HAWQ Master is not placed on Ambari Server and HAWQ Master port is different from  Ambari Server Postgres Port
+    self.serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False)
+    configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432"
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 0)
+
+    # -------- test query limits warning ----------
+    services = {
+      "services":  [
+        { "StackServices": {"service_name": "HAWQ"},
+          "components": [{
+            "StackServiceComponents": {
+              "component_name": "HAWQSEGMENT",
+              "hostnames": []
+            }}]
+          }],
+      "configurations": {}
+    }
+    # setup default configuration values
+    configurations = services["configurations"]
+    configurations["hawq-site"] = {"properties": {"default_hash_table_bucket_number": "600",
+                                                  "hawq_rm_nvseg_perquery_limit": "500"}}
+    properties = configurations["hawq-site"]["properties"]
+    defaults = {}
+    hosts = {}
+
+    expected = {
+      'config-type': 'hawq-site',
+      'message': 'Default buckets for Hash Distributed tables parameter value should not be greater than the value of Virtual Segments Limit per Query (Total) parameter, currently set to 500.',
+      'type': 'configuration',
+      'config-name': 'default_hash_table_bucket_number',
+      'level': 'ERROR'
+    }
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 1)
+    self.assertEqual(problems[0], expected)
+
+    configurations["hawq-site"] = {"properties": {"default_hash_table_bucket_number": "500",
+                                                  "hawq_rm_nvseg_perquery_limit": "500"}}
+    properties = configurations["hawq-site"]["properties"]
+    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 0)
+
+
+  def test_validateHAWQHdfsClientConfigurations(self):
+    services = {
+      "services":  [
+        { "StackServices": {"service_name": "HAWQ"},
+          "components": [{
+            "StackServiceComponents": {
+              "component_name": "HAWQSEGMENT",
+              "hostnames": []
+            }}]
+          }],
+      "configurations": {}
+    }
+    # setup default configuration values
+    configurations = services["configurations"]
+    configurations["hdfs-client"] = {"properties": {"output.replace-datanode-on-failure": "true"}}
+    properties = configurations["hdfs-client"]["properties"]
+    defaults = {}
+    hosts = {}
+
+    # 1. Try with no hosts
+    expected = {
+        'config-type': 'hdfs-client',
+        'message': 'output.replace-datanode-on-failure should be set to false (unchecked) for clusters with 3 or less HAWQ Segments',
+        'type': 'configuration',
+        'config-name': 'output.replace-datanode-on-failure',
+        'level': 'WARN'
+    }
+
+    problems = self.serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 1)
+    self.assertEqual(problems[0], expected)
+
+    # 2. Try with 3 hosts
+    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["host1", "host2", "host3"]
+    problems = self.serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 1)
+    self.assertEqual(problems[0], expected)
+
+    # 3. Try with 4 hosts - default value
+    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["host1", "host2", "host3", "host4"]
+    problems = self.serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 0)
+
+    # 4. Try with 4 hosts
+    properties = {"output.replace-datanode-on-failure": "false"}
+    expected = {
+      'config-type': 'hdfs-client',
+      'message': 'output.replace-datanode-on-failure should be set to true (checked) for clusters with more than 3 HAWQ Segments',
+      'type': 'configuration',
+      'config-name': 'output.replace-datanode-on-failure',
+      'level': 'WARN'
+    }
+    problems = self.serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 1)
+    self.assertEqual(problems[0], expected)

http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/HAWQ/test_utils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_utils.py b/ambari-server/src/test/python/common-services/HAWQ/test_utils.py
new file mode 100644
index 0000000..07737eb
--- /dev/null
+++ b/ambari-server/src/test/python/common-services/HAWQ/test_utils.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from stacks.utils.RMFTestCase import *
+
+import utils
+
+class TestUtils(RMFTestCase):
+
+  def test_generate_hawq_process_status_cmd(self):
+    cmd = utils.generate_hawq_process_status_cmd("master", 12345)
+    self.assertEqual(cmd, "netstat -tupln | egrep ':12345\s' | egrep postgres")

http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/PXF/test_alerts_api_status.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/PXF/test_alerts_api_status.py b/ambari-server/src/test/python/common-services/PXF/test_alerts_api_status.py
new file mode 100644
index 0000000..ee187e2
--- /dev/null
+++ b/ambari-server/src/test/python/common-services/PXF/test_alerts_api_status.py
@@ -0,0 +1,77 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+# System imports
+import os
+import sys
+
+from mock.mock import patch
+
+# Local imports
+from stacks.utils.RMFTestCase import RMFTestCase
+
+COMMON_SERVICES_ALERTS_DIR_PATH = "../../../../../main/resources/common-services/PXF/3.0.0/package/alerts"
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+alerts_dir = os.path.abspath(os.path.join(current_dir, COMMON_SERVICES_ALERTS_DIR_PATH))
+
+class TestAlertsApiStatus(RMFTestCase):
+
+  def setUp(self):
+    """
+    Import the class under test.
+    Because the class is present in a different folder, append its dir to the system path.
+    Also, shorten the import name and make it a global so the test functions can access it.
+    """
+    sys.path.append(alerts_dir)
+    global api_status
+    import api_status
+
+  @patch("api_status._makeHTTPCall")
+  def test_get_pxf_protocol_version(self, makeHTTPCall_mock):
+
+    mock_response = '{ "version": "v14"}'
+    makeHTTPCall_mock.return_value = mock_response
+    BASE_URL = "http://localhost:51200/pxf/"
+
+    version = api_status._get_pxf_protocol_version(BASE_URL)
+    self.assertEqual(version, "v14")
+
+    mock_response = 'BAD RESPONSE'
+    makeHTTPCall_mock.return_value = mock_response
+
+    try:
+      api_status._get_pxf_protocol_version(BASE_URL)
+      self.fail()
+    except Exception as e:
+      self.assertEqual(str(e), "version could not be found in response BAD RESPONSE")
+
+  @patch("api_status._makeHTTPCall")
+  def test_execute(self, makeHTTPCall_mock):
+
+    mock_response = '{ "version": "v14"}'
+    makeHTTPCall_mock.return_value = mock_response
+
+    result = api_status.execute(configurations = {})
+    self.assertEqual(result, (api_status.RESULT_STATE_OK, ['PXF is functional']))
+
+    mock_response = 'BAD RESPONSE'
+    makeHTTPCall_mock.return_value = mock_response
+
+    result = api_status.execute(configurations = {})
+    self.assertEqual(result, ('WARNING', ['PXF is not functional on host, None: version could not be found in response BAD RESPONSE']))

http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/PXF/test_pxf.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/PXF/test_pxf.py b/ambari-server/src/test/python/common-services/PXF/test_pxf.py
new file mode 100644
index 0000000..49df75d
--- /dev/null
+++ b/ambari-server/src/test/python/common-services/PXF/test_pxf.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os, copy, json
+from mock.mock import patch
+from stacks.utils.RMFTestCase import Template, RMFTestCase
+
+
+class TestPxf(RMFTestCase):
+  COMMON_SERVICES_PACKAGE_DIR = "PXF/3.0.0/package"
+  STACK_VERSION = "2.3"
+  PXF_USER = 'pxf'
+  PXF_GROUP = 'pxf'
+  TOMCAT_GROUP = 'tomcat'
+  BASH_SHELL = '/bin/bash'
+  DEFAULT_TIMEOUT = 600
+  CONFIG_FILE = os.path.join(os.path.dirname(__file__), '../configs/pxf_default.json')
+  with open(CONFIG_FILE, "r") as f:
+     pxf_config = json.load(f)
+
+  def setUp(self):
+    self.config_dict = copy.deepcopy(self.pxf_config)
+
+  def assert_configure_default(self):
+    self.assertResourceCalled('User', self.PXF_USER,
+                              groups=[self.getConfig()['configurations']['hdfs-site']['dfs.permissions.superusergroup'],
+                              self.getConfig()['configurations']['cluster-env']['user_group'],
+                              self.TOMCAT_GROUP],
+        shell=self.BASH_SHELL)
+
+    self.assertResourceCalled('File', '/etc/pxf/conf/pxf-env.sh',
+                content=Template('pxf-env.j2'))
+
+    self.assertResourceCalled('File', '/etc/pxf/conf/pxf-public.classpath',
+                content = self.getConfig()['configurations']['pxf-public-classpath']['content'].lstrip())
+
+    self.assertResourceCalled('File', '/etc/pxf/conf/pxf-profiles.xml',
+                content = self.getConfig()['configurations']['pxf-profiles']['content'].lstrip())
+
+    self.assertResourceCalled('XmlConfig', 'pxf-site.xml',
+                              conf_dir='/etc/pxf/conf',
+                              configurations=self.getConfig()['configurations']['pxf-site'],
+                              configuration_attributes=self.getConfig()['configuration_attributes']['pxf-site'])
+
+    self.assertResourceCalled('Execute', 'service pxf-service init',
+                              timeout=self.DEFAULT_TIMEOUT,
+                              logoutput=True)
+
+  @patch('shutil.copy2')
+  def test_install_default(self, shutil_copy2_mock):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py",
+                       classname="Pxf",
+                       command="install",
+                       config_dict=self.config_dict,
+                       stack_version=self.STACK_VERSION,
+                       target=RMFTestCase.TARGET_COMMON_SERVICES,
+                       try_install=True)
+
+    self.assertResourceCalled('Package', 'pxf-service',
+                              retry_count=5,
+                              retry_on_repo_unavailability=False)
+    self.assertResourceCalled('Package', 'apache-tomcat',
+                              retry_count=5,
+                              retry_on_repo_unavailability=False)
+    self.assertResourceCalled('Package', 'pxf-hive',
+                              retry_count=5,
+                              retry_on_repo_unavailability=False)
+    self.assertResourceCalled('Package', 'pxf-hdfs',
+                              retry_count=5,
+                              retry_on_repo_unavailability=False)
+    self.assertResourceCalled('Package', 'pxf-hbase',
+                              retry_count=5,
+                              retry_on_repo_unavailability=False)
+
+    self.assert_configure_default()
+
+  @patch('shutil.copy2')
+  def test_configure_default(self, shutil_copy2_mock):
+      self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py",
+                   classname="Pxf",
+                   command="configure",
+                   config_dict=self.config_dict,
+                   stack_version=self.STACK_VERSION,
+                   target=RMFTestCase.TARGET_COMMON_SERVICES,
+                   try_install=True)
+
+      self.assert_configure_default()
+
+  @patch('shutil.copy2')
+  def test_start_default(self, shutil_copy2_mock):
+      self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py",
+                   classname="Pxf",
+                   command="start",
+                   config_dict=self.config_dict,
+                   stack_version=self.STACK_VERSION,
+                   target=RMFTestCase.TARGET_COMMON_SERVICES,
+                   try_install=True)
+
+      self.assert_configure_default()
+
+      self.assertResourceCalled('Directory', '/var/pxf',
+              owner=self.PXF_USER,
+              group=self.PXF_GROUP,
+              create_parents = True)
+
+      self.assertResourceCalled('Execute', 'service pxf-service restart',
+                          timeout=self.DEFAULT_TIMEOUT,
+                          logoutput=True)
+
+  def test_stop_default(self):
+      self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py",
+                   classname="Pxf",
+                   command="stop",
+                   config_dict=self.config_dict,
+                   stack_version=self.STACK_VERSION,
+                   target=RMFTestCase.TARGET_COMMON_SERVICES,
+                   try_install=True)
+
+      self.assertResourceCalled('Execute', 'service pxf-service stop',
+                          timeout=self.DEFAULT_TIMEOUT,
+                          logoutput=True)
+
+  def test_status_default(self):
+      self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pxf.py",
+                   classname="Pxf",
+                   command="status",
+                   config_dict=self.config_dict,
+                   stack_version=self.STACK_VERSION,
+                   target=RMFTestCase.TARGET_COMMON_SERVICES,
+                   try_install=True)
+
+      self.assertResourceCalled('Execute', 'service pxf-service status',
+                          timeout=self.DEFAULT_TIMEOUT,
+                          logoutput=True)
+
+  def tearDown(self):
+      self.assertNoMoreResources()

http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/PXF/test_service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/PXF/test_service_advisor.py b/ambari-server/src/test/python/common-services/PXF/test_service_advisor.py
new file mode 100644
index 0000000..7510e5f
--- /dev/null
+++ b/ambari-server/src/test/python/common-services/PXF/test_service_advisor.py
@@ -0,0 +1,427 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os, json
+from unittest import TestCase
+
+
+class TestPXF300ServiceAdvisor(TestCase):
+
+  def setUp(self):
+    import imp
+    self.testDirectory = os.path.dirname(os.path.abspath(__file__))
+    self.pxf300ServiceAdvisorPath = os.path.join(self.testDirectory, '../../../../main/resources/common-services/PXF/3.0.0/service_advisor.py')
+    with open(self.pxf300ServiceAdvisorPath, 'rb') as fp:
+      service_advisor_impl = imp.load_module('service_advisor_impl', fp, self.pxf300ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+    serviceAdvisorClass = getattr(service_advisor_impl, 'PXF300ServiceAdvisor')
+    self.serviceAdvisor = serviceAdvisorClass()
+
+    self.PXF_PATH = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/pxf/pxf-hbase.jar"
+
+  def load_json(self, filename):
+    file = os.path.join(self.testDirectory, "../configs", filename)
+    with open(file, 'rb') as f:
+      data = json.load(f)
+    return data
+
+  def prepareHosts(self, hostsNames):
+    hosts = { "items": [] }
+    for hostName in hostsNames:
+      nextHost = {"Hosts":{"host_name" : hostName}}
+      hosts["items"].append(nextHost)
+    return hosts
+
+  def getHosts(self, componentsList, componentName):
+    return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == componentName][0]
+
+  def insertPXFServiceAdvisorInfo(self, services):
+    for service in services["services"]:
+      if service["StackServices"]["service_name"] == 'PXF':
+        service["StackServices"]["advisor_name"] = "PXF300ServiceAdvisor"
+        service["StackServices"]["advisor_path"] = self.pxf300ServiceAdvisorPath
+
+  def test_getServiceConfigurationRecommendations(self):
+    services = {
+      "configurations": {
+        "hbase-env": {
+          "properties": {
+            "content": "# Some hbase-env content text"
+          }
+        }
+      }
+    }
+
+    ## Test is PXF_PATH is being added to hbase-env content
+
+    # Case 1: Test pxf-hbase.jar classpath line was added to content
+    expected = "# Some hbase-env content text\n\n#Add pxf-hbase.jar to HBASE_CLASSPATH\n" + self.PXF_PATH
+    self.serviceAdvisor.getServiceConfigurationRecommendations(services["configurations"], None, services, None)
+    self.assertEquals(services["configurations"]["hbase-env"]["properties"]["content"], expected)
+
+    # Case 2: Test pxf-hbase.jar classpath line is not added again if content already has it
+    services["configurations"]["hbase-env"]["properties"]["content"] = self.PXF_PATH
+    expected = self.PXF_PATH
+    self.serviceAdvisor.getServiceConfigurationRecommendations(services["configurations"], None, services, None)
+    self.assertEquals(services["configurations"]["hbase-env"]["properties"]["content"], expected)
+
+  def test_getConfigurationsValidationItems(self):
+    services = {
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "PXF",
+            "service_version": "2.0",
+            "stack_name": "HDP",
+            "stack_version": "2.3"
+          }
+        },
+        {
+          "StackServices": {
+            "service_name": "HBASE",
+            "service_version": "2.0",
+            "stack_name": "HDP",
+            "stack_version": "2.3"
+          }
+        }
+      ],
+      "configurations": {
+        "hbase-env": {
+          "properties": {
+            "content": "# Some hbase-env content text"
+          }
+        }
+      }
+    }
+    properties = services["configurations"]
+
+    ## Test if PXF_PATH present in hbase-env content
+
+    # Case 1: Generate warning item if PXF_PATH is not present in hbase-env
+    expected = [
+      {
+        "config-type": "hbase-env",
+        "message": "HBASE_CLASSPATH must contain the location of pxf-hbase.jar",
+        "type": "configuration",
+        "config-name": "content",
+        "level": "WARN"
+      }
+    ]
+    items = self.serviceAdvisor.getConfigurationsValidationItems(properties, properties, services, None)
+    self.assertEquals(items, expected)
+
+    # Case 2: No warning should be generated if PXF_PATH is present in hbase-env
+    properties = services["configurations"]["hbase-env"]["properties"]["content"] = self.PXF_PATH
+    items = self.serviceAdvisor.getConfigurationsValidationItems(properties, properties, services, None)
+    self.assertEquals(items, [])
+
+
+  def test_createComponentLayoutRecommendations_pxf_cluster_install(self):
+    """ Test that PXF gets recommended correctly during Cluster Install Wizard, when PXF is selected for installation """
+
+    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
+    services = {
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "HDFS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "MASTER",
+                "component_name": "NAMENODE",
+                "is_master": "true",
+                "hostnames": []
+              }
+            },
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "SLAVE",
+                "component_name": "DATANODE",
+                "hostnames": []
+              }
+            }
+          ]
+        },
+        {
+          "StackServices": {
+            "service_name": "PXF"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "SLAVE",
+                "component_name": "PXF",
+                "hostnames": []
+              }
+            }
+          ]
+        }
+      ]
+    }
+
+    pxfHosts = set(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
+
+    self.insertPXFServiceAdvisorInfo(services)
+    recommendations = self.serviceAdvisor.createComponentLayoutRecommendations(services, hosts)
+    hostGroups = [hostgroup["name"] for hostgroup in recommendations["blueprint"]["host_groups"] if
+                  {"name": "PXF"} in hostgroup["components"]]
+    hostNames = [host["fqdn"] for hostgroup in recommendations["blueprint_cluster_binding"]["host_groups"] if
+                 hostgroup["name"] in hostGroups for host in hostgroup["hosts"]]
+    self.assertEquals(set(hostNames), pxfHosts)
+
+
+  def test_createComponentLayoutRecommendations_pxf_add_service_wizard_to_be_installed(self):
+    """ Test that PXF gets recommended correctly during Add Service Wizard, when PXF is selected for installation """
+
+    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
+    services = {
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "HDFS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "MASTER",
+                "component_name": "NAMENODE",
+                "is_master": "true",
+                "hostnames": ["c6401.ambari.apache.org"]
+              }
+            },
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "SLAVE",
+                "component_name": "DATANODE",
+                "hostnames": ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+              }
+            }
+          ]
+        },
+        {
+          "StackServices": {
+            "service_name": "PXF"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "SLAVE",
+                "component_name": "PXF",
+                "hostnames": []
+              }
+            }
+          ]
+        }
+      ]
+    }
+
+    pxfHosts = set(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
+
+    self.insertPXFServiceAdvisorInfo(services)
+    recommendations = self.serviceAdvisor.createComponentLayoutRecommendations(services, hosts)
+    hostGroups = [hostgroup["name"] for hostgroup in recommendations["blueprint"]["host_groups"] if
+                  {"name": "PXF"} in hostgroup["components"]]
+    hostNames = [host["fqdn"] for hostgroup in recommendations["blueprint_cluster_binding"]["host_groups"] if
+                 hostgroup["name"] in hostGroups for host in hostgroup["hosts"]]
+    self.assertEquals(set(hostNames), pxfHosts)
+
+
+  def test_createComponentLayoutRecommendations_pxf_add_service_wizard_already_installed(self):
+    """ Test that PXF does not get recommended during Add Service Wizard, when PXF has already been installed """
+
+    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
+    services = {
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "HDFS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "MASTER",
+                "component_name": "NAMENODE",
+                "is_master": "true",
+                "hostnames": ["c6401.ambari.apache.org"]
+              }
+            },
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "SLAVE",
+                "component_name": "DATANODE",
+                "hostnames": ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+              }
+            }
+          ]
+        },
+        {
+          "StackServices": {
+            "service_name": "PXF"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "SLAVE",
+                "component_name": "PXF",
+                "hostnames": ["c6402.ambari.apache.org"]
+              }
+            }
+          ]
+        },
+        {
+          "StackServices": {
+            "service_name": "HBASE"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "cardinality": "1+",
+                "component_category": "MASTER",
+                "component_name": "HBASE_MASTER",
+                "is_master": "true",
+                "hostnames": []
+              }
+            }
+          ]
+        }
+      ]
+    }
+
+    pxfHosts = set(["c6402.ambari.apache.org"])
+
+    self.insertPXFServiceAdvisorInfo(services)
+    recommendations = self.serviceAdvisor.createComponentLayoutRecommendations(services, hosts)
+    hostGroups = [hostgroup["name"] for hostgroup in recommendations["blueprint"]["host_groups"] if
+                  {"name": "PXF"} in hostgroup["components"]]
+    hostNames = [host["fqdn"] for hostgroup in recommendations["blueprint_cluster_binding"]["host_groups"] if
+                 hostgroup["name"] in hostGroups for host in hostgroup["hosts"]]
+    self.assertEquals(set(hostNames), pxfHosts)
+
+  def test_getComponentLayoutValidations_pxf_not_co_located_with_nn(self):
+    """ Test warning is generated when PXF is not co-located with NAMENODE """
+
+    services = self.load_json("services-hawq-pxf-hdfs.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    nameNodeComponent = self.getHosts(componentsList, "NAMENODE")
+    dataNodeComponent = self.getHosts(componentsList, "DATANODE")
+    pxfComponent = self.getHosts(componentsList, "PXF")
+    nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
+    dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+    pxfComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = [validation for validation in self.serviceAdvisor.getServiceComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"]
+    self.assertEquals(len(validations), 1)
+    expected = {
+      "type": 'host-component',
+      "level": 'WARN',
+      "component-name": 'PXF',
+      "message": 'PXF must be installed on the NameNode, Standby NameNode and all DataNodes. The following 1 host(s) do not satisfy the colocation recommendation: c6401.ambari.apache.org'
+    }
+    self.assertEquals(validations[0], expected)
+
+
+  def test_getComponentLayoutValidations_pxf_not_co_located_with_dn(self):
+    """ Test warning is generated when PXF is not co-located with NAMENODE or DATANODE """
+
+    services = self.load_json("services-hawq-pxf-hdfs.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    nameNodeComponent = self.getHosts(componentsList, "NAMENODE")
+    dataNodeComponent = self.getHosts(componentsList, "DATANODE")
+    pxfComponent = self.getHosts(componentsList, "PXF")
+    nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
+    dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+    pxfComponent["hostnames"] = ["c6401.ambari.apache.org"]
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = [validation for validation in self.serviceAdvisor.getServiceComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"]
+    self.assertEquals(len(validations), 1)
+    expected = {
+      "type": 'host-component',
+      "level": 'WARN',
+      "component-name": 'PXF',
+      "message": 'PXF must be installed on the NameNode, Standby NameNode and all DataNodes. The following 2 host(s) do not satisfy the colocation recommendation: c6402.ambari.apache.org, c6403.ambari.apache.org'
+    }
+    self.assertEquals(validations[0], expected)
+
+
+  def test_getComponentLayoutValidations_pxf_not_co_located_with_nn_or_dn(self):
+    """ Test warning is generated when PXF is not co-located with NAMENODE or DATANODE """
+
+    services = self.load_json("services-hawq-pxf-hdfs.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    nameNodeComponent = self.getHosts(componentsList, "NAMENODE")
+    dataNodeComponent = self.getHosts(componentsList, "DATANODE")
+    pxfComponent = self.getHosts(componentsList, "PXF")
+    nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
+    dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org"]
+    pxfComponent["hostnames"] = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = [validation for validation in self.serviceAdvisor.getServiceComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"]
+    self.assertEquals(len(validations), 1)
+    expected = {
+      "type": 'host-component',
+      "level": 'WARN',
+      "component-name": 'PXF',
+      "message": 'PXF must be installed on the NameNode, Standby NameNode and all DataNodes. The following 1 host(s) do not satisfy the colocation recommendation: c6403.ambari.apache.org'
+    }
+    self.assertEquals(validations[0], expected)
+
+
+  def test_getComponentLayoutValidations_pxf_co_located_with_nn_and_dn(self):
+    """ Test NO warning is generated when PXF is co-located with NAMENODE and DATANODE """
+
+    services = self.load_json("services-hawq-pxf-hdfs.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    nameNodeComponent = self.getHosts(componentsList, "NAMENODE")
+    dataNodeComponent = self.getHosts(componentsList, "DATANODE")
+    pxfComponent = self.getHosts(componentsList, "PXF")
+    nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"]
+    dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+    pxfComponent["hostnames"] = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = [validation for validation in self.serviceAdvisor.getServiceComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"]
+    self.assertEquals(len(validations), 0)

http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/configs/hawq_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/configs/hawq_default.json b/ambari-server/src/test/python/common-services/configs/hawq_default.json
new file mode 100644
index 0000000..79864a9
--- /dev/null
+++ b/ambari-server/src/test/python/common-services/configs/hawq_default.json
@@ -0,0 +1,108 @@
+{
+    "roleCommand": "SERVICE_CHECK",
+    "clusterName": "phd",
+    "hostname": "c6401.ambari.apache.org",
+    "commandType": "EXECUTION_COMMAND",
+    "serviceName": "HAWQ",
+    "role": "HAWQ_SERVICE_CHECK",
+    "hostLevelParams": {
+        "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
+        "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
+        "stack_name": "PHD",
+        "stack_version": "3.3",
+        "jdk_name": "jdk-8u60-linux-x64.tar.gz"
+    },
+    "configuration_attributes": {
+        "yarn-client": {},
+        "hdfs-site": {},
+        "core-site": {},
+        "hawq-env": {},
+        "hawq-site": {},
+        "hawq-check-env": {},
+        "hdfs-client": {},
+        "yarn-site": {},
+        "cluster-env": {}
+    },
+    "configurations": {
+        "yarn-client": {
+            "rpc.client.timeout": "3600000"
+        },
+        "hdfs-site": {
+            "dfs.allow.truncate": "true"
+        },
+        "core-site": {
+            "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020"
+        },
+        "hawq-env": {
+            "hawq_password": "gpadmin",
+            "hawq_ssh_exkeys": "false"
+        },
+        "hawq-site": {
+            "hawq_re_cgroup_hierarchy_name": "hawq",
+            "hawq_master_directory": "/data/hawq/master",
+            "hawq_segment_address_port": "40000",
+            "hawq_master_temp_directory": "/data/hawq/tmp/master",
+            "hawq_standby_address_host": "c6402.ambari.apache.org",
+            "hawq_master_address_port": "5432",
+            "hawq_segment_temp_directory": "/data/hawq/tmp/segment",
+            "hawq_master_address_host": "c6403.ambari.apache.org",
+            "hawq_rm_yarn_queue_name": "default",
+            "hawq_rm_yarn_address": "c6402.ambari.apache.org:8032",
+            "hawq_re_cgroup_mount_point": "/sys/fs/cgroup",
+            "hawq_dfs_url": "c6401.ambari.apache.org:8020/hawq_data",
+            "hawq_global_rm_type": "none",
+            "hawq_segment_directory": "/data/hawq/segment",
+            "hawq_rm_memory_limit_perseg": "64GB",
+            "hawq_rm_yarn_scheduler_address": "c6402.ambari.apache.org:8030",
+            "hawq_rm_yarn_app_name": "hawq",
+            "hawq_re_cpu_enable": "false",
+            "hawq_rm_nvcore_limit_perseg": "16"
+        },
+        "hawq-check-env": {
+            "content": ""
+        },
+        "hadoop-env": {
+            "hdfs_user": "hdfs"
+        },
+        "hdfs-client": {
+            "dfs.default.blocksize": "134217728"
+        },
+        "yarn-site": {
+            "yarn.resourcemanager.ha.enabled": "false"
+        },
+        "cluster-env": {
+            "managed_hdfs_resource_property_names": "",
+            "security_enabled": "false",
+            "user_group": "hadoop"
+        }
+    },
+    "clusterHostInfo": {
+        "slave_hosts": [
+            "c6402.ambari.apache.org",
+            "c6403.ambari.apache.org",
+            "c6401.ambari.apache.org"
+        ],
+        "hawqmaster_hosts": [
+            "c6403.ambari.apache.org"
+        ],
+        "hawqstandby_hosts": [
+            "c6402.ambari.apache.org"
+        ],
+        "hawqsegment_hosts": [
+            "c6402.ambari.apache.org",
+            "c6403.ambari.apache.org",
+            "c6401.ambari.apache.org"
+        ],
+        "namenode_host": [
+            "c6401.ambari.apache.org"
+        ],
+        "ambari_server_host": [
+            "c6401.ambari.apache.org"
+        ],
+        "all_hosts": [
+            "c6402.ambari.apache.org",
+            "c6403.ambari.apache.org",
+            "c6401.ambari.apache.org"
+        ]
+    }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/fc3e7830/ambari-server/src/test/python/common-services/configs/hosts-1-host.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/configs/hosts-1-host.json b/ambari-server/src/test/python/common-services/configs/hosts-1-host.json
new file mode 100644
index 0000000..5d6a5b6
--- /dev/null
+++ b/ambari-server/src/test/python/common-services/configs/hosts-1-host.json
@@ -0,0 +1,93 @@
+{
+  "href" : "/api/v1/hosts?fields=Hosts/*&Hosts/host_name.in(c6401.ambari.apache.org)",
+  "items" : [
+    {
+      "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+      "Hosts" : {
+        "cpu_count" : 1,
+        "desired_configs" : null,
+        "disk_info" : [
+          {
+            "available" : "481199976",
+            "device" : "/dev/mapper/VolGroup-lv_root",
+            "used" : "5713880",
+            "percent" : "2%",
+            "size" : "512971376",
+            "type" : "ext4",
+            "mountpoint" : "/"
+          },
+          {
+            "available" : "1478456",
+            "device" : "tmpfs",
+            "used" : "0",
+            "percent" : "0%",
+            "size" : "1478456",
+            "type" : "tmpfs",
+            "mountpoint" : "/dev/shm"
+          },
+          {
+            "available" : "438284",
+            "device" : "/dev/sda1",
+            "used" : "31960",
+            "percent" : "7%",
+            "size" : "495844",
+            "type" : "ext4",
+            "mountpoint" : "/boot"
+          },
+          {
+            "available" : "191037148",
+            "device" : "vagrant",
+            "used" : "296051072",
+            "percent" : "61%",
+            "size" : "487088220",
+            "type" : "vboxsf",
+            "mountpoint" : "/vagrant"
+          }
+        ],
+        "host_health_report" : "",
+        "host_name" : "c6401.ambari.apache.org",
+        "host_state" : "HEALTHY",
+        "host_status" : "HEALTHY",
+        "ip" : "192.168.64.101",
+        "last_agent_env" : {
+          "stackFoldersAndFiles" : [ ],
+          "alternatives" : [ ],
+          "existingUsers" : [ ],
+          "existingRepos" : [ ],
+          "installedPackages" : [ ],
+          "hostHealth" : {
+            "activeJavaProcs" : [ ],
+            "agentTimeStampAtReporting" : 1445288922364,
+            "serverTimeStampAtReporting" : 1445288922425,
+            "liveServices" : [
+              {
+                "desc" : "",
+                "name" : "ntpd",
+                "status" : "Healthy"
+              }
+            ]
+          },
+          "umask" : 18,
+          "transparentHugePage" : "always",
+          "firewallRunning" : false,
+          "firewallName" : "iptables",
+          "reverseLookup" : true
+        },
+        "last_heartbeat_time" : 1445288922425,
+        "last_registration_time" : 1445288888619,
+        "os_arch" : "x86_64",
+        "os_family" : "redhat6",
+        "os_type" : "centos6",
+        "ph_cpu_count" : 1,
+        "public_host_name" : "c6401.ambari.apache.org",
+        "rack_info" : "/default-rack",
+        "recovery_report" : {
+          "summary" : "DISABLED",
+          "component_reports" : [ ]
+        },
+        "recovery_summary" : "DISABLED",
+        "total_mem" : 2956916
+      }
+    }
+  ]
+}
\ No newline at end of file


Mime
View raw message