ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ja...@apache.org
Subject ambari git commit: AMBARI-14494: Generate warning on deploy wizard UI when HAWQ segment is not co-located with a datanode. (lavjain via jaoki)
Date Tue, 12 Jan 2016 01:49:09 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 3b645ae08 -> 842c51a39


AMBARI-14494: Generate warning on deploy wizard UI when HAWQ segment is not co-located with
a datanode. (lavjain via jaoki)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/842c51a3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/842c51a3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/842c51a3

Branch: refs/heads/branch-2.2
Commit: 842c51a39a6a6724426ac2fd614da21af1a2889c
Parents: 3b645ae
Author: Jun Aoki <jaoki@apache.org>
Authored: Mon Jan 11 17:49:02 2016 -0800
Committer: Jun Aoki <jaoki@apache.org>
Committed: Mon Jan 11 17:49:02 2016 -0800

----------------------------------------------------------------------
 .../stacks/HDP/2.3/services/stack_advisor.py    | 93 +++++++++++---------
 .../stacks/2.3/common/test_stack_advisor.py     | 24 +++++
 2 files changed, 75 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/842c51a3/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 3970844..13cb817 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -53,50 +53,59 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
   def getComponentLayoutValidations(self, services, hosts):
     parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services,
hosts)
 
-    hiveExists = "HIVE" in [service["StackServices"]["service_name"] for service in services["services"]]
-    sparkExists = "SPARK" in [service["StackServices"]["service_name"] for service in services["services"]]
-
-    if not "HAWQ" in [service["StackServices"]["service_name"] for service in services["services"]]
and not sparkExists:
-      return parentItems
-
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item["StackServiceComponents"] for sublist in componentsListList for
item in sublist]
     childItems = []
-    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
-    hostsCount = len(hostsList)
 
-    componentsListList = [service["components"] for service in services["services"]]
-    componentsList = [item for sublist in componentsListList for item in sublist]
-    hawqMasterHosts = [hostname for component in componentsList if component["StackServiceComponents"]["component_name"]
== "HAWQMASTER" for hostname in component["StackServiceComponents"]["hostnames"]]
-    hawqStandbyHosts = [hostname for component in componentsList if component["StackServiceComponents"]["component_name"]
== "HAWQSTANDBY" for hostname  in component["StackServiceComponents"]["hostnames"]]
-
-    # single node case is not analyzed because HAWQ Standby Master will not be present in
single node topology due to logic in createComponentLayoutRecommendations()
-    if len(hawqMasterHosts) == 1 and len(hawqStandbyHosts) == 1 and hawqMasterHosts == hawqStandbyHosts:
-      message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host."
-      childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
-
-    if len(hawqMasterHosts) ==  1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]):
-      message = "HAWQ Master and Ambari Server should not be deployed on the same host. "
\
-                "If you leave them collocated, make sure to set HAWQ Master Port property
" \
-                "to a value different from the port number used by Ambari Server database."
-      childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } )
-
-    if len(hawqStandbyHosts) ==  1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]):
-      message = "HAWQ Standby Master and Ambari Server should not be deployed on the same
host. " \
-                "If you leave them collocated, make sure to set HAWQ Master Port property
" \
-                "to a value different from the port number used by Ambari Server database."
-      childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
-
-    if "SPARK_THRIFTSERVER" in [service["StackServices"]["service_name"] for service in services["services"]]:
-      if not "HIVE_SERVER" in [service["StackServices"]["service_name"] for service in services["services"]]:
-        message = "SPARK_THRIFTSERVER requires HIVE services to be selected."
-        childItems.append( {"type": 'host-component', "level": 'ERROR', "message": messge,
"component-name": 'SPARK_THRIFTSERVER'} )
-
-    hmsHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList
if component["StackServiceComponents"]["component_name"] == "HIVE_METASTORE"][0] if hiveExists
else []
-    sparkTsHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList
if component["StackServiceComponents"]["component_name"] == "SPARK_THRIFTSERVER"][0] if sparkExists
else []
-
-    # if Spark Thrift Server is deployed but no Hive Server is deployed
-    if len(sparkTsHosts) > 0 and len(hmsHosts) == 0:
-      message = "SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed."
-      childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'SPARK_THRIFTSERVER' } )
+    if "HAWQ" in servicesList:
+      hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+      hostsCount = len(hostsList)
+
+      hawqMasterHosts = [component["hostnames"] for component in componentsList if component["component_name"]
== "HAWQMASTER"][0]
+      hawqStandbyHosts = [component["hostnames"] for component in componentsList if component["component_name"]
== "HAWQSTANDBY"][0]
+      hawqSegmentHosts = [component["hostnames"] for component in componentsList if component["component_name"]
== "HAWQSEGMENT"][0]
+      datanodeHosts = [component["hostnames"] for component in componentsList if component["component_name"]
== "DATANODE"][0]
+
+      # Generate WARNING if any HAWQSEGMENT is not colocated with a DATANODE
+      mismatchHosts = set(hawqSegmentHosts).symmetric_difference(set(datanodeHosts))
+      if len(mismatchHosts) > 0:
+        hostsString = ', '.join(mismatchHosts)
+        message = "HAWQSEGMENT is not co-located with DATANODE on the following {0} host(s):
{1}".format(len(mismatchHosts), hostsString)
+        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQSEGMENT' } )
+
+      # single node case is not analyzed because HAWQ Standby Master will not be present
in single node topology due to logic in createComponentLayoutRecommendations()
+      if len(hawqMasterHosts) == 1 and len(hawqStandbyHosts) == 1 and hawqMasterHosts ==
hawqStandbyHosts:
+        message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same
host."
+        childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
+
+      if len(hawqMasterHosts) ==  1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]):
+        message = "HAWQ Master and Ambari Server should not be deployed on the same host.
" \
+                  "If you leave them collocated, make sure to set HAWQ Master Port property
" \
+                  "to a value different from the port number used by Ambari Server database."
+        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } )
+
+      if len(hawqStandbyHosts) ==  1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]):
+        message = "HAWQ Standby Master and Ambari Server should not be deployed on the same
host. " \
+                  "If you leave them collocated, make sure to set HAWQ Master Port property
" \
+                  "to a value different from the port number used by Ambari Server database."
+        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
+
+    if "SPARK" in servicesList:
+      if "SPARK_THRIFTSERVER" in servicesList:
+        if not "HIVE_SERVER" in servicesList:
+          message = "SPARK_THRIFTSERVER requires HIVE services to be selected."
+          childItems.append( {"type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'SPARK_THRIFTSERVER'} )
+
+      hmsHosts = [component["hostnames"] for component in componentsList
+                  if component["component_name"] == "HIVE_METASTORE"][0] if "HIVE" in servicesList
else []
+      sparkTsHosts = [component["hostnames"] for component in componentsList
+                      if component["component_name"] == "SPARK_THRIFTSERVER"][0] if "SPARK"
in servicesList else []
+
+      # if Spark Thrift Server is deployed but no Hive Server is deployed
+      if len(sparkTsHosts) > 0 and len(hmsHosts) == 0:
+        message = "SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed."
+        childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'SPARK_THRIFTSERVER' } )
 
     parentItems.extend(childItems)
     return parentItems

http://git-wip-us.apache.org/repos/asf/ambari/blob/842c51a3/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index baf3180..3d478e3 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -182,6 +182,30 @@ class TestHDP23StackAdvisor(TestCase):
       return 'c6401.ambari.apache.org' if value is None else value
 
 
+  def test_validations_hawqsegment(self):
+    """ Test validation warning for HAWQ segment not colocated with DATANODE """
+
+    services = self.load_json("services-normal-hawq-3-hosts.json")
+    hosts = self.load_json("hosts-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+
+    hawqsegmentComponent = [component["StackServiceComponents"] for component in componentsList
+                            if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
+    hawqsegmentComponent["hostnames"] = ['c6401.ambari.apache.org']
+    datanodeComponent = [component["StackServiceComponents"] for component in componentsList
+                         if component["StackServiceComponents"]["component_name"] == "DATANODE"][0]
+    datanodeComponent["hostnames"] = ['c6402.ambari.apache.org']
+
+    validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
+    expected = {'component-name': 'HAWQSEGMENT', 'message': 'HAWQSEGMENT is not co-located
with DATANODE on the following 2 host(s): c6402.ambari.apache.org, c6401.ambari.apache.org',
'type': 'host-component', 'level': 'WARN'}
+    self.assertEquals(validations[0], expected)
+
+    datanodeComponent["hostnames"] = ['c6401.ambari.apache.org']
+    validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 0)
+
+
   @patch('socket.getfqdn', side_effect=fqdn_mock_result)
   def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock):
     """ Test layout validations for HAWQ components on a 3-node cluster """


Mime
View raw message