ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From odiache...@apache.org
Subject ambari git commit: AMBARI-14339. Stack advisor validations for HAWQ(Newton Alex via odiachenko).
Date Tue, 15 Dec 2015 01:44:25 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 4196801e8 -> 1bc270aac


AMBARI-14339. Stack advisor validations for HAWQ(Newton Alex via odiachenko).


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1bc270aa
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1bc270aa
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1bc270aa

Branch: refs/heads/trunk
Commit: 1bc270aac7282aef824b55e46c64c38a51eb9ed0
Parents: 4196801
Author: Oleksandr Diachenko <odiachenko@pivotal.io>
Authored: Mon Dec 14 17:43:59 2015 -0800
Committer: Oleksandr Diachenko <odiachenko@pivotal.io>
Committed: Mon Dec 14 17:43:59 2015 -0800

----------------------------------------------------------------------
 .../stacks/HDP/2.3/services/stack_advisor.py    | 132 ++++++++++++++-----
 1 file changed, 97 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/1bc270aa/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index ecf74b5..5ab05fd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -51,31 +51,25 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
 
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item for sublist in componentsListList for item in sublist]
-    hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
-    hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
+    hawqMasterHosts = [component["StackServiceComponents"]["hostnames"][0] for component
in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
+    hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"][0] for component
in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
 
     # single node case is not analyzed because HAWQ Standby Master will not be present in
single node topology due to logic in createComponentLayoutRecommendations()
-    if len(hawqMasterHosts) > 0 and len(hawqStandbyHosts) > 0:
-      commonHosts = [host for host in hawqMasterHosts[0] if host in hawqStandbyHosts[0]]
-      for host in commonHosts:
-        message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same
host."
-        childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'HAWQSTANDBY', "host": host } )
-
-    if len(hawqMasterHosts) > 0 and hostsCount > 1:
-      ambariServerHosts = [host for host in hawqMasterHosts[0] if self.isLocalHost(host)]
-      for host in ambariServerHosts:
-        message = "HAWQ Master and Ambari Server should not be deployed on the same host.
" \
-                  "If you leave them collocated, make sure to set HAWQ Master Port property
" \
-                  "to a value different from the port number used by Ambari Server database."
-        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQMASTER', "host": host } )
-
-    if len(hawqStandbyHosts) > 0 and hostsCount > 1:
-      ambariServerHosts = [host for host in hawqStandbyHosts[0] if self.isLocalHost(host)]
-      for host in ambariServerHosts:
-        message = "HAWQ Standby Master and Ambari Server should not be deployed on the same
host. " \
-                  "If you leave them collocated, make sure to set HAWQ Master Port property
" \
-                  "to a value different from the port number used by Ambari Server database."
-        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQSTANDBY', "host": host } )
+    if len(hawqMasterHosts) == 1 and len(hawqStandbyHosts) == 1 and hawqMasterHosts == hawqStandbyHosts:
+      message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host."
+      childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
+
+    if len(hawqMasterHosts) ==  1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]):
+      message = "HAWQ Master and Ambari Server should not be deployed on the same host. "
\
+                "If you leave them collocated, make sure to set HAWQ Master Port property
" \
+                "to a value different from the port number used by Ambari Server database."
+      childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } )
+
+    if len(hawqStandbyHosts) ==  1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]):
+      message = "HAWQ Standby Master and Ambari Server should not be deployed on the same
host. " \
+                "If you leave them collocated, make sure to set HAWQ Master Port property
" \
+                "to a value different from the port number used by Ambari Server database."
+      childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
 
     if "SPARK_THRIFTSERVER" in [service["StackServices"]["service_name"] for service in services["services"]]:
       if not "HIVE_SERVER" in [service["StackServices"]["service_name"] for service in services["services"]]:
@@ -117,7 +111,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
       "HIVE": self.recommendHIVEConfigurations,
       "HBASE": self.recommendHBASEConfigurations,
       "KAFKA": self.recommendKAFKAConfigurations,
-      "RANGER": self.recommendRangerConfigurations
+      "RANGER": self.recommendRangerConfigurations,
+      "HAWQ": self.recommendHAWQConfigurations
     }
     parentRecommendConfDict.update(childRecommendConfDict)
     return parentRecommendConfDict
@@ -577,18 +572,35 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
       if 'yarn-site' in services["configurations"] and 'yarn.resourcemanager.proxy-user-privileges.enabled'
in services["configurations"]["yarn-site"]["properties"]:
         putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false')
 
+
+  def isHawqMasterComponentOnAmbariServer(self, services):
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    hawqMasterComponentHosts = [component["StackServiceComponents"]["hostnames"][0] for component
in componentsList 
+                               if component["StackServiceComponents"]["component_name"] in
("HAWQMASTER", "HAWQSTANDBY")]
+    return any([self.isLocalHost(host) for host in hawqMasterComponentHosts])
+
+
+  def recommendHAWQConfigurations(self, configurations, clusterData, services, hosts):
+    putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services)
+    if self.isHawqMasterComponentOnAmbariServer(services):
+      if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]:
+        putHawqSiteProperty('hawq_master_address_port', '')
+          
+          
   def getServiceConfigurationValidators(self):
-      parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
-      childValidators = {
-        "HDFS": {"hdfs-site": self.validateHDFSConfigurations},
-        "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations,
-                 "hive-site": self.validateHiveConfigurations},
-        "HBASE": {"hbase-site": self.validateHBASEConfigurations},
-        "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations},
-        "YARN": {"yarn-site": self.validateYARNConfigurations}
-      }
-      self.mergeValidators(parentValidators, childValidators)
-      return parentValidators
+    parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
+    childValidators = {
+      "HDFS": {"hdfs-site": self.validateHDFSConfigurations},
+      "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations,
+               "hive-site": self.validateHiveConfigurations},
+      "HBASE": {"hbase-site": self.validateHBASEConfigurations},
+      "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations},
+      "YARN": {"yarn-site": self.validateYARNConfigurations},
+      "HAWQ": {"hawq-site": self.validateHAWQConfigurations}
+    }
+    self.mergeValidators(parentValidators, childValidators)
+    return parentValidators
 
   def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services,
hosts):
     super(HDP23StackAdvisor, self).validateHDFSConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
@@ -774,5 +786,55 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
 
     return self.toConfigurationValidationProblems(validationItems, "yarn-site")
 
+
+  def isHawqMasterPortConflict(self, configurations):
+    prop_name = 'hawq_master_address_port'
+    default_ambari_port = 5432
+    if prop_name in configurations["hawq-site"]["properties"]:
+      portValue = int(configurations["hawq-site"]["properties"][prop_name])
+      return portValue == default_ambari_port
+
+    return False
+
+
+  def validateIfRootDir(self, properties, validationItems, prop_name, display_name):
+    root_dir = '/'
+    if prop_name in properties and properties[prop_name].strip() == root_dir:
+      validationItems.append({"config-name": prop_name,
+                              "item": self.getWarnItem(
+                              "It is not advisable to have " + display_name + " at " + root_dir
+". Consider creating a sub directory for HAWQ")})
+
+
+  def validateHAWQConfigurations(self, properties, recommendedDefaults, configurations, services,
hosts):
+    hawq_site = properties
+    validationItems = []
+
+    # 1. Check if HAWQ master/standby port numbers don't conflict with Ambari ports. Both
Ambari and HAWQ use postgres DB and 5432 port.
+    if self.isHawqMasterComponentOnAmbariServer(services) and self.isHawqMasterPortConflict(configurations):
+      prop_name = 'hawq_master_address_port'
+      validationItems.append({"config-name": prop_name,
+                                "item": self.getWarnItem(
+                                "HAWQ Master or Standby Master cannot use the port 5432 when
installed on the same host as the Ambari Server. Ambari Postgres DB uses the same port. Please
choose a different value (e.g. 10432)")})
+
+    # 2. Check if any data directories are pointing to root dir '/'
+    prop_name = 'hawq_master_directory'
+    display_name = 'HAWQ Master directory'
+    self.validateIfRootDir (properties, validationItems, prop_name, display_name)
+
+    prop_name = 'hawq_master_temp_directory'
+    display_name = 'HAWQ Master temp directory'
+    self.validateIfRootDir (properties, validationItems, prop_name, display_name)
+
+    prop_name = 'hawq_segment_directory'
+    display_name = 'HAWQ Segment directory'
+    self.validateIfRootDir (properties, validationItems, prop_name, display_name)
+
+    prop_name = 'hawq_segment_temp_directory'
+    display_name = 'HAWQ Segment temp directory'
+    self.validateIfRootDir (properties, validationItems, prop_name, display_name)
+
+    return self.toConfigurationValidationProblems(validationItems, "hawq-site")
+  
+  
   def isComponentUsingCardinalityForLayout(self, componentName):
     return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']


Mime
View raw message