ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [07/50] [abbrv] git commit: AMBARI-6885. stack_advisor.py from one stack-version contains another's logic
Date Wed, 20 Aug 2014 15:30:08 GMT
AMBARI-6885. stack_advisor.py from one stack-version contains another's logic


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/32caa435
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/32caa435
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/32caa435

Branch: refs/heads/branch-alerts-dev
Commit: 32caa4359b7fd777fc7ca9b74e6d8a1212d6f6a0
Parents: eefa2fd
Author: Srimanth Gunturi <sgunturi@hortonworks.com>
Authored: Fri Aug 15 17:14:51 2014 -0700
Committer: Srimanth Gunturi <sgunturi@hortonworks.com>
Committed: Mon Aug 18 10:36:04 2014 -0700

----------------------------------------------------------------------
 .../stacks/HDP/2.0.6/services/stack_advisor.py  | 274 ++++++++-----------
 .../stacks/HDP/2.1/services/stack_advisor.py    |  68 +++++
 .../stacks/2.0.6/common/test_stack_advisor.py   |  49 ++--
 3 files changed, 211 insertions(+), 180 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/32caa435/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 025caa2..77c57e2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -41,15 +41,6 @@ class HDP206StackAdvisor(StackAdvisor):
       "services": servicesList,
       "recommendations": {
         "blueprint": {
-          "configurations": {
-            "global": {
-              "properties": { }
-            },
-            "core-site": { },
-            "hdfs-site": { },
-            "yarn-site": { },
-            "hbase-site": { }
-          },
           "host_groups": [ ]
         },
         "blueprint_cluster_binding": {
@@ -71,7 +62,7 @@ class HDP206StackAdvisor(StackAdvisor):
           hostsForComponent = component["StackServiceComponents"]["hostnames"]
         else:
           availableHosts = hostsList
-          if len(hostsList) > 1 and isNotPreferableOnAmbariServerHost(component):
+          if len(hostsList) > 1 and self.isNotPreferableOnAmbariServerHost(component):
             availableHosts = [hostName for hostName in hostsList if not isLocalHost(hostName)]
 
           if isMasterWithMultipleInstances(component):
@@ -81,9 +72,9 @@ class HDP206StackAdvisor(StackAdvisor):
                 hostsCount = len(availableHosts)
               hostsForComponent = availableHosts[:hostsCount]
             else:
-              hostsForComponent = [getHostForComponent(component, availableHosts)]
+              hostsForComponent = [self.getHostForComponent(component, availableHosts)]
           else:
-            hostsForComponent = [getHostForComponent(component, availableHosts)]
+            hostsForComponent = [self.getHostForComponent(component, availableHosts)]
 
         #extend 'hostsComponentsMap' with 'hostsForComponent'
         for hostName in hostsForComponent:
@@ -94,7 +85,7 @@ class HDP206StackAdvisor(StackAdvisor):
     #extend 'hostsComponentsMap' with Slave and Client Components
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item for sublist in componentsListList for item in sublist]
-    usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if not isNotValuable(component)]
+    usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if not self.isNotValuable(component)]
     utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
     freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
 
@@ -135,6 +126,46 @@ class HDP206StackAdvisor(StackAdvisor):
     return recommendations
   pass
 
+  def getHostForComponent(self, component, hostsList):
+    componentName = component["StackServiceComponents"]["component_name"]
+    scheme = self.defineSelectionScheme(componentName)
+
+    if len(hostsList) == 1:
+      return hostsList[0]
+    else:
+      for key in scheme.keys():
+        if isinstance(key, ( int, long )):
+          if len(hostsList) < key:
+            return hostsList[scheme[key]]
+      return hostsList[scheme['else']]
+
+  def defineSelectionScheme(self, componentName):
+    scheme = self.selectionScheme(componentName)
+    if scheme is None:
+      scheme = {"else": 0}
+    return scheme
+
+  def selectionScheme(self, componentName):
+    return {
+      'NAMENODE': {"else": 0},
+      'SECONDARY_NAMENODE': {"else": 1},
+      'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
+
+      'HISTORYSERVER': {31: 1, "else": 2},
+      'RESOURCEMANAGER': {31: 1, "else": 2},
+
+      'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
+
+      'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
+      'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
+      'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
+      }.get(componentName, None)
+
+  def isNotPreferableOnAmbariServerHost(self, component):
+    componentName = component["StackServiceComponents"]["component_name"]
+    service = ['GANGLIA_SERVER', 'NAGIOS_SERVER']
+    return componentName in service
+
   def validateComponentLayout(self, services, hosts):
     """Returns array of Validation objects about issues with hostnames components assigned
to"""
     stackName = services["Versions"]["stack_name"]
@@ -195,7 +226,7 @@ class HDP206StackAdvisor(StackAdvisor):
            items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Cardinality
violation, cardinality={0}, hosts count={1}'.format(cardinality, str(componentHostsCount)),
"component-name": str(componentName) } )
 
     # Validating host-usage
-    usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if not isNotValuable(component)]
+    usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if not self.isNotValuable(component)]
     usedHostsList = [item for sublist in usedHostsListList for item in sublist]
     nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
     for host in nonUsedHostsList:
@@ -204,6 +235,11 @@ class HDP206StackAdvisor(StackAdvisor):
     return validations
   pass
 
+  def isNotValuable(self, component):
+    componentName = component["StackServiceComponents"]["component_name"]
+    service = ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR']
+    return componentName in service
+
   def recommendConfigurations(self, services, hosts):
     stackName = services["Versions"]["stack_name"]
     stackVersion = services["Versions"]["stack_version"]
@@ -242,9 +278,7 @@ class HDP206StackAdvisor(StackAdvisor):
   def recommendServiceConfigurations(self, service):
     return {
       "YARN": self.recommendYARNConfigurations,
-      "MAPREDUCE2": self.recommendMapReduce2Configurations,
-      "HIVE": self.recommendHiveConfigurations,
-      "OOZIE": self.recommendOozieConfigurations
+      "MAPREDUCE2": self.recommendMapReduce2Configurations
     }.get(service, None)
 
   def putProperty(self, config, configType):
@@ -259,15 +293,6 @@ class HDP206StackAdvisor(StackAdvisor):
     putYarnProperty('yarn.scheduler.minimum-allocation-mb', clusterData['ramPerContainer'])
     putYarnProperty('yarn.scheduler.maximum-allocation-mb', clusterData['containers'] * clusterData['ramPerContainer'])
 
-  def recommendHiveConfigurations(self, configurations, clusterData):
-    containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else clusterData['reduceMemory']
-    containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
-    putHiveProperty = self.putProperty(configurations, "hive-site")
-    putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(containerSize /
3) * 1048576)
-    putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(0.8 * containerSize))
-                    + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC")
-    putHiveProperty('hive.tez.container.size', containerSize)
-
   def recommendMapReduce2Configurations(self, configurations, clusterData):
     putMapredProperty = self.putProperty(configurations, "mapred-site")
     putMapredProperty('yarn.app.mapreduce.am.resource.mb', clusterData['amMemory'])
@@ -278,14 +303,6 @@ class HDP206StackAdvisor(StackAdvisor):
     putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8 * clusterData['reduceMemory']))
+ "m")
     putMapredProperty('mapreduce.task.io.sort.mb', int(min(0.4 * clusterData['mapMemory'],
1024)))
 
-  def recommendOozieConfigurations(self, configurations, clusterData):
-    if "FALCON_SERVER" in clusterData["components"]:
-      putMapredProperty = self.putProperty(configurations, "oozie-site")
-      putMapredProperty("oozie.services.ext",
-                        "org.apache.oozie.service.JMSAccessorService," +
-                        "org.apache.oozie.service.PartitionDependencyManagerService," +
-                        "org.apache.oozie.service.HCatAccessorService")
-
   def getClusterData(self, servicesList, hosts, components):
 
     hBaseInstalled = False
@@ -380,26 +397,72 @@ class HDP206StackAdvisor(StackAdvisor):
     configurations = services["configurations"]
     for service in services["services"]:
       serviceName = service["StackServices"]["service_name"]
-      if serviceName == "MAPREDUCE2":
-        mapReduceErrors = validateMapReduce2Configurations(getSiteProperties(configurations,
"mapred-site"), recommendedDefaults["mapred-site"]["properties"])
-        items.extend(mapReduceErrors)
-      elif serviceName == "HIVE":
-        hiveErrors = validateHiveConfigurations(getSiteProperties(configurations, "hive-site"),
recommendedDefaults["hive-site"]["properties"])
-        items.extend(hiveErrors)
-      elif serviceName == "STORM":
-        oozieErrors = [] #validateStormConfigurations(getSiteProperties(configurations, "storm-site"),
recommendedDefaults["storm-site"]["properties"])
-        items.extend(oozieErrors)
-      elif serviceName == "TEZ":
-        tezErrors = validateTezConfigurations(getSiteProperties(configurations, "tez-site"),
recommendedDefaults["tez-site"]["properties"])
-        items.extend(tezErrors)
-      elif serviceName == "YARN":
-        yarnErrors = validateYARNConfigurations(getSiteProperties(configurations, "yarn-site"),
recommendedDefaults["yarn-site"]["properties"])
-        items.extend(yarnErrors)
-      else:
-        pass
+      validator = self.validateServiceConfigurations(serviceName)
+      if validator is not None:
+        siteName = validator[0]
+        method = validator[1]
+        resultItems = method(getSiteProperties(configurations, siteName), recommendedDefaults[siteName]["properties"])
+        items.extend(resultItems)
     return validations
     pass
 
+  def validateServiceConfigurations(self, serviceName):
+    return {
+      "MAPREDUCE2": ["mapred-site", self.validateMapReduce2Configurations],
+      "YARN": ["yarn-site", self.validateYARNConfigurations]
+    }.get(serviceName, None)
+
+  def toConfigurationValidationErrors(self, items, siteName):
+    result = []
+    for item in items:
+      if item["message"] is not None:
+        error = { "type": 'configuration', "level": 'ERROR', "message": item["message"],
"config-type": siteName, "config-name": item["config-name"] }
+        result.append(error)
+    return result
+
+  def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName):
+    value = to_number(properties[propertyName])
+    if value is None:
+      return "Value should be integer"
+    defaultValue = to_number(recommendedDefaults[propertyName])
+    if defaultValue is None:
+      return None
+    if value < defaultValue:
+      return "Value is less than the recommended default of {0}".format(defaultValue)
+    return None
+
+  def validateXmxValue(self, properties, recommendedDefaults, propertyName):
+    value = properties[propertyName]
+    defaultValue = recommendedDefaults[propertyName]
+    if defaultValue is None:
+      return "Config's default value can't be null or undefined"
+    if not checkXmxValueFormat(value):
+      return 'Invalid value format'
+    valueInt = formatXmxSizeToBytes(getXmxSize(value))
+    defaultValueXmx = getXmxSize(defaultValue)
+    defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
+    if valueInt < defaultValueInt:
+      return "Value is less than the recommended default of -Xmx" + defaultValueXmx
+    return None
+
+  def validateMapReduce2Configurations(self, properties, recommendedDefaults):
+    validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'mapreduce.map.java.opts')},
+                        {"config-name": 'mapreduce.reduce.java.opts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'mapreduce.reduce.java.opts')},
+                        {"config-name": 'mapreduce.task.io.sort.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.task.io.sort.mb')},
+                        {"config-name": 'mapreduce.map.memory.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.map.memory.mb')},
+                        {"config-name": 'mapreduce.reduce.memory.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.reduce.memory.mb')},
+                        {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
+                        {"config-name": 'yarn.app.mapreduce.am.command-opts', "message":
self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')}
]
+    return self.toConfigurationValidationErrors(validationItems, "mapred-site")
+
+  def validateYARNConfigurations(self, properties, recommendedDefaults):
+    validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message":
self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
+                        {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message":
self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
+                        {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message":
self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')}
]
+    return self.toConfigurationValidationErrors(validationItems, "yarn-site")
+
+
+# Validation helper methods
 def getSiteProperties(configurations, siteName):
   if configurations[siteName] is None:
     return {}
@@ -413,39 +476,6 @@ def to_number(s):
   except ValueError:
     return None
 
-def toConfigurationValidationErrors(items, siteName):
-  result = []
-  for item in items:
-    if item["message"] is not None:
-      error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type":
siteName, "config-name": item["config-name"] }
-      result.append(error)
-  return result
-
-def validatorLessThenDefaultValue(properties, recommendedDefaults, propertyName):
-  value = to_number(properties[propertyName])
-  if value is None:
-    return "Value should be integer"
-  defaultValue = to_number(recommendedDefaults[propertyName])
-  if defaultValue is None:
-    return None
-  if value < defaultValue:
-    return "Value is less than the recommended default of {0}".format(defaultValue)
-  return None
-
-def validateXmxValue(properties, recommendedDefaults, propertyName):
-  value = properties[propertyName]
-  defaultValue = recommendedDefaults[propertyName]
-  if defaultValue is None:
-    return "Config's default value can't be null or undefined"
-  if not checkXmxValueFormat(value):
-    return 'Invalid value format'
-  valueInt = formatXmxSizeToBytes(getXmxSize(value))
-  defaultValueXmx = getXmxSize(defaultValue)
-  defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
-  if valueInt < defaultValueInt:
-    return "Value is less than the recommended default of -Xmx" + defaultValueXmx
-  return None
-
 def checkXmxValueFormat(value):
   p = re.compile('-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)?')
   matches = p.findall(value)
@@ -477,58 +507,8 @@ def formatXmxSizeToBytes(value):
     }[1]
   return to_number(value) * m
 
-def validateMapReduce2Configurations(properties, recommendedDefaults):
-  validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": validateXmxValue(properties,
recommendedDefaults, 'mapreduce.map.java.opts')},
-                      {"config-name": 'mapreduce.reduce.java.opts', "message": validateXmxValue(properties,
recommendedDefaults, 'mapreduce.reduce.java.opts')},
-                      {"config-name": 'mapreduce.task.io.sort.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.task.io.sort.mb')},
-                      {"config-name": 'mapreduce.map.memory.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.map.memory.mb')},
-                      {"config-name": 'mapreduce.reduce.memory.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.reduce.memory.mb')},
-                      {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
-                      {"config-name": 'yarn.app.mapreduce.am.command-opts', "message": validateXmxValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
-  return toConfigurationValidationErrors(validationItems, "mapred-site")
-
-def validateHiveConfigurations(properties, recommendedDefaults):
-  validationItems = [ {"config-name": 'hive.tez.container.size', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'hive.tez.container.size')},
-                      {"config-name": 'hive.tez.java.opts', "message": validateXmxValue(properties,
recommendedDefaults, 'hive.tez.java.opts')},
-                      {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "message":
validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')}
]
-  return toConfigurationValidationErrors(validationItems, "hive-site")
-
-def validateStormConfigurations(properties, recommendedDefaults):
-  validationItems = [ {"config-name": 'drpc.childopts', "message": validateXmxValue(properties,
recommendedDefaults, 'drpc.childopts')},
-                      {"config-name": 'ui.childopts', "message": validateXmxValue(properties,
recommendedDefaults, 'ui.childopts')},
-                      {"config-name": 'logviewer.childopts', "message": validateXmxValue(properties,
recommendedDefaults, 'logviewer.childopts')} ]
-  return toConfigurationValidationErrors(validationItems, "storm-site")
-
-def validateTezConfigurations(properties, recommendedDefaults):
-  validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.am.resource.memory.mb')},
-                      {"config-name": 'tez.am.java.opts', "message": validateXmxValue(properties,
recommendedDefaults, 'tez.am.java.opts')} ]
-  return toConfigurationValidationErrors(validationItems, "tez-site")
-
-def validateYARNConfigurations(properties, recommendedDefaults):
-  validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
-                      {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message":
validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
-                      {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message":
validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')}
]
-  return toConfigurationValidationErrors(validationItems, "yarn-site")
-
-# Helper methods
-def getHostForComponent(component, hostsList):
-  componentName = component["StackServiceComponents"]["component_name"]
-  scheme = selectionScheme(componentName)
-
-  if len(hostsList) == 1:
-    return hostsList[0]
-  else:
-    for key in scheme.keys():
-      if isinstance(key, ( int, long )):
-        if len(hostsList) < key:
-          return hostsList[scheme[key]]
-    return hostsList[scheme['else']]
-
-def isNotValuable(component):
-  componentName = component["StackServiceComponents"]["component_name"]
-  service = ['JOURNALNODE', 'ZKFC', 'APP_TIMELINE_SERVER', 'GANGLIA_MONITOR']
-  return componentName in service
 
+# Recommendation helper methods
 def isAlreadyPopulated(component):
   if component["StackServiceComponents"]["hostnames"] is not None:
     return len(component["StackServiceComponents"]["hostnames"]) > 0
@@ -550,11 +530,6 @@ def isMaster(component):
 def isLocalHost(hostName):
   return socket.getfqdn(hostName) == socket.getfqdn()
 
-def isNotPreferableOnAmbariServerHost(component):
-  componentName = component["StackServiceComponents"]["component_name"]
-  service = ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER',
'NAGIOS_SERVER', 'HUE_SERVER']
-  return componentName in service
-
 def isMasterWithMultipleInstances(component):
   componentName = component["StackServiceComponents"]["component_name"]
   masters = ['ZOOKEEPER_SERVER', 'HBASE_MASTER']
@@ -572,22 +547,3 @@ def cardinality(componentName):
     'HBASE_MASTER': {min: 1},
     }.get(componentName, {min:1, max:1})
 
-def selectionScheme(componentName):
-  return {
-    'NAMENODE': {"else": 0},
-    'SECONDARY_NAMENODE': {"else": 1},
-    'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
-
-    'JOBTRACKER': {31: 1, "else": 2},
-    'HISTORYSERVER': {31: 1, "else": 2},
-    'RESOURCEMANAGER': {31: 1, "else": 2},
-    'APP_TIMELINE_SERVER': {31: 1, "else": 2},
-
-    'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
-    'FALCON_SERVER': {6: 1, 31: 2, "else": 3},
-
-    'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
-    'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
-    'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
-    }.get(componentName, {"else": 0})
-

http://git-wip-us.apache.org/repos/asf/ambari/blob/32caa435/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index eb2faa1..5d7a3bc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -27,11 +27,30 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
     calculator = super(HDP21StackAdvisor, self).recommendServiceConfigurations(service)
     if calculator is None:
       return {
+        "OOZIE": self.recommendOozieConfigurations,
+        "HIVE": self.recommendHiveConfigurations,
         "TEZ": self.recommendTezConfigurations
       }.get(service, None)
     else:
       return calculator
 
+  def recommendOozieConfigurations(self, configurations, clusterData):
+    if "FALCON_SERVER" in clusterData["components"]:
+      putMapredProperty = self.putProperty(configurations, "oozie-site")
+      putMapredProperty("oozie.services.ext",
+                        "org.apache.oozie.service.JMSAccessorService," +
+                        "org.apache.oozie.service.PartitionDependencyManagerService," +
+                        "org.apache.oozie.service.HCatAccessorService")
+
+  def recommendHiveConfigurations(self, configurations, clusterData):
+    containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else clusterData['reduceMemory']
+    containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
+    putHiveProperty = self.putProperty(configurations, "hive-site")
+    putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(containerSize /
3) * 1048576)
+    putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(0.8 * containerSize))
+                    + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC")
+    putHiveProperty('hive.tez.container.size', containerSize)
+
   def recommendTezConfigurations(self, configurations, clusterData):
     putTezProperty = self.putProperty(configurations, "tez-site")
     putTezProperty("tez.am.resource.memory.mb", clusterData['amMemory'])
@@ -39,3 +58,52 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
                    "-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
                    + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC")
 
+  def isNotPreferableOnAmbariServerHost(self, component):
+    componentName = component["StackServiceComponents"]["component_name"]
+    service = ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER',
'NAGIOS_SERVER']
+    return componentName in service
+
+  def isNotValuable(self, component):
+    componentName = component["StackServiceComponents"]["component_name"]
+    service = ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER']
+    return componentName in service
+
+  def selectionScheme(self, componentName):
+    scheme = super(HDP21StackAdvisor, self).selectionScheme(componentName)
+    if scheme is None:
+      return {
+        'APP_TIMELINE_SERVER': {31: 1, "else": 2},
+        'FALCON_SERVER': {6: 1, 31: 2, "else": 3}
+        }.get(componentName, None)
+    else:
+      return scheme
+
+  def validateServiceConfigurations(self, serviceName):
+    validator = super(HDP21StackAdvisor, self).validateServiceConfigurations(serviceName)
+    if validator is None:
+      return {
+        "STORM": ["storm-site", self.validateStormConfigurations],
+        "HIVE": ["hive-site", self.validateHiveConfigurations],
+        "TEZ": ["tez-site", self.validateTezConfigurations]
+      }.get(serviceName, None)
+    else:
+      return validator
+
+  def validateHiveConfigurations(self, properties, recommendedDefaults):
+    validationItems = [ {"config-name": 'hive.tez.container.size', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'hive.tez.container.size')},
+                        {"config-name": 'hive.tez.java.opts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'hive.tez.java.opts')},
+                        {"config-name": 'hive.auto.convert.join.noconditionaltask.size',
"message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')}
]
+    return self.toConfigurationValidationErrors(validationItems, "hive-site")
+
+  def validateStormConfigurations(self, properties, recommendedDefaults):
+    validationItems = [ {"config-name": 'drpc.childopts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'drpc.childopts')},
+                        {"config-name": 'ui.childopts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'ui.childopts')},
+                        {"config-name": 'logviewer.childopts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'logviewer.childopts')} ]
+    return self.toConfigurationValidationErrors(validationItems, "storm-site")
+
+  def validateTezConfigurations(self, properties, recommendedDefaults):
+    validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.am.resource.memory.mb')},
+                        {"config-name": 'tez.am.java.opts', "message": self.validateXmxValue(properties,
recommendedDefaults, 'tez.am.java.opts')} ]
+    return self.toConfigurationValidationErrors(validationItems, "tez-site")
+
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/32caa435/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 3139a3e..b5bdba6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -35,7 +35,7 @@ class TestHDP206StackAdvisor(TestCase):
       stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath,
('.py', 'rb', imp.PY_SOURCE))
     clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName)
     self.stackAdvisor = clazz()
-
+    
   def test_recommendationCardinalityALL(self):
     servicesInfo = [
       {
@@ -98,12 +98,12 @@ class TestHDP206StackAdvisor(TestCase):
     hosts = self.prepareHosts(["host1", "host2"])
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
-    expectedMessages = [
-      "NameNode and Secondary NameNode cannot be hosted on same machine",
-      "NameNode and Secondary NameNode cannot be hosted on same machine",
-      "Host is not used"
+    expectedItems = [
+      {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host":
"host1"},
+      {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host":
"host1"},
+      {"message": "Host is not used", "host": "host2"}
     ]
-    self.assertValidationMessages(expectedMessages, result)
+    self.assertValidationResult(expectedItems, result)
 
   def test_validationCardinalityALL(self):
     servicesInfo = [
@@ -119,10 +119,10 @@ class TestHDP206StackAdvisor(TestCase):
     hosts = self.prepareHosts(["host1", "host2"])
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
-    expectedMessages = [
-      "Cardinality violation, cardinality=ALL, hosts count=1"
+    expectedItems = [
+      {"message": "Cardinality violation, cardinality=ALL, hosts count=1"}
     ]
-    self.assertValidationMessages(expectedMessages, result)
+    self.assertValidationResult(expectedItems, result)
 
   def test_validationHostIsNotUsedForNonValuableComponent(self):
     servicesInfo = [
@@ -138,10 +138,10 @@ class TestHDP206StackAdvisor(TestCase):
     hosts = self.prepareHosts(["host1", "host2"])
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
-    expectedMessages = [
-      "Host is not used"
+    expectedItems = [
+      {"message": "Host is not used", "host": "host1"}
     ]
-    self.assertValidationMessages(expectedMessages, result)
+    self.assertValidationResult(expectedItems, result)
 
   def test_validationCardinality01TwoHostsAssigned(self):
     servicesInfo = [
@@ -156,10 +156,10 @@ class TestHDP206StackAdvisor(TestCase):
     hosts = self.prepareHosts(["host1", "host2"])
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
-    expectedMessages = [
-      "Cardinality violation, cardinality=0-1, hosts count=2"
+    expectedItems = [
+      {"message": "Cardinality violation, cardinality=0-1, hosts count=2"}
     ]
-    self.assertValidationMessages(expectedMessages, result)
+    self.assertValidationResult(expectedItems, result)
 
   def test_validationHostIsNotUsed(self):
     servicesInfo = [
@@ -174,10 +174,10 @@ class TestHDP206StackAdvisor(TestCase):
     hosts = self.prepareHosts(["host1", "host2"])
     result = self.stackAdvisor.validateComponentLayout(services, hosts)
 
-    expectedMessages = [
-      "Host is not used"
+    expectedItems = [
+      {"message": "Host is not used", "host": "host2"}
     ]
-    self.assertValidationMessages(expectedMessages, result)
+    self.assertValidationResult(expectedItems, result)
 
 
   def prepareHosts(self, hostsNames):
@@ -241,8 +241,15 @@ class TestHDP206StackAdvisor(TestCase):
     if not len(l1) == len(l2) or not sorted(l1) == sorted(l2):
       raise AssertionError("list1={0}, list2={1}".format(l1, l2))
 
-  def assertValidationMessages(self, expectedMessages, result):
-    realMessages = [item["message"] for item in result["items"]]
-    self.checkEqual(expectedMessages, realMessages)
+  def assertValidationResult(self, expectedItems, result):
+    actualItems = []
+    for item in result["items"]:
+      next = { "message": item["message"] }
+      try:
+        next["host"] = item["host"]
+      except KeyError, err:
+        pass
+      actualItems.append(next)
+    self.checkEqual(expectedItems, actualItems)
 
 


Mime
View raw message