ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [2/2] git commit: AMBARI-7953. Modify HDP-2.2 stack advisor for the tez config properties (aonishuk)
Date Fri, 24 Oct 2014 18:15:30 GMT
AMBARI-7953. Modify HDP-2.2 stack advisor for the tez config properties (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d70bc04
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d70bc04
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d70bc04

Branch: refs/heads/branch-1.7.0
Commit: 6d70bc044bc519ba8bf488a54bc51b3c1d8aa2c3
Parents: 7f82db5
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Fri Oct 24 21:15:21 2014 +0300
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Fri Oct 24 21:15:21 2014 +0300

----------------------------------------------------------------------
 .../stacks/HDP/2.2/services/stack_advisor.py    | 18 ++++++
 .../stacks/2.2/common/test_stack_advisor.py     | 66 ++++++++++++++++++++
 2 files changed, 84 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6d70bc04/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 19b1065..b291530 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -23,6 +23,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     parentRecommendConfDict = super(HDP22StackAdvisor, self).getServiceConfigurationRecommenderDict()
     childRecommendConfDict = {
       "HDFS": self.recommendHDFSConfigurations,
+      "TEZ": self.recommendTezConfigurations
     }
     parentRecommendConfDict.update(childRecommendConfDict)
     return parentRecommendConfDict
@@ -31,14 +32,31 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
   def recommendHDFSConfigurations(self, configurations, clusterData):
     self.putProperty(configurations, "hdfs-site")
 
+  def recommendTezConfigurations(self, configurations, clusterData):
+    putTezProperty = self.putProperty(configurations, "tez-site")
+    putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']) * 2 if int(clusterData['amMemory'])
< 3072 else int(clusterData['amMemory']))
+
+    taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else
int(clusterData['reduceMemory'])
+    taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'],
taskResourceMemory)
+    putTezProperty("tez.task.resource.memory.mb", taskResourceMemory)
+    putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4) if int(taskResourceMemory
* 0.4) <= 2147483644 else 2147483644)
+    putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory
* 0.075))
+
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP22StackAdvisor, self).getServiceConfigurationValidators()
     childValidators = {
       "HDFS": ["hdfs-site", self.validateHDFSConfigurations],
+      "TEZ": ["tez-site", self.validateTezConfigurations]
     }
     parentValidators.update(childValidators)
     return parentValidators
 
+  def validateTezConfigurations(self, properties, recommendedDefaults, configurations):
+    validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.am.resource.memory.mb')},
+                        {"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.task.resource.memory.mb')},
+                        {"config-name": 'tez.runtime.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.runtime.io.sort.mb')},
+                        {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item":
self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},]
+    return self.toConfigurationValidationProblems(validationItems, "tez-site")
 
   def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations):
     # We can not access property hadoop.security.authentication from the

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d70bc04/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 3d6b2e6..06c476e 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -42,6 +42,72 @@ class TestHDP22StackAdvisor(TestCase):
     clazz = getattr(stack_advisor_impl, hdp22StackAdvisorClassName)
     self.stackAdvisor = clazz()
 
+  def test_recommendTezConfigurations(self):
+    configurations = {}
+    clusterData = {
+      "mapMemory": 3000,
+      "amMemory": 2000,
+      "reduceMemory": 2056,
+      "containers": 3,
+      "ramPerContainer": 256
+    }
+    expected = {
+      "tez-site": {
+        "properties": {
+          "tez.am.resource.memory.mb": "4000",
+          "tez.task.resource.memory.mb": "768",
+          "tez.runtime.io.sort.mb": "307",
+          "tez.runtime.unordered.output.buffer.size-mb": "57"
+        }
+      }
+    }
+    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData)
+    self.assertEquals(configurations, expected)
+
+  def test_recommendTezConfigurations_amMemoryMoreThan3072(self):
+    configurations = {}
+    clusterData = {
+      "mapMemory": 4000,
+      "amMemory": 3100,
+      "reduceMemory": 2056,
+      "containers": 3,
+      "ramPerContainer": 256
+    }
+    expected = {
+      "tez-site": {
+        "properties": {
+          "tez.am.resource.memory.mb": "3100",
+          "tez.task.resource.memory.mb": "768",
+          "tez.runtime.io.sort.mb": "307",
+          "tez.runtime.unordered.output.buffer.size-mb": "57"
+        }
+      }
+    }
+    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData)
+    self.assertEquals(configurations, expected)
+
+  def test_recommendTezConfigurations_mapMemoryLessThan768(self):
+    configurations = {}
+    clusterData = {
+      "mapMemory": 760,
+      "amMemory": 2000,
+      "reduceMemory": 760,
+      "containers": 3,
+      "ramPerContainer": 256
+    }
+    expected = {
+      "tez-site": {
+        "properties": {
+          "tez.am.resource.memory.mb": "4000",
+          "tez.task.resource.memory.mb": "760",
+          "tez.runtime.io.sort.mb": "304",
+          "tez.runtime.unordered.output.buffer.size-mb": "57"
+        }
+      }
+    }
+    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData)
+    self.assertEquals(configurations, expected)
+
 
   def test_validateHDFSConfigurations(self):
     self.maxDiff = None


Mime
View raw message